diff --git a/py/routes/lora_routes.py b/py/routes/lora_routes.py index 5f278805..cb99b66f 100644 --- a/py/routes/lora_routes.py +++ b/py/routes/lora_routes.py @@ -348,7 +348,7 @@ class LoraRoutes(BaseModelRoutes): # Store creator in the civitai nested structure metadata['civitai']['creator'] = creator - await MetadataManager.save_metadata(file_path, metadata, True) + await MetadataManager.save_metadata(file_path, metadata) except Exception as e: logger.error(f"Error saving model metadata: {e}") diff --git a/py/services/download_manager.py b/py/services/download_manager.py index db3c1e14..c89a9c90 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -491,7 +491,7 @@ class DownloadManager: metadata.update_file_info(save_path) # 5. Final metadata update - await MetadataManager.save_metadata(save_path, metadata, True) + await MetadataManager.save_metadata(save_path, metadata) # 6. Update cache based on model type if model_type == "checkpoint": diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index 4b6c391a..9eef1ca6 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -585,6 +585,7 @@ class ModelScanner: if entry.is_file(follow_symlinks=True) and any(entry.name.endswith(ext) for ext in self.file_extensions): file_path = entry.path.replace(os.sep, "/") result = await self._process_model_file(file_path, original_root) + # Only add to models if result is not None (skip corrupted metadata) if result: models.append(result) await asyncio.sleep(0) @@ -624,7 +625,12 @@ class ModelScanner: async def _process_model_file(self, file_path: str, root_path: str) -> Dict: """Process a single model file and return its metadata""" - metadata = await MetadataManager.load_metadata(file_path, self.model_class) + metadata, should_skip = await MetadataManager.load_metadata(file_path, self.model_class) + + if should_skip: + # Metadata file exists but cannot be parsed - skip this model + logger.warning(f"Skipping model {file_path} due to corrupted metadata file") + return None if metadata is None: civitai_info_path = f"{os.path.splitext(file_path)[0]}.civitai.info" @@ -640,7 +646,7 @@ class ModelScanner: metadata = self.model_class.from_civitai_info(version_info, file_info, file_path) metadata.preview_url = find_preview_file(file_name, os.path.dirname(file_path)) - await MetadataManager.save_metadata(file_path, metadata, True) + await MetadataManager.save_metadata(file_path, metadata) logger.debug(f"Created metadata from .civitai.info for {file_path}") except Exception as e: logger.error(f"Error creating metadata from .civitai.info for {file_path}: {e}") @@ -667,7 +673,7 @@ class ModelScanner: metadata.modelDescription = version_info['model']['description'] # Save the updated metadata - await MetadataManager.save_metadata(file_path, metadata, True) + await MetadataManager.save_metadata(file_path, metadata) logger.debug(f"Updated metadata with civitai info for {file_path}") except Exception as e: logger.error(f"Error restoring civitai data from .civitai.info for {file_path}: {e}") @@ -747,7 +753,7 @@ class ModelScanner: model_data['civitai']['creator'] = model_metadata['creator'] - await MetadataManager.save_metadata(file_path, model_data, True) + await MetadataManager.save_metadata(file_path, model_data) except Exception as e: logger.error(f"Failed to update metadata from Civitai for {file_path}: {e}") diff --git a/py/utils/metadata_manager.py b/py/utils/metadata_manager.py index 15ab4b1f..9eaa890f 100644 --- a/py/utils/metadata_manager.py +++ b/py/utils/metadata_manager.py @@ -1,7 +1,6 @@ from datetime import datetime import os import json -import shutil import logging from typing import Dict, Optional, Type, Union @@ -17,7 +16,7 @@ class MetadataManager: This class is responsible for: 1. Loading metadata safely with fallback mechanisms - 2. Saving metadata with atomic operations and backups + 2. Saving metadata with atomic operations 3. Creating default metadata for models 4. Handling unknown fields gracefully """ @@ -25,81 +24,44 @@ class MetadataManager: @staticmethod async def load_metadata(file_path: str, model_class: Type[BaseModelMetadata] = LoraMetadata) -> Optional[BaseModelMetadata]: """ - Load metadata with robust error handling and data preservation. + Load metadata safely. - Args: - file_path: Path to the model file - model_class: Class to instantiate (LoraMetadata, CheckpointMetadata, etc.) - Returns: - BaseModelMetadata instance or None if file doesn't exist + tuple: (metadata, should_skip) + - metadata: BaseModelMetadata instance or None + - should_skip: True if corrupted metadata file exists and model should be skipped """ metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" - backup_path = f"{metadata_path}.bak" - # Try loading the main metadata file - if os.path.exists(metadata_path): - try: - with open(metadata_path, 'r', encoding='utf-8') as f: - data = json.load(f) - - # Create model instance - metadata = model_class.from_dict(data) - - # Normalize paths - await MetadataManager._normalize_metadata_paths(metadata, file_path) - - return metadata - - except json.JSONDecodeError: - # JSON parsing error - try to restore from backup - logger.warning(f"Invalid JSON in metadata file: {metadata_path}") - return await MetadataManager._restore_from_backup(backup_path, file_path, model_class) - - except Exception as e: - # Other errors might be due to unknown fields or schema changes - logger.error(f"Error loading metadata from {metadata_path}: {str(e)}") - return await MetadataManager._restore_from_backup(backup_path, file_path, model_class) + # Check if metadata file exists + if not os.path.exists(metadata_path): + return None, False - return None - - @staticmethod - async def _restore_from_backup(backup_path: str, file_path: str, model_class: Type[BaseModelMetadata]) -> Optional[BaseModelMetadata]: - """ - Try to restore metadata from backup file - - Args: - backup_path: Path to backup file - file_path: Path to the original model file - model_class: Class to instantiate + try: + with open(metadata_path, 'r', encoding='utf-8') as f: + data = json.load(f) - Returns: - BaseModelMetadata instance or None if restoration fails - """ - if os.path.exists(backup_path): - try: - logger.info(f"Attempting to restore metadata from backup: {backup_path}") - with open(backup_path, 'r', encoding='utf-8') as f: - data = json.load(f) + # Create model instance + metadata = model_class.from_dict(data) - # Process data similarly to normal loading - metadata = model_class.from_dict(data) - await MetadataManager._normalize_metadata_paths(metadata, file_path) - return metadata - except Exception as e: - logger.error(f"Failed to restore from backup: {str(e)}") - - return None + # Normalize paths + await MetadataManager._normalize_metadata_paths(metadata, file_path) + + return metadata, False + + except (json.JSONDecodeError, Exception) as e: + error_type = "Invalid JSON" if isinstance(e, json.JSONDecodeError) else "Parse error" + logger.error(f"{error_type} in metadata file: {metadata_path}. Error: {str(e)}. Skipping model to preserve existing data.") + return None, True # should_skip = True @staticmethod - async def save_metadata(path: str, metadata: Union[BaseModelMetadata, Dict], create_backup: bool = False) -> bool: + async def save_metadata(path: str, metadata: Union[BaseModelMetadata, Dict]) -> bool: """ - Save metadata with atomic write operations and backup creation. + Save metadata with atomic write operations. Args: path: Path to the model file or directly to the metadata file metadata: Metadata to save (either BaseModelMetadata object or dict) - create_backup: Whether to create a new backup of existing file if a backup doesn't already exist Returns: bool: Success or failure @@ -112,19 +74,8 @@ class MetadataManager: file_path = path metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" temp_path = f"{metadata_path}.tmp" - backup_path = f"{metadata_path}.bak" try: - # Create backup if file exists and either: - # 1. create_backup is True, OR - # 2. backup file doesn't already exist - if os.path.exists(metadata_path) and (create_backup or not os.path.exists(backup_path)): - try: - shutil.copy2(metadata_path, backup_path) - logger.debug(f"Created metadata backup at: {backup_path}") - except Exception as e: - logger.warning(f"Failed to create metadata backup: {str(e)}") - # Convert to dict if needed if isinstance(metadata, BaseModelMetadata): metadata_dict = metadata.to_dict() @@ -240,7 +191,7 @@ class MetadataManager: # await MetadataManager._enrich_metadata(metadata, real_path) # Save the created metadata - await MetadataManager.save_metadata(file_path, metadata, create_backup=False) + await MetadataManager.save_metadata(file_path, metadata) return metadata @@ -310,4 +261,4 @@ class MetadataManager: # If path attributes were changed, save the metadata back to disk if need_update: - await MetadataManager.save_metadata(file_path, metadata, create_backup=False) + await MetadataManager.save_metadata(file_path, metadata) diff --git a/py/utils/routes_common.py b/py/utils/routes_common.py index e961d81c..1f32b369 100644 --- a/py/utils/routes_common.py +++ b/py/utils/routes_common.py @@ -156,7 +156,7 @@ class ModelRouteUtils: local_metadata['preview_nsfw_level'] = first_preview.get('nsfwLevel', 0) # Save updated metadata - await MetadataManager.save_metadata(metadata_path, local_metadata, True) + await MetadataManager.save_metadata(metadata_path, local_metadata) @staticmethod async def fetch_and_update_model(