From 297d8787bd80adfabdad9dc2aaabab059e13bf76 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 26 Sep 2026 09:02:37 +0800 Subject: [PATCH 1/4] refactor: route sidecar/preview path derivation through sidecar_paths helpers Phase 1 of #1045 (optional centralized sidecar storage): introduce py/utils/sidecar_paths.py as the single place that resolves .metadata.json and preview locations, and replace all inline splitext-based derivations across scanners, services, download manager, and route handlers. No behavior change: the default 'alongside' storage mode resolves every path exactly as before. .civitai.info (third-party sidecar) derivation is intentionally left co-located. --- py/metadata_ops/__init__.py | 3 +- py/routes/handlers/misc_handlers.py | 29 ++- py/routes/handlers/model_handlers.py | 3 +- py/services/autov3_backfill_service.py | 4 +- py/services/download_manager.py | 30 ++- py/services/metadata_sync_service.py | 11 +- py/services/model_lifecycle_service.py | 17 +- py/services/model_scanner.py | 9 +- py/services/pending_delete_service.py | 6 +- py/services/preview_asset_service.py | 5 +- py/services/tag_update_service.py | 6 +- .../use_cases/filename_template_use_case.py | 3 +- py/utils/metadata_manager.py | 18 +- py/utils/sidecar_paths.py | 215 ++++++++++++++++++ 14 files changed, 302 insertions(+), 57 deletions(-) create mode 100644 py/utils/sidecar_paths.py diff --git a/py/metadata_ops/__init__.py b/py/metadata_ops/__init__.py index a83356d8..40781ff4 100644 --- a/py/metadata_ops/__init__.py +++ b/py/metadata_ops/__init__.py @@ -172,12 +172,13 @@ async def download_preview( """ from ..services.downloader import get_downloader from ..utils.exif_utils import ExifUtils + from ..utils.sidecar_paths import get_preview_dir if not url or not url.strip(): return None base_name = os.path.splitext(os.path.basename(model_path))[0] - preview_dir = os.path.dirname(model_path) + preview_dir = get_preview_dir(model_path) output_path = os.path.join(preview_dir, base_name + ".webp") downloader = await get_downloader() diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index f3021935..8e156f6a 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -68,6 +68,7 @@ from ...utils.example_images_paths import ( ) from ...utils.lora_metadata import extract_trained_words from ...utils.session_logging import get_standalone_session_log_snapshot +from ...utils.sidecar_paths import get_metadata_path, get_preview_dir from ...utils.usage_stats import UsageStats from .base_model_handlers import BaseModelHandlerSet @@ -943,15 +944,24 @@ class DoctorHandler: os.rename(path, new_path) - for suffix in (".metadata.json", ".civitai.info"): - old_sidecar = old_base_no_ext + suffix - new_sidecar = new_base_no_ext + suffix - if os.path.exists(old_sidecar): - os.rename(old_sidecar, new_sidecar) + old_metadata_path = get_metadata_path(path) + new_metadata_path = get_metadata_path(new_path) + if os.path.exists(old_metadata_path): + os.rename(old_metadata_path, new_metadata_path) + + old_sidecar = old_base_no_ext + ".civitai.info" + new_sidecar = new_base_no_ext + ".civitai.info" + if os.path.exists(old_sidecar): + os.rename(old_sidecar, new_sidecar) for preview_ext in PREVIEW_EXTENSIONS: - old_preview = old_base_no_ext + preview_ext - new_preview = new_base_no_ext + preview_ext + old_preview = os.path.join( + get_preview_dir(path), base_name + preview_ext + ) + new_preview = os.path.join( + get_preview_dir(new_path), + candidate_base + preview_ext, + ) if os.path.exists(old_preview): os.rename(old_preview, new_preview) @@ -963,7 +973,10 @@ class DoctorHandler: old_preview_url = entry["preview_url"].replace("\\", "/") preview_ext = os.path.splitext(old_preview_url)[1] if preview_ext: - entry["preview_url"] = (new_base_no_ext + preview_ext).replace(os.sep, "/") + entry["preview_url"] = os.path.join( + get_preview_dir(new_path), + candidate_base + preview_ext, + ).replace(os.sep, "/") await scanner.update_single_model_cache( path, new_path, entry ) diff --git a/py/routes/handlers/model_handlers.py b/py/routes/handlers/model_handlers.py index 2968fbf8..43d7cbc2 100644 --- a/py/routes/handlers/model_handlers.py +++ b/py/routes/handlers/model_handlers.py @@ -50,6 +50,7 @@ from ...services.errors import RateLimitError, ResourceNotFoundError from ...utils.civitai_utils import resolve_license_payload from ...utils.file_utils import calculate_sha256 from ...utils.metadata_manager import MetadataManager +from ...utils.sidecar_paths import get_metadata_path from ...utils.url_utils import relative_root_prefix LICENSE_FIELDS = ( @@ -676,7 +677,7 @@ class ModelManagementHandler: status=400, ) - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) local_metadata = await self._metadata_sync.load_local_metadata( metadata_path ) diff --git a/py/services/autov3_backfill_service.py b/py/services/autov3_backfill_service.py index 60ea965d..b6778992 100644 --- a/py/services/autov3_backfill_service.py +++ b/py/services/autov3_backfill_service.py @@ -27,6 +27,8 @@ import os import threading from typing import TYPE_CHECKING, Optional +from ..utils.sidecar_paths import get_metadata_path + if TYPE_CHECKING: # pragma: no cover - type-check only; runtime imports are local from .model_scanner import ModelScanner @@ -41,7 +43,7 @@ def _resolve_autov3(file_path: str) -> str: safetensors header hash. Returns ``''`` when neither is available. """ try: - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) if os.path.exists(metadata_path): with open(metadata_path, "r", encoding="utf-8") as handle: payload = json.load(handle) diff --git a/py/services/download_manager.py b/py/services/download_manager.py index 82164a5d..52518ec7 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -38,6 +38,7 @@ from ..utils.preview_selection import resolve_mature_threshold, select_preview_m from ..utils.utils import calculate_filename_for_model, sanitize_folder_name from ..utils.exif_utils import ExifUtils from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir from .service_registry import ServiceRegistry from .download_routing import is_diffusion_model_download, resolve_other_download_sub_type from .settings_manager import get_settings_manager @@ -831,7 +832,7 @@ class DownloadManager: ) for file_path in target_files: - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) deleted = await self._delete_file_with_retries(metadata_path) if not deleted and os.path.exists(metadata_path): logger.error(f"Error deleting metadata file: {metadata_path}") @@ -2447,7 +2448,7 @@ class DownloadManager: return {"success": False, "error": save_path} part_path = save_path + ".part" - metadata_path = os.path.splitext(save_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(save_path) pause_control = self._pause_events.get(download_id) if download_id else None @@ -2503,7 +2504,10 @@ class DownloadManager: if media_type == "video": preview_ext = _extension_from_url(preview_url, ".mp4") - preview_path = os.path.splitext(save_path)[0] + preview_ext + preview_path = os.path.join( + get_preview_dir(save_path), + os.path.splitext(os.path.basename(save_path))[0] + preview_ext, + ) rewritten_url, rewritten = rewrite_preview_url( preview_url, media_type="video" ) @@ -2530,7 +2534,10 @@ class DownloadManager: ) if rewritten and rewritten_url: preview_ext = _extension_from_url(preview_url, ".png") - preview_path = os.path.splitext(save_path)[0] + preview_ext + preview_path = os.path.join( + get_preview_dir(save_path), + os.path.splitext(os.path.basename(save_path))[0] + preview_ext, + ) success, _ = await downloader.download_file( rewritten_url, preview_path, use_auth=False ) @@ -2557,8 +2564,9 @@ class DownloadManager: temp_file_handle.write( content if isinstance(content, bytes) else content.encode("utf-8") ) - preview_path = ( - os.path.splitext(save_path)[0] + ".webp" + preview_path = os.path.join( + get_preview_dir(save_path), + os.path.splitext(os.path.basename(save_path))[0] + ".webp", ) optimized_data, _ = ExifUtils.optimize_image( @@ -2788,9 +2796,7 @@ class DownloadManager: entry = cast(Any, adjusted_entry) metadata_entries[index] = entry - metadata_file_path = ( - os.path.splitext(entry.file_path)[0] + ".metadata.json" - ) + metadata_file_path = get_metadata_path(entry.file_path) metadata_files_for_cleanup.append(metadata_file_path) await MetadataManager.save_metadata(entry.file_path, entry) @@ -3049,7 +3055,11 @@ class DownloadManager: extension = os.path.splitext(preview_path)[1] or ".webp" targets = [ - os.path.splitext(entry.file_path)[0] + extension for entry in entries + os.path.join( + get_preview_dir(entry.file_path), + os.path.splitext(os.path.basename(entry.file_path))[0] + extension, + ) + for entry in entries ] if not targets: diff --git a/py/services/metadata_sync_service.py b/py/services/metadata_sync_service.py index 92262a0e..337ec381 100644 --- a/py/services/metadata_sync_service.py +++ b/py/services/metadata_sync_service.py @@ -12,6 +12,7 @@ from ..services.settings_manager import SettingsManager from ..utils.civitai_utils import resolve_license_payload from ..utils.model_utils import determine_base_model from ..utils.models import autov3_from_civitai_files +from ..utils.sidecar_paths import get_metadata_path from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error from .errors import RateLimitError from .model_sources import has_external_source @@ -216,7 +217,7 @@ class MetadataSyncService: logger.error(error) return False, error - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) enable_archive = self._settings.get("enable_metadata_archive_db", False) previous_source = model_data.get("metadata_source") or (model_data.get("civitai") or {}).get("source") @@ -485,7 +486,7 @@ class MetadataSyncService: + (f" with version: {model_version_id}" if model_version_id else "") ) - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) await self.update_model_metadata( metadata_path, metadata, @@ -505,7 +506,7 @@ class MetadataSyncService: ) -> Dict[str, Any]: """Apply metadata updates and persist to disk and cache.""" - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) metadata = await metadata_loader(metadata_path) for key, value in updates.items(): @@ -554,7 +555,7 @@ class MetadataSyncService: } expected_hash: Optional[str] = None - first_metadata_path = os.path.splitext(file_paths[0])[0] + ".metadata.json" + first_metadata_path = get_metadata_path(file_paths[0]) first_metadata = await metadata_loader(first_metadata_path) if first_metadata and "sha256" in first_metadata: expected_hash = first_metadata["sha256"].lower() @@ -565,7 +566,7 @@ class MetadataSyncService: try: actual_hash = await hash_calculator(path) - metadata_path = os.path.splitext(path)[0] + ".metadata.json" + metadata_path = get_metadata_path(path) metadata = await metadata_loader(metadata_path) stored_hash = metadata.get("sha256", "").lower() diff --git a/py/services/model_lifecycle_service.py b/py/services/model_lifecycle_service.py index f180baec..bb59f97f 100644 --- a/py/services/model_lifecycle_service.py +++ b/py/services/model_lifecycle_service.py @@ -11,6 +11,7 @@ from ..services.service_registry import ServiceRegistry from ..services.pending_delete_service import get_pending_delete_service from ..utils.constants import PREVIEW_EXTENSIONS from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_metadata_path logger = logging.getLogger(__name__) @@ -45,7 +46,10 @@ async def delete_model_artifacts( main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [main_file, f"{file_name}.metadata.json"] + patterns = [ + main_file, + os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), + ] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") @@ -260,7 +264,7 @@ class ModelLifecycleService: _require_path_in_library_roots(file_path, self._scanner, label="File path") - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) metadata = await self._metadata_loader(metadata_path) metadata["exclude"] = True @@ -315,7 +319,7 @@ class ModelLifecycleService: if not os.path.exists(file_path): raise ValueError("Model file does not exist") - metadata_path = os.path.splitext(file_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(file_path) metadata_payload = await self._metadata_loader(metadata_path) metadata_payload["exclude"] = False @@ -384,10 +388,11 @@ class ModelLifecycleService: if os.path.exists(new_file_path): raise ValueError("A file with this name already exists") + metadata_filename = os.path.basename(get_metadata_path(file_path)) patterns = [ f"{old_file_name}{old_extension}", - f"{old_file_name}.metadata.json", - f"{old_file_name}.metadata.json.bak", + metadata_filename, + f"{metadata_filename}.bak", ] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{old_file_name}{ext}") @@ -398,7 +403,7 @@ class ModelLifecycleService: if os.path.exists(path): existing_files.append((path, pattern)) - metadata_path = os.path.join(target_dir, f"{old_file_name}.metadata.json") + metadata_path = get_metadata_path(file_path) metadata: Optional[Dict[str, object]] = None hash_value: Optional[str] = None diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index 542671ae..a8525165 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -11,6 +11,7 @@ from ..utils.models import BaseModelMetadata, autov3_from_civitai_files from ..config import config from ..utils.file_utils import find_preview_file, get_preview_extension, calculate_sha256, calculate_autov3 from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir from ..utils.civitai_utils import resolve_license_info from .model_cache import ModelCache from .model_hash_index import ModelHashIndex @@ -1709,7 +1710,7 @@ class ModelScanner: file_path = item.get("file_path") if not file_path: continue - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) if not os.path.exists(metadata_path): continue try: @@ -2339,7 +2340,7 @@ class ModelScanner: target_associated_path = os.path.join(target_path, new_associated_filename) # Store metadata file path for special handling - if file == f"{base_name}.metadata.json": + if file == os.path.basename(get_metadata_path(source_path)): source_metadata = source_file_path moved_metadata_path = target_associated_path else: @@ -2399,7 +2400,7 @@ class ModelScanner: metadata['file_name'] = os.path.splitext(os.path.basename(model_path))[0] if 'preview_url' in metadata and metadata['preview_url']: - preview_dir = os.path.dirname(model_path) + preview_dir = get_preview_dir(model_path) # Update preview filename to match the new base name new_base_name = os.path.splitext(os.path.basename(model_path))[0] preview_ext = get_preview_extension(metadata['preview_url']) @@ -2759,7 +2760,7 @@ class ModelScanner: # Sidecar write-back: JSON null encodes the checked-unavailable # state. Skip silently when the sidecar does not exist. - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) if os.path.exists(metadata_path): with open(metadata_path, 'r', encoding='utf-8') as handle: payload = json.load(handle) diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index 39d36ea8..70bc0746 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -38,6 +38,7 @@ from typing import ( ) from ..utils.constants import PREVIEW_EXTENSIONS +from ..utils.sidecar_paths import get_metadata_path from ..utils import settings_paths logger = logging.getLogger(__name__) @@ -669,7 +670,10 @@ class PendingDeleteService: """Enumerate existing artifacts exactly like delete_model_artifacts.""" main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [main_file, f"{file_name}.metadata.json"] + patterns = [ + main_file, + os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), + ] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") diff --git a/py/services/preview_asset_service.py b/py/services/preview_asset_service.py index ac84ec5e..1e215347 100644 --- a/py/services/preview_asset_service.py +++ b/py/services/preview_asset_service.py @@ -10,6 +10,7 @@ from urllib.parse import urlparse from ..utils.constants import CARD_PREVIEW_WIDTH, PREVIEW_EXTENSIONS from ..utils.civitai_utils import rewrite_preview_url from ..utils.preview_selection import resolve_mature_threshold, select_preview_media +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir from .settings_manager import get_settings_manager logger = logging.getLogger(__name__) @@ -159,7 +160,7 @@ class PreviewAssetService: """Replace an existing preview asset for a model.""" base_name = os.path.splitext(os.path.basename(model_path))[0] - folder = os.path.dirname(model_path) + folder = get_preview_dir(model_path) extension, optimized_data = await self._convert_preview( preview_data, content_type, original_filename @@ -179,7 +180,7 @@ class PreviewAssetService: with open(preview_path, "wb") as handle: handle.write(optimized_data) - metadata_path = os.path.splitext(model_path)[0] + ".metadata.json" + metadata_path = get_metadata_path(model_path) metadata = await metadata_loader(metadata_path) metadata["preview_url"] = preview_path metadata["preview_nsfw_level"] = nsfw_level diff --git a/py/services/tag_update_service.py b/py/services/tag_update_service.py index c1081384..d87ab221 100644 --- a/py/services/tag_update_service.py +++ b/py/services/tag_update_service.py @@ -2,10 +2,9 @@ from __future__ import annotations -import os - from typing import Awaitable, Callable, Dict, List, Sequence, Tuple +from ..utils.sidecar_paths import get_metadata_path from .auto_tag_service import extract_auto_tags @@ -24,8 +23,7 @@ class TagUpdateService: update_cache: Callable[[str, str, Dict[str, object]], Awaitable[bool]], ) -> Tuple[List[str], List[str]]: """Add tags to a metadata entry and return updated tags and auto_tags.""" - base, _ = os.path.splitext(file_path) - metadata_path = f"{base}.metadata.json" + metadata_path = get_metadata_path(file_path) metadata = await metadata_loader(metadata_path) raw_tags = metadata.get("tags", []) diff --git a/py/services/use_cases/filename_template_use_case.py b/py/services/use_cases/filename_template_use_case.py index 0a0813ef..e569d681 100644 --- a/py/services/use_cases/filename_template_use_case.py +++ b/py/services/use_cases/filename_template_use_case.py @@ -12,6 +12,7 @@ import os from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence from ...utils.constants import AUTO_ORGANIZE_BATCH_SIZE +from ...utils.sidecar_paths import get_metadata_path from ...utils.utils import calculate_filename_for_model from ..model_file_service import AutoOrganizeResult, ProgressCallback from ..model_lifecycle_service import ModelLifecycleService, load_local_metadata @@ -200,7 +201,7 @@ class FilenameTemplateUseCase: sidecar or no ``original_file_name`` entry exists (models never renamed, or renamed before the recording shipped). """ - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) metadata = await self._metadata_loader(metadata_path) original = metadata.get("original_file_name") if not isinstance(original, str): diff --git a/py/utils/metadata_manager.py b/py/utils/metadata_manager.py index 43cfe976..1b03a813 100644 --- a/py/utils/metadata_manager.py +++ b/py/utils/metadata_manager.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Optional, Type, Union, cast from .models import BaseModelMetadata, CheckpointMetadata, EmbeddingMetadata, LoraMetadata from .file_utils import normalize_path, find_preview_file, calculate_sha256, calculate_autov3 from .lora_metadata import extract_lora_metadata, extract_checkpoint_metadata +from .sidecar_paths import get_metadata_path, resolve_metadata_path logger = logging.getLogger(__name__) @@ -32,7 +33,7 @@ class MetadataManager: - metadata: BaseModelMetadata instance or None - should_skip: True if corrupted metadata file exists and model should be skipped """ - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) # Check if metadata file exists if not os.path.exists(metadata_path): @@ -98,11 +99,7 @@ class MetadataManager: payload.update(unknown_fields) else: if not should_skip: - metadata_path = ( - file_path - if file_path.endswith(".metadata.json") - else f"{os.path.splitext(file_path)[0]}.metadata.json" - ) + metadata_path = resolve_metadata_path(file_path) if os.path.exists(metadata_path): try: with open(metadata_path, "r", encoding="utf-8") as handle: @@ -150,7 +147,7 @@ class MetadataManager: return model_data folder = model_data.get("folder") - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = get_metadata_path(file_path) sidecar_exists = os.path.exists(metadata_path) cached = model_data.copy() payload = await MetadataManager.load_metadata_payload(file_path) @@ -188,12 +185,7 @@ class MetadataManager: bool: Success or failure """ # Determine if the input is a metadata path or a model file path - if path.endswith('.metadata.json'): - metadata_path = path - else: - # Use existing logic for model file paths - file_path = path - metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json" + metadata_path = resolve_metadata_path(path) temp_path = f"{metadata_path}.tmp" try: diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py new file mode 100644 index 00000000..05d89e2e --- /dev/null +++ b/py/utils/sidecar_paths.py @@ -0,0 +1,215 @@ +"""Resolution of sidecar metadata and preview storage paths. + +All code that needs the on-disk location of a model's ``.metadata.json`` +sidecar or preview assets MUST go through these helpers instead of deriving +paths inline (``splitext(model_path)[0] + ".metadata.json"`` and friends). + +Two storage modes are supported, selected by the ``sidecar_storage_mode`` +setting: + +- ``alongside`` (default): sidecars and previews live next to the model + file, the historical layout other tools may rely on. +- ``centralized``: sidecars and previews live under a configurable root + (``sidecar_storage_path`` setting, default ``/sidecars``), + mirroring the library-relative directory structure: + ``////.metadata.json``. + +All helpers are pure path computations: no directory scans and no file I/O +on the hot path. Settings lookups go through ``SettingsManager.get`` (a dict +read); config roots come from the already-initialized ``config`` singleton. +""" + +from __future__ import annotations + +import logging +import os +import re +from typing import List, Optional + +logger = logging.getLogger(__name__) + +METADATA_SUFFIX = ".metadata.json" + +STORAGE_MODE_ALONGSIDE = "alongside" +STORAGE_MODE_CENTRALIZED = "centralized" + +_VALID_MODES = frozenset({STORAGE_MODE_ALONGSIDE, STORAGE_MODE_CENTRALIZED}) + + +def _get_settings_value(key: str, default=None): + """Read a setting defensively; never fail path resolution on settings errors.""" + + try: + from ..services.settings_manager import get_settings_manager + + value = get_settings_manager().get(key) + except Exception as exc: # pragma: no cover - defensive fallback + logger.debug("sidecar_paths: settings lookup for %r failed: %s", key, exc) + return default + return default if value is None else value + + +def get_storage_mode() -> str: + """Return the active sidecar storage mode (``alongside`` unless configured).""" + + mode = _get_settings_value("sidecar_storage_mode", STORAGE_MODE_ALONGSIDE) + if mode not in _VALID_MODES: + return STORAGE_MODE_ALONGSIDE + return mode + + +def is_centralized() -> bool: + """Return True when centralized sidecar storage is active and resolvable.""" + + return get_storage_mode() == STORAGE_MODE_CENTRALIZED and bool(get_sidecar_root()) + + +def get_sidecar_root() -> str: + """Return the absolute root directory for centralized sidecar storage. + + Empty string when centralized storage is not usable (mode alongside or an + unresolvable configured path). + """ + + if get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return "" + + configured = _get_settings_value("sidecar_storage_path", "") + if configured and isinstance(configured, str): + root = os.path.abspath(os.path.expanduser(configured.strip())) + if root: + return root + + # Default: /sidecars + try: + from .settings_paths import get_settings_dir + + return os.path.join(get_settings_dir(), "sidecars") + except Exception as exc: # pragma: no cover - defensive fallback + logger.warning("sidecar_paths: cannot resolve default sidecar root: %s", exc) + return "" + + +def sanitize_path_component(name: str) -> str: + """Return a filesystem-safe single path component.""" + + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", name or "") + return safe or "_" + + +def _iter_model_roots() -> List[str]: + """Return every configured model root for the active library.""" + + try: + from ..config import config + except Exception as exc: # pragma: no cover - defensive fallback + logger.debug("sidecar_paths: config unavailable: %s", exc) + return [] + + roots: List[str] = [] + for attr in ( + "loras_roots", + "base_models_roots", + "embeddings_roots", + "other_roots", + "extra_loras_roots", + "extra_checkpoints_roots", + "extra_unet_roots", + "extra_embeddings_roots", + ): + value = getattr(config, attr, None) + if value: + roots.extend(value) + return roots + + +def _normalize_for_match(path: str) -> str: + return os.path.normpath(os.path.abspath(path)) + + +def resolve_centralized_dir(model_path: str) -> Optional[str]: + """Return the centralized mirror directory for ``model_path``. + + The mirror layout is ``///`` + where ``rel_dir`` is the model's directory relative to the model root that + contains it. The longest matching root wins so nested roots resolve to the + most specific mirror. Returns ``None`` when centralized storage is inactive + or the path is not under any configured model root. + """ + + root = get_sidecar_root() + if not root: + return None + + target = _normalize_for_match(model_path) + model_dir = os.path.dirname(target) + + best_root: Optional[str] = None + for candidate in _iter_model_roots(): + if not candidate: + continue + normalized = _normalize_for_match(candidate) + if model_dir == normalized or model_dir.startswith(normalized + os.sep): + if best_root is None or len(normalized) > len(best_root): + best_root = normalized + + if best_root is None: + return None + + try: + from ..services.settings_manager import get_settings_manager + + library = get_settings_manager().get_active_library_name() or "default" + except Exception: # pragma: no cover - defensive fallback + library = "default" + + rel_dir = os.path.relpath(model_dir, best_root) + parts = [root, sanitize_path_component(library), sanitize_path_component(os.path.basename(best_root))] + if rel_dir and rel_dir != os.curdir: + parts.extend(sanitize_path_component(part) for part in rel_dir.split(os.sep) if part not in ("", os.curdir)) + return os.path.join(*parts) + + +def get_sidecar_dir(model_path: str) -> str: + """Return the directory holding the model's sidecar/preview assets. + + Centralized mode falls back to the model's own directory (with a warning) + when the path lies outside every configured model root. + """ + + if get_storage_mode() == STORAGE_MODE_CENTRALIZED: + mirror = resolve_centralized_dir(model_path) + if mirror: + return mirror + logger.warning( + "sidecar_paths: %s is outside configured model roots; storing sidecar alongside", + model_path, + ) + return os.path.dirname(os.path.abspath(model_path)) + + +def get_metadata_path(model_path: str) -> str: + """Return the ``.metadata.json`` sidecar path for a model file.""" + + base_name = os.path.splitext(os.path.basename(model_path))[0] + METADATA_SUFFIX + return os.path.join(get_sidecar_dir(model_path), base_name) + + +def is_metadata_path(path: str) -> bool: + """Return True when ``path`` already points at a metadata sidecar file.""" + + return path.endswith(METADATA_SUFFIX) + + +def resolve_metadata_path(path: str) -> str: + """Accept either a model path or a sidecar path and return the sidecar path.""" + + if is_metadata_path(path): + return path + return get_metadata_path(path) + + +def get_preview_dir(model_path: str) -> str: + """Return the directory holding the model's preview assets.""" + + return get_sidecar_dir(model_path) From f5e983eaaab67e0df7866f5822b561bf993cf630 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 26 Sep 2026 10:00:58 +0800 Subject: [PATCH 2/4] feat: optional centralized storage for sidecar metadata and previews (#1045) Add an opt-in 'centralized' sidecar storage mode alongside the default 'alongside' layout. In centralized mode, .metadata.json sidecars and preview assets live under a configurable root (sidecar_storage_path, default /sidecars), mirroring the library-relative directory structure: ////. Backend: - settings: sidecar_storage_mode / sidecar_storage_path with validation; changing either refreshes the preview allowlist - config: centralized root added to preview-serving allowlist - lifecycle: delete / move / rename / folder-rename / folder-delete and undoable-delete staging all operate on the mirror tree in centralized mode (model files themselves never move); EXDEV-tolerant cross- filesystem moves - scanners: pending-hash filesystem scan walks the mirror tree in centralized mode; preview discovery reads from the sidecar dir; .civitai.info stays co-located in both modes - migration: SidecarMigrationUseCase moves sidecars+previews between layouts both directions (keep-newer conflict resolution, preview_url rewriting, WebSocket progress), exposed as POST+GET /api/lm/sidecars/migrate with a mode guard (force=true for the settings-first flow) Frontend: - settings modal: sidecar storage section (mode select + path input with browse/validation), mode-change confirmation offering immediate migration (force=true), and a 'Migrate Sidecars Now' action - i18n keys synced to all locales ([TODO: Translate] placeholders) Docs: metadata-json-schema.md gains a storage-location section; AGENTS.md records the sidecar_paths helper convention. --- AGENTS.md | 7 + docs/metadata-json-schema.md | 22 + locales/de.json | 27 ++ locales/en.json | 27 ++ locales/es.json | 27 ++ locales/fr.json | 27 ++ locales/he.json | 27 ++ locales/ja.json | 27 ++ locales/ko.json | 27 ++ locales/ru.json | 27 ++ locales/zh-CN.json | 27 ++ locales/zh-TW.json | 27 ++ py/config.py | 20 + py/metadata_ops/__init__.py | 3 + py/routes/handlers/misc_handlers.py | 57 +++ py/routes/misc_route_registrar.py | 7 + py/routes/misc_routes.py | 3 + py/services/checkpoint_scanner.py | 9 +- py/services/download_manager.py | 6 + py/services/model_file_service.py | 16 + py/services/model_lifecycle_service.py | 51 ++- py/services/model_scanner.py | 194 ++++++-- py/services/other_scanner.py | 9 +- py/services/pending_delete_service.py | 33 +- py/services/preview_asset_service.py | 6 + py/services/settings_manager.py | 31 +- py/services/use_cases/__init__.py | 6 + .../use_cases/sidecar_migration_use_case.py | 418 ++++++++++++++++++ py/utils/metadata_manager.py | 20 +- py/utils/sidecar_paths.py | 65 ++- static/js/managers/SettingsManager.js | 192 ++++++++ static/js/state/index.js | 2 + .../components/modals/confirm_modals.html | 14 + .../components/modals/settings/library.html | 56 +++ .../settingsManager.sidecarStorage.test.js | 284 ++++++++++++ tests/routes/test_misc_routes.py | 84 ++++ .../test_centralized_sidecar_storage.py | 361 +++++++++++++++ .../test_sidecar_migration_use_case.py | 294 ++++++++++++ tests/utils/test_sidecar_paths.py | 250 +++++++++++ 39 files changed, 2704 insertions(+), 86 deletions(-) create mode 100644 py/services/use_cases/sidecar_migration_use_case.py create mode 100644 tests/frontend/managers/settingsManager.sidecarStorage.test.js create mode 100644 tests/services/test_centralized_sidecar_storage.py create mode 100644 tests/services/use_cases/test_sidecar_migration_use_case.py create mode 100644 tests/utils/test_sidecar_paths.py diff --git a/AGENTS.md b/AGENTS.md index 09d2b032..87561abe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -266,6 +266,13 @@ If a cross-layer issue ever needs a live server, the sandboxed helpers live in them during migration/import. Never write, modify, or delete them, and never propose doing so as a fix — LoRA Manager's own metadata lives in the `.metadata.json` sidecar it owns. +- **Sidecar/preview path derivation must go through `py/utils/sidecar_paths.py`** + helpers (never inline `splitext + ".metadata.json"`): the centralized storage + mode (`sidecar_storage_mode` / `sidecar_storage_path` settings) relocates + `.metadata.json` files and preview images under a mirror tree, so any + hand-built path is wrong in that mode. `.civitai.info` stays co-located with + the model file in both modes. The new settings keys live only in + `DEFAULT_SETTINGS` — `settings.json.example` stays minimal (see below). - **`settings.json.example` must stay minimal**: only `use_portable_settings`, `civitai_api_key`, and the four core `folder_paths` keys (`loras`, `checkpoints`, `unet`, `embeddings`). Do NOT add optional/default keys diff --git a/docs/metadata-json-schema.md b/docs/metadata-json-schema.md index 235733c6..00672a6a 100644 --- a/docs/metadata-json-schema.md +++ b/docs/metadata-json-schema.md @@ -11,6 +11,28 @@ This document defines the complete schema for `.metadata.json` files used by Lor --- +## Storage Location (Alongside vs Centralized) + +By default, `.metadata.json` sidecars and preview images live **alongside** their model files. An optional centralized mode stores them under a single root directory instead. Two settings control this (Settings → Library → Sidecar Storage): + +| Setting | Values | Default | +|---------|--------|---------| +| `sidecar_storage_mode` | `"alongside"` \| `"centralized"` | `"alongside"` | +| `sidecar_storage_path` | Absolute path string; empty = `/sidecars` | `""` | + +In centralized mode, sidecars and previews mirror the library-relative directory structure: + +``` +////.metadata.json +``` + +- `` is the active library name, `` the basename of the model root containing the file, and `` the model's directory relative to that root. Each component is sanitized to filesystem-safe characters. +- `.civitai.info` files always stay next to the model file, in both modes. +- Changing the mode does **not** move existing files automatically — run the migration (`POST /api/lm/sidecars/migrate` with `{"direction": "to_centralized" | "to_alongside"}`, or the "Migrate Sidecars Now" button in settings). +- All sidecar/preview path derivation goes through the helpers in `py/utils/sidecar_paths.py`; never construct paths inline. + +--- + ## Base Fields (All Model Types) These fields are present in all model metadata files. diff --git a/locales/de.json b/locales/de.json index c2a4a3bd..ca6b8990 100644 --- a/locales/de.json +++ b/locales/de.json @@ -383,6 +383,7 @@ "exampleImages": "Beispielbilder", "autoOrganize": "Auto-Organisierung", "metadata": "Metadaten", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Proxy-Einstellungen" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "App-Proxy aktivieren", "enableProxyHelp": "Aktivieren Sie benutzerdefinierte Proxy-Einstellungen für diese Anwendung. Überschreibt die System-Proxy-Einstellungen.", @@ -1637,6 +1659,11 @@ "titleRevert": "Ursprüngliche Dateinamen wiederherstellen?", "revertButton": "Ursprüngliche Dateinamen wiederherstellen" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Tags zu mehreren Modellen hinzufügen", "description": "Tags hinzufügen zu", diff --git a/locales/en.json b/locales/en.json index de13e421..40eff655 100644 --- a/locales/en.json +++ b/locales/en.json @@ -383,6 +383,7 @@ "exampleImages": "Example Images", "autoOrganize": "Auto-organize", "metadata": "Metadata", + "sidecarStorage": "Sidecar Storage", "proxySettings": "Proxy Settings" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "Sidecar Storage Mode", + "modeHelp": "Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "Alongside model files (default)", + "centralized": "Centralized storage" + }, + "path": "Centralized Storage Path", + "pathHelp": "Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "Empty = /sidecars", + "management": "Sidecar Migration", + "managementHelp": "Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "Migrate Sidecars Now", + "migratingButton": "Migrating...", + "migrating": "Migrating sidecars...", + "migrateSuccess": "Sidecar migration completed successfully", + "migrateFailed": "Sidecar migration failed: {message}", + "migrationDeferred": "Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Enable App-level Proxy", "enableProxyHelp": "Enable custom proxy settings for this application, overriding system proxy settings", @@ -1637,6 +1659,11 @@ "titleRevert": "Restore original filenames?", "revertButton": "Restore Original Filenames" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "Move sidecars to centralized storage?", + "titleToAlongside": "Move sidecars back next to model files?", + "confirmButton": "Migrate Now" + }, "bulkAddTags": { "title": "Add Tags to Multiple Models", "description": "Add tags to", diff --git a/locales/es.json b/locales/es.json index b8a2a25b..54654810 100644 --- a/locales/es.json +++ b/locales/es.json @@ -383,6 +383,7 @@ "exampleImages": "Imágenes de ejemplo", "autoOrganize": "Organización automática", "metadata": "Metadatos", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Configuración de proxy" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Habilitar proxy a nivel de aplicación", "enableProxyHelp": "Habilita la configuración de proxy personalizada para esta aplicación, sobrescribiendo la configuración de proxy del sistema", @@ -1637,6 +1659,11 @@ "titleRevert": "¿Restaurar los nombres de archivo originales?", "revertButton": "Restaurar nombres de archivo originales" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Añadir etiquetas a múltiples modelos", "description": "Añadir etiquetas a", diff --git a/locales/fr.json b/locales/fr.json index adabe9a7..792745be 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -383,6 +383,7 @@ "exampleImages": "Images d'exemple", "autoOrganize": "Organisation automatique", "metadata": "Métadonnées", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Paramètres du proxy" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Activer le proxy au niveau de l'application", "enableProxyHelp": "Activer les paramètres de proxy personnalisés pour cette application, remplaçant les paramètres de proxy système", @@ -1637,6 +1659,11 @@ "titleRevert": "Restaurer les noms de fichier d'origine ?", "revertButton": "Restaurer les noms de fichier d'origine" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Ajouter des tags à plusieurs modèles", "description": "Ajouter des tags à", diff --git a/locales/he.json b/locales/he.json index 203e0ca4..672b8464 100644 --- a/locales/he.json +++ b/locales/he.json @@ -383,6 +383,7 @@ "exampleImages": "תמונות דוגמה", "autoOrganize": "ארגון אוטומטי", "metadata": "מטא-נתונים", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "הגדרות פרוקסי" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "הפעל פרוקסי ברמת האפליקציה", "enableProxyHelp": "אפשר הגדרות פרוקסי מותאמות אישית עבור יישום זה, במקום הגדרות הפרוקסי של המערכת", @@ -1637,6 +1659,11 @@ "titleRevert": "לשחזר שמות קבצים מקוריים?", "revertButton": "שחזר שמות קבצים מקוריים" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "הוסף תגיות למספר מודלים", "description": "הוסף תגיות ל-", diff --git a/locales/ja.json b/locales/ja.json index 8f99755c..6de03706 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -383,6 +383,7 @@ "exampleImages": "例画像", "autoOrganize": "自動整理", "metadata": "メタデータ", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "プロキシ設定" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "アプリレベルのプロキシを有効化", "enableProxyHelp": "このアプリケーション専用のカスタムプロキシ設定を有効にします（システムのプロキシ設定を上書きします）", @@ -1637,6 +1659,11 @@ "titleRevert": "元のファイル名を復元しますか？", "revertButton": "元のファイル名を復元" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "複数モデルにタグを追加", "description": "タグを追加するモデル：", diff --git a/locales/ko.json b/locales/ko.json index 35e558f9..cbfb665b 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -383,6 +383,7 @@ "exampleImages": "예시 이미지", "autoOrganize": "자동 정리", "metadata": "메타데이터", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "프록시 설정" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "앱 수준 프록시 활성화", "enableProxyHelp": "이 애플리케이션에 대한 사용자 지정 프록시 설정을 활성화하여 시스템 프록시 설정을 무시합니다", @@ -1637,6 +1659,11 @@ "titleRevert": "원본 파일명을 복원하시겠습니까?", "revertButton": "원본 파일명 복원" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "여러 모델에 태그 추가", "description": "다음에 태그를 추가합니다:", diff --git a/locales/ru.json b/locales/ru.json index 2d8e8552..9e26faf1 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -383,6 +383,7 @@ "exampleImages": "Примеры изображений", "autoOrganize": "Автоорганизация", "metadata": "Метаданные", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Настройки прокси" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Включить прокси на уровне приложения", "enableProxyHelp": "Включить пользовательские настройки прокси для этого приложения, переопределяя системные настройки прокси", @@ -1637,6 +1659,11 @@ "titleRevert": "Восстановить исходные имена файлов?", "revertButton": "Восстановить исходные имена файлов" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Добавить теги к нескольким моделям", "description": "Добавить теги к", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 84a730b7..53728dd3 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -383,6 +383,7 @@ "exampleImages": "示例图片", "autoOrganize": "自动整理", "metadata": "元数据", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "代理设置" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "启用应用级代理", "enableProxyHelp": "为此应用启用自定义代理设置，覆盖系统代理设置", @@ -1637,6 +1659,11 @@ "titleRevert": "恢复原始文件名？", "revertButton": "恢复原始文件名" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "批量添加标签", "description": "为多个模型添加标签", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 3e536578..2770a105 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -383,6 +383,7 @@ "exampleImages": "範例圖片", "autoOrganize": "自動整理", "metadata": "中繼資料", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "代理設定" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "啟用應用程式代理", "enableProxyHelp": "啟用此應用程式的自訂代理設定，將覆蓋系統代理設定", @@ -1637,6 +1659,11 @@ "titleRevert": "要還原原始檔案名稱嗎？", "revertButton": "還原原始檔案名稱" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "新增標籤到多個模型", "description": "新增標籤到", diff --git a/py/config.py b/py/config.py index 3c5465c4..8ec4b9af 100644 --- a/py/config.py +++ b/py/config.py @@ -891,6 +891,17 @@ class Config: if self.recipes_path: preview_roots.update(self._expand_preview_root(self.recipes_path)) + # Centralized sidecar storage holds preview assets outside the model + # roots; allow serving them when the mode is active. + try: + from .utils.sidecar_paths import get_sidecar_root # Local import to avoid circular dependency + + sidecar_root = get_sidecar_root() + except Exception: # pragma: no cover - defensive fallback + sidecar_root = "" + if sidecar_root: + preview_roots.update(self._expand_preview_root(sidecar_root)) + for target, link in self._path_mappings.items(): preview_roots.update(self._expand_preview_root(target)) preview_roots.update(self._expand_preview_root(link)) @@ -1494,6 +1505,15 @@ class Config: self.other_roots = self._init_other_paths() self._rebuild_preview_roots() + def refresh_preview_roots(self) -> None: + """Rebuild the preview allowlist after path-affecting settings change. + + Called when ``sidecar_storage_mode`` / ``sidecar_storage_path`` are + updated so centralized preview assets become servable (or stop being + servable) without a restart. + """ + self._rebuild_preview_roots() + def get_other_models_availability(self) -> Dict[str, Any]: """Report the other-model folders the host can actually expose. diff --git a/py/metadata_ops/__init__.py b/py/metadata_ops/__init__.py index 40781ff4..15b0f1d9 100644 --- a/py/metadata_ops/__init__.py +++ b/py/metadata_ops/__init__.py @@ -179,6 +179,9 @@ async def download_preview( base_name = os.path.splitext(os.path.basename(model_path))[0] preview_dir = get_preview_dir(model_path) + # Centralized mirrors may not exist yet (unlike the model's own directory + # in alongside mode). + os.makedirs(preview_dir, exist_ok=True) output_path = os.path.join(preview_dir, base_name + ".webp") downloader = await get_downloader() diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index 8e156f6a..a365daf3 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -45,6 +45,8 @@ from ...services.llm_service import ( get_provider_model_ids, ) from ...services.cache_health_monitor import CacheHealthMonitor, CacheHealthStatus +from ...services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase +from ...services.websocket_progress_callback import WebSocketBroadcastCallback from ...utils.models import BaseModelMetadata from ...utils.constants import ( CIVITAI_USER_MODEL_TYPES, @@ -4136,6 +4138,57 @@ class NodeRegistryHandler: return web.json_response({"success": False, "error": str(exc)}, status=500) +class SidecarMigrationHandler: + """Migrate sidecar metadata and previews between storage layouts.""" + + _VALID_DIRECTIONS = ("to_centralized", "to_alongside") + + def __init__( + self, + *, + use_case_factory: Callable[[], SidecarMigrationUseCase] = SidecarMigrationUseCase, + progress_callback_factory: Callable[[], Any] = WebSocketBroadcastCallback, + ) -> None: + self._use_case_factory = use_case_factory + self._progress_callback_factory = progress_callback_factory + + async def migrate_sidecars(self, request: web.Request) -> web.Response: + """Run a sidecar migration; accepts POST JSON or GET query params.""" + try: + if request.method == "GET": + params: Mapping[str, Any] = request.query + else: + try: + params = await request.json() + except Exception: # empty/invalid body: fall back to query + params = request.query + + direction = str(params.get("direction") or "").strip() + if direction not in self._VALID_DIRECTIONS: + return web.json_response( + { + "success": False, + "error": "direction must be 'to_centralized' or 'to_alongside'", + }, + status=400, + ) + + force = params.get("force") in (True, 1, "true", "1") + + use_case = self._use_case_factory() + progress_cb = self._progress_callback_factory() + result = await use_case.execute_with_error_handling( + direction=direction, + progress_cb=progress_cb, + force=force, + ) + status = 200 if result.get("success") else 400 + return web.json_response(result, status=status) + except Exception as exc: + logger.error("Sidecar migration failed: %s", exc, exc_info=True) + return web.json_response({"success": False, "error": str(exc)}, status=500) + + class MiscHandlerSet: """Aggregate handlers into a lookup compatible with the registrar.""" @@ -4162,6 +4215,7 @@ class MiscHandlerSet: model_source_handler: Any = None, agent_handler: Any = None, download_routing: Any = None, + sidecar_migration: Any = None, ) -> None: self.health = health self.settings = settings @@ -4183,6 +4237,7 @@ class MiscHandlerSet: self.model_source_handler = model_source_handler self.agent_handler = agent_handler self.download_routing = download_routing + self.sidecar_migration = sidecar_migration def to_route_mapping( self, @@ -4249,6 +4304,8 @@ class MiscHandlerSet: "cancel_agent_skill": self.agent_handler.cancel_agent_skill, # Download routing handler "get_download_routing": self.download_routing.get_download_routing, + # Sidecar migration handler + "migrate_sidecars": self.sidecar_migration.migrate_sidecars, # Base model handlers "get_base_models": self.base_model.get_base_models, "refresh_base_models": self.base_model.refresh_base_models, diff --git a/py/routes/misc_route_registrar.py b/py/routes/misc_route_registrar.py index 0c4dde79..ccfa440a 100644 --- a/py/routes/misc_route_registrar.py +++ b/py/routes/misc_route_registrar.py @@ -113,6 +113,13 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = ( RouteDefinition( "POST", "/api/lm/download/routing", "get_download_routing" ), + # Sidecar storage layout migration (GET supported for the extension) + RouteDefinition( + "POST", "/api/lm/sidecars/migrate", "migrate_sidecars" + ), + RouteDefinition( + "GET", "/api/lm/sidecars/migrate", "migrate_sidecars" + ), RouteDefinition( "POST", "/api/lm/download-model-source", "download_model_source" ), diff --git a/py/routes/misc_routes.py b/py/routes/misc_routes.py index 8db36ce4..03b980a0 100644 --- a/py/routes/misc_routes.py +++ b/py/routes/misc_routes.py @@ -32,6 +32,7 @@ from .handlers.misc_handlers import ( NodeRegistry, NodeRegistryHandler, SettingsHandler, + SidecarMigrationHandler, SupportersHandler, TrainedWordsHandler, UsageStatsHandler, @@ -142,6 +143,7 @@ class MiscRoutes: model_source_handler = ModelSourceHandler() agent_handler = AgentHandler() download_routing = DownloadRoutingHandler() + sidecar_migration = SidecarMigrationHandler() return self._handler_set_factory( health=health, @@ -164,6 +166,7 @@ class MiscRoutes: model_source_handler=model_source_handler, agent_handler=agent_handler, download_routing=download_routing, + sidecar_migration=sidecar_migration, ) diff --git a/py/services/checkpoint_scanner.py b/py/services/checkpoint_scanner.py index 02402798..d5842499 100644 --- a/py/services/checkpoint_scanner.py +++ b/py/services/checkpoint_scanner.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional from ..utils.models import CheckpointMetadata from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3 from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_preview_dir, is_centralized from ..config import config from .model_scanner import ModelScanner, _is_excluded_dir from .model_hash_index import ModelHashIndex @@ -61,10 +62,9 @@ class CheckpointScanner(ModelScanner): return None base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # AutoV3 reads only the safetensors header, so it is cheap even for # large checkpoints; record the checked state at creation time ("" = @@ -322,6 +322,11 @@ class CheckpointScanner(ModelScanner): async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]: """Scan filesystem for checkpoint metadata files with pending hash status.""" + # Centralized mode stores sidecars in the mirror tree, not next to the + # models; walk the mirror instead of the model folders. + if is_centralized(): + return self._find_pending_models_in_sidecar_mirror() + pending_models = [] for root_path in self.get_model_roots(): diff --git a/py/services/download_manager.py b/py/services/download_manager.py index 52518ec7..b2cf40ec 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -2465,6 +2465,10 @@ class DownloadManager: # Download preview image if available images = version_info.get("images", []) if images: + # Centralized preview mirrors may not exist yet (unlike the + # model's own directory in alongside mode). + os.makedirs(get_preview_dir(save_path), exist_ok=True) + if progress_callback: await progress_callback( 1 @@ -3067,10 +3071,12 @@ class DownloadManager: first_target = targets[0] if preview_path != first_target: + os.makedirs(os.path.dirname(first_target), exist_ok=True) os.replace(preview_path, first_target) source_path = first_target for target in targets[1:]: + os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copyfile(source_path, target) return targets diff --git a/py/services/model_file_service.py b/py/services/model_file_service.py index 659805cd..8fef00a7 100644 --- a/py/services/model_file_service.py +++ b/py/services/model_file_service.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from ..utils.utils import calculate_relative_path_for_model, remove_empty_dirs from ..utils.constants import AUTO_ORGANIZE_BATCH_SIZE, MODEL_FILE_EXTENSIONS +from ..utils.sidecar_paths import is_centralized, resolve_centralized_dir_for_dir from ..services.settings_manager import get_settings_manager from ..services.model_lifecycle_service import _require_path_in_library_roots from ..services.pending_delete_service import PENDING_DELETE_DIR_NAME @@ -631,6 +632,21 @@ class ModelMoveService: shutil.rmtree(absolute_path) + # Centralized mode: prune the folder's mirror subtree when it no + # longer holds any sidecar files (per-model deletes already + # removed their sidecars, so only empty directories are expected; + # a non-empty mirror keeps its orphan sidecars). + if is_centralized(): + mirror_dir = resolve_centralized_dir_for_dir(absolute_path) + if mirror_dir and os.path.isdir(mirror_dir): + for root, _dirs, files in os.walk(mirror_dir, topdown=False): + if files: + continue + try: + os.rmdir(root) + except OSError: # pragma: no cover - best-effort cleanup + pass + await self._forget_folder(relative_folder) return { diff --git a/py/services/model_lifecycle_service.py b/py/services/model_lifecycle_service.py index bb59f97f..f5c8fb4b 100644 --- a/py/services/model_lifecycle_service.py +++ b/py/services/model_lifecycle_service.py @@ -11,7 +11,7 @@ from ..services.service_registry import ServiceRegistry from ..services.pending_delete_service import get_pending_delete_service from ..utils.constants import PREVIEW_EXTENSIONS from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_dir logger = logging.getLogger(__name__) @@ -42,19 +42,22 @@ async def load_local_metadata(metadata_path: str) -> Dict[str, Any]: async def delete_model_artifacts( target_dir: str, file_name: str, main_extension: str | None = None ) -> List[str]: - """Delete the primary model artefacts within ``target_dir``.""" + """Delete the primary model artefacts within ``target_dir``. + + Sidecars and previews are taken from the model's sidecar directory — the + model's own directory in alongside mode, the centralized mirror otherwise. + """ main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [ - main_file, - os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), - ] + model_path = os.path.join(target_dir, main_file) + sidecar_dir = get_sidecar_dir(model_path) + patterns = [os.path.basename(get_metadata_path(model_path))] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") deleted: List[str] = [] - main_path = os.path.join(target_dir, main_file).replace(os.sep, "/") + main_path = model_path.replace(os.sep, "/") if os.path.exists(main_path): os.remove(main_path) @@ -62,8 +65,8 @@ async def delete_model_artifacts( else: logger.warning("Model file not found: %s", main_file) - for pattern in patterns[1:]: - path = os.path.join(target_dir, pattern) + for pattern in patterns: + path = os.path.join(sidecar_dir, pattern) if os.path.exists(path): try: os.remove(path) @@ -389,17 +392,21 @@ class ModelLifecycleService: raise ValueError("A file with this name already exists") metadata_filename = os.path.basename(get_metadata_path(file_path)) - patterns = [ - f"{old_file_name}{old_extension}", - metadata_filename, - f"{metadata_filename}.bak", + # Sidecars/previews live in the sidecar dir (the model's own dir in + # alongside mode, the centralized mirror otherwise); the model file + # itself always stays in target_dir. + sidecar_dir = get_sidecar_dir(file_path) + patterns: List[tuple[str, str]] = [ + (target_dir, f"{old_file_name}{old_extension}"), + (sidecar_dir, metadata_filename), + (sidecar_dir, f"{metadata_filename}.bak"), ] for ext in PREVIEW_EXTENSIONS: - patterns.append(f"{old_file_name}{ext}") + patterns.append((sidecar_dir, f"{old_file_name}{ext}")) existing_files: List[tuple[str, str]] = [] - for pattern in patterns: - path = os.path.join(target_dir, pattern) + for pattern_dir, pattern in patterns: + path = os.path.join(pattern_dir, pattern) if os.path.exists(path): existing_files.append((path, pattern)) @@ -418,9 +425,9 @@ class ModelLifecycleService: for old_path, pattern in existing_files: ext = self._get_multipart_ext(pattern) - new_path = os.path.join(target_dir, f"{new_file_name}{ext}").replace( - os.sep, "/" - ) + new_path = os.path.join( + os.path.dirname(old_path), f"{new_file_name}{ext}" + ).replace(os.sep, "/") os.rename(old_path, new_path) renamed_files.append(new_path) @@ -437,9 +444,9 @@ class ModelLifecycleService: if metadata.get("preview_url"): old_preview = str(metadata["preview_url"]) ext = self._get_multipart_ext(old_preview) - new_preview = os.path.join(target_dir, f"{new_file_name}{ext}").replace( - os.sep, "/" - ) + new_preview = os.path.join( + get_preview_dir(new_file_path), f"{new_file_name}{ext}" + ).replace(os.sep, "/") metadata["preview_url"] = new_preview await self._metadata_manager.save_metadata(new_file_path, metadata) diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index a8525165..c884c9de 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -11,7 +11,13 @@ from ..utils.models import BaseModelMetadata, autov3_from_civitai_files from ..config import config from ..utils.file_utils import find_preview_file, get_preview_extension, calculate_sha256, calculate_autov3 from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir +from ..utils.sidecar_paths import ( + get_metadata_path, + get_preview_dir, + get_sidecar_dir, + is_centralized, + resolve_centralized_dir_for_dir, +) from ..utils.civitai_utils import resolve_license_info from .model_cache import ModelCache from .model_hash_index import ModelHashIndex @@ -1614,6 +1620,25 @@ class ModelScanner: old_abs_prefix = f"{str(previous_path).replace(chr(92), '/').rstrip('/')}/" new_abs_prefix = f"{str(new_path).replace(chr(92), '/').rstrip('/')}/" + # Centralized sidecar mode: sidecars/previews live in the mirror tree, + # not under the renamed model directory, so the mirror subtree must + # move too and mirror-prefixed preview URLs need their own rekey. + old_mirror_dir: Optional[str] = None + new_mirror_dir: Optional[str] = None + if is_centralized(): + old_mirror_dir = resolve_centralized_dir_for_dir(str(previous_path)) + new_mirror_dir = resolve_centralized_dir_for_dir(str(new_path)) + old_mirror_prefix = ( + f"{old_mirror_dir.replace(chr(92), '/').rstrip('/')}/" + if old_mirror_dir + else "" + ) + new_mirror_prefix = ( + f"{new_mirror_dir.replace(chr(92), '/').rstrip('/')}/" + if new_mirror_dir + else "" + ) + cache = self._cache if cache is None: return False @@ -1671,8 +1696,24 @@ class ModelScanner: item["preview_url"] = self._rekey_path( item["preview_url"], old_abs_prefix, new_abs_prefix ) + if old_mirror_prefix: + item["preview_url"] = self._rekey_path( + item["preview_url"], old_mirror_prefix, new_mirror_prefix + ) touched.append(item) + if old_mirror_dir and new_mirror_dir and os.path.isdir(old_mirror_dir): + try: + os.makedirs(os.path.dirname(new_mirror_dir), exist_ok=True) + shutil.move(old_mirror_dir, new_mirror_dir) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "Failed to move centralized sidecar mirror %s -> %s: %s", + old_mirror_dir, + new_mirror_dir, + exc, + ) + if touched: changed = True await self._rewrite_sidecar_paths(touched) @@ -1701,7 +1742,9 @@ class ModelScanner: async def _rewrite_sidecar_paths(self, entries: List[Dict[str, Any]]) -> None: """Point each model's metadata sidecar at its new location. - Sidecars travel with the renamed directory, so only the recorded + In alongside mode sidecars travel with the renamed directory; in + centralized mode the mirror subtree has already been moved by the + caller (:meth:`rename_known_folder`). Either way only the recorded ``file_path``/``preview_url`` inside them need rewriting. Failures are logged and skipped — a stale sidecar is repaired by the next metadata refresh, and must not abort the rename. @@ -1720,6 +1763,94 @@ class ModelScanner: "Failed to rewrite metadata sidecar %s: %s", metadata_path, exc ) + def _find_pending_models_in_sidecar_mirror(self) -> List[Dict[str, Any]]: + """Mirror-tree counterpart of the alongside pending-hash filesystem scan. + + Centralized mode stores ``.metadata.json`` sidecars in the mirror + tree, so walking the model folders finds nothing. Each mirror base is + resolved from a configured model root; a sidecar's recorded + ``file_path`` locates its model, with a stem-based probe under the + mapped model root as fallback (mirror path components are sanitized, + so reverse mapping is best-effort). Orphan sidecars whose model file + no longer exists are skipped, matching the alongside scan. + """ + + pending_models: List[Dict[str, Any]] = [] + + for root_path in self.get_model_roots(): + mirror_base = resolve_centralized_dir_for_dir(root_path) + if not mirror_base or not os.path.isdir(mirror_base): + continue + + for dirpath, dirnames, filenames in os.walk(mirror_base): + dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)] + for filename in filenames: + if not filename.endswith(".metadata.json"): + continue + + metadata_path = os.path.join(dirpath, filename) + try: + with open(metadata_path, "r", encoding="utf-8") as f: + data = json.load(f) + + # Check if hash is pending + hash_status = data.get("hash_status", "completed") + sha256 = data.get("sha256", "") + + if hash_status != "completed" or not sha256: + # Find corresponding model file: prefer the + # sidecar's recorded path, then probe by stem + # under the mapped model root. + model_path = None + recorded_path = data.get("file_path") + if ( + isinstance(recorded_path, str) + and recorded_path + and os.path.exists(recorded_path) + ): + model_path = recorded_path + else: + model_name = filename.replace(".metadata.json", "") + rel_dir = os.path.relpath(dirpath, mirror_base) + candidate_dir = ( + root_path + if rel_dir == os.curdir + else os.path.join(root_path, rel_dir) + ) + for ext in self.file_extensions: + potential_path = os.path.join( + candidate_dir, model_name + ext + ) + if os.path.exists(potential_path): + model_path = potential_path + break + + if model_path: + pending_models.append( + { + "file_path": model_path.replace(os.sep, "/"), + "hash_status": hash_status, + "sha256": sha256, + **{ + k: v + for k, v in data.items() + if k + not in [ + "file_path", + "hash_status", + "sha256", + ] + }, + } + ) + except (json.JSONDecodeError, Exception) as e: + logger.debug( + f"Error reading metadata file {metadata_path}: {e}" + ) + continue + + return pending_models + def _schedule_all_folders_backfill(self) -> None: """Kick off a one-shot background folder walk if none is running.""" if self._all_folders_backfill_running: @@ -1890,7 +2021,7 @@ class ModelScanner: file_info['name'] = os.path.basename(file_path) metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path) - metadata.preview_url = find_preview_file(local_stem, os.path.dirname(file_path)) + metadata.preview_url = find_preview_file(local_stem, get_preview_dir(file_path)) await MetadataManager.save_metadata(file_path, metadata) logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)") except Exception as e: @@ -2327,38 +2458,51 @@ class ModelScanner: # Move all associated files with the same base name source_metadata = None moved_metadata_path = None - - # Find all files with the same base name in the source directory + + # Associated files (sidecar metadata, previews) sit next to the + # model in alongside mode and in the mirror tree in centralized + # mode; collect from every directory that holds them. + source_sidecar_dir = get_sidecar_dir(source_path) + target_sidecar_dir = get_sidecar_dir(target_file) + associated_dirs = [(source_dir, target_path)] + if os.path.normpath(source_sidecar_dir) != os.path.normpath(source_dir): + associated_dirs.append((source_sidecar_dir, target_sidecar_dir)) + + # Find all files with the same base name in the source directories files_to_move = [] - try: - for file in os.listdir(source_dir): - if file.startswith(base_name + ".") and file != os.path.basename(source_path): - source_file_path = os.path.join(source_dir, file) - # Generate new filename with the same base name as the model file - file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png") - new_associated_filename = f"{final_base_name}{file_suffix}" - target_associated_path = os.path.join(target_path, new_associated_filename) - - # Store metadata file path for special handling - if file == os.path.basename(get_metadata_path(source_path)): - source_metadata = source_file_path - moved_metadata_path = target_associated_path - else: - files_to_move.append((source_file_path, target_associated_path)) - except Exception as e: - logger.error(f"Error listing files in {source_dir}: {e}") - + metadata_filename = os.path.basename(get_metadata_path(source_path)) + for assoc_source_dir, assoc_target_dir in associated_dirs: + try: + for file in os.listdir(assoc_source_dir): + if file.startswith(base_name + ".") and file != os.path.basename(source_path): + source_file_path = os.path.join(assoc_source_dir, file) + # Generate new filename with the same base name as the model file + file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png") + new_associated_filename = f"{final_base_name}{file_suffix}" + target_associated_path = os.path.join(assoc_target_dir, new_associated_filename) + + # Store metadata file path for special handling + if file == metadata_filename: + source_metadata = source_file_path + moved_metadata_path = target_associated_path + else: + files_to_move.append((source_file_path, target_associated_path)) + except Exception as e: + logger.error(f"Error listing files in {assoc_source_dir}: {e}") + # Move all associated files metadata = None for source_file, target_file_path in files_to_move: try: + os.makedirs(os.path.dirname(target_file_path), exist_ok=True) shutil.move(source_file, target_file_path) except Exception as e: logger.error(f"Error moving associated file {source_file}: {e}") - + # Handle metadata file specially to update paths if source_metadata and moved_metadata_path and os.path.exists(source_metadata): try: + os.makedirs(os.path.dirname(moved_metadata_path), exist_ok=True) shutil.move(source_metadata, moved_metadata_path) metadata = await self._update_metadata_paths(moved_metadata_path, target_file) except Exception as e: @@ -2826,7 +2970,7 @@ class ModelScanner: if not file_path: return None - dir_path = os.path.dirname(file_path) + dir_path = get_preview_dir(file_path) base_name = os.path.splitext(os.path.basename(file_path))[0] preview_path = find_preview_file(base_name, dir_path) if preview_path: diff --git a/py/services/other_scanner.py b/py/services/other_scanner.py index 0db2101a..b2a58ef4 100644 --- a/py/services/other_scanner.py +++ b/py/services/other_scanner.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional from ..utils.models import OtherModelMetadata from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3 from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_preview_dir, is_centralized from ..config import config from .model_scanner import ModelScanner, _is_excluded_dir from .model_hash_index import ModelHashIndex @@ -72,10 +73,9 @@ class OtherScanner(ModelScanner): return None base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # AutoV3 reads only the safetensors header, so it is cheap even for # large files; record the checked state at creation time ("" = @@ -333,6 +333,11 @@ class OtherScanner(ModelScanner): async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]: """Scan filesystem for other-model metadata files with pending hash status.""" + # Centralized mode stores sidecars in the mirror tree, not next to the + # models; walk the mirror instead of the model folders. + if is_centralized(): + return self._find_pending_models_in_sidecar_mirror() + pending_models = [] for root_path in self.get_model_roots(): diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index 70bc0746..c57cf6e4 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -38,7 +38,7 @@ from typing import ( ) from ..utils.constants import PREVIEW_EXTENSIONS -from ..utils.sidecar_paths import get_metadata_path +from ..utils.sidecar_paths import get_metadata_path, get_sidecar_dir from ..utils import settings_paths logger = logging.getLogger(__name__) @@ -670,16 +670,22 @@ class PendingDeleteService: """Enumerate existing artifacts exactly like delete_model_artifacts.""" main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [ - main_file, - os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), - ] + model_path = os.path.join(target_dir, main_file) + + artifacts: List[str] = [] + main_path = os.path.abspath(model_path) + if os.path.exists(main_path): + artifacts.append(main_path) + + # Sidecars/previews live in the sidecar dir (the model's own dir in + # alongside mode, the centralized mirror otherwise). + sidecar_dir = get_sidecar_dir(model_path) + patterns = [os.path.basename(get_metadata_path(model_path))] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") - artifacts: List[str] = [] for pattern in patterns: - path = os.path.abspath(os.path.join(target_dir, pattern)) + path = os.path.abspath(os.path.join(sidecar_dir, pattern)) if os.path.exists(path): artifacts.append(path) return artifacts @@ -698,7 +704,9 @@ class PendingDeleteService: """ for original_path in artifacts: staged_path = os.path.join(batch_dir, os.path.basename(original_path)) - os.rename(original_path, staged_path) + # EXDEV-tolerant: centralized sidecars may live on a different + # filesystem than the staging batch dir under the model root. + self._restore_file(original_path, staged_path) staged_pairs.append( { "staged": os.path.abspath(staged_path), @@ -740,13 +748,14 @@ class PendingDeleteService: return staged_pairs def _restore_file(self, staged_path: str, original_path: str) -> None: - """Restore a staged file to its original path, tolerating EXDEV. + """Move a file between staging and library paths, tolerating EXDEV. ``os.rename`` is atomic and preferred (model staging and most recipe restores are same-volume). Recipe staging copies into the settings-dir - staging parent, which may live on a DIFFERENT filesystem than the - recipes dir; rename then raises EXDEV. Fall back to ``shutil.copy2`` + - ``os.remove`` so the bytes are restored and the staged copy removed. + staging parent, and centralized sidecars live under the configured + sidecar root; both may live on a DIFFERENT filesystem than the target + dir, so rename can raise EXDEV. Fall back to ``shutil.copy2`` + + ``os.remove`` so the bytes are moved and the source copy removed. """ try: os.rename(staged_path, original_path) diff --git a/py/services/preview_asset_service.py b/py/services/preview_asset_service.py index 1e215347..a6ada6a3 100644 --- a/py/services/preview_asset_service.py +++ b/py/services/preview_asset_service.py @@ -64,6 +64,9 @@ class PreviewAssetService: base_name = os.path.splitext(os.path.splitext(os.path.basename(metadata_path))[0])[0] preview_dir = os.path.dirname(metadata_path) + # Centralized mirrors may not exist yet (unlike the model's own + # directory in alongside mode). + os.makedirs(preview_dir, exist_ok=True) is_video = first_preview.get("type") == "video" preview_url = first_preview.get("url") @@ -161,6 +164,9 @@ class PreviewAssetService: base_name = os.path.splitext(os.path.basename(model_path))[0] folder = get_preview_dir(model_path) + # Centralized mirrors may not exist yet (unlike the model's own + # directory in alongside mode). + os.makedirs(folder, exist_ok=True) extension, optimized_data = await self._convert_preview( preview_data, content_type, original_filename diff --git a/py/services/settings_manager.py b/py/services/settings_manager.py index 4f25ef0e..0e874776 100644 --- a/py/services/settings_manager.py +++ b/py/services/settings_manager.py @@ -99,6 +99,8 @@ DEFAULT_SETTINGS: Dict[str, Any] = { "enable_other_models": False, "enabled_other_sub_types": list(DEFAULT_ENABLED_OTHER_SUB_TYPES), "recipes_path": "", + "sidecar_storage_mode": "alongside", + "sidecar_storage_path": "", "base_model_path_mappings": {}, "download_path_templates": {}, "download_filename_templates": {}, @@ -1616,9 +1618,30 @@ class SettingsManager: return os.path.abspath(os.path.normpath(os.path.expanduser(stripped))) + @staticmethod + def _normalize_sidecar_storage_mode(value: Any) -> str: + """Return a valid sidecar storage mode, falling back to ``alongside``.""" + + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in ("alongside", "centralized"): + return normalized + return "alongside" + + def _refresh_sidecar_storage_config(self) -> None: + """Rebuild dependent config state after sidecar storage settings change.""" + + try: + from ..config import config # Local import to avoid circular dependency + + config.refresh_preview_roots() + except Exception as exc: # pragma: no cover - defensive logging + logger.debug( + "Failed to refresh config after sidecar storage change: %s", exc + ) + def _get_effective_recipes_dir(self, recipes_path: Optional[str] = None) -> str: """Resolve the effective recipes directory for the active library.""" - normalized_custom = self._normalize_recipes_path_value( self.settings.get("recipes_path", "") if recipes_path is None @@ -1815,6 +1838,10 @@ class SettingsManager: target_recipes_dir = self._get_effective_recipes_dir(value) self._validate_recipes_storage_path(target_recipes_dir) self._migrate_recipes_directory(current_recipes_dir, target_recipes_dir) + elif key == "sidecar_storage_mode": + value = self._normalize_sidecar_storage_mode(value) + elif key == "sidecar_storage_path": + value = self._normalize_recipes_path_value(value) self.settings[key] = value portable_switch_pending = False if key == "use_portable_settings" and isinstance(value, bool): @@ -1845,6 +1872,8 @@ class SettingsManager: self._save_settings() if key == "recipes_path": self._notify_library_change(self.get_active_library_name()) + if key in ("sidecar_storage_mode", "sidecar_storage_path"): + self._refresh_sidecar_storage_config() if key in ("enable_other_models", "enabled_other_sub_types"): self._apply_other_model_settings_change() if portable_switch_pending: diff --git a/py/services/use_cases/__init__.py b/py/services/use_cases/__init__.py index ae6a6900..02756173 100644 --- a/py/services/use_cases/__init__.py +++ b/py/services/use_cases/__init__.py @@ -21,6 +21,10 @@ from .example_images import ( ImportExampleImagesValidationError, ) from .filename_template_use_case import FilenameTemplateUseCase +from .sidecar_migration_use_case import ( + SidecarMigrationProgressReporter, + SidecarMigrationUseCase, +) __all__ = [ "AutoOrganizeInProgressError", @@ -36,4 +40,6 @@ __all__ = [ "ImportExampleImagesUseCase", "ImportExampleImagesValidationError", "FilenameTemplateUseCase", + "SidecarMigrationProgressReporter", + "SidecarMigrationUseCase", ] diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py new file mode 100644 index 00000000..9f50b6ff --- /dev/null +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -0,0 +1,418 @@ +"""Use case migrating sidecar metadata and previews between storage layouts. + +Two storage layouts exist (see :mod:`py.utils.sidecar_paths`): + +- ``alongside``: ``/.metadata.json`` and preview files live + next to the model file. +- ``centralized``: the same files live under the configured sidecar root, + mirroring the library-relative directory structure. + +This use case moves the ``.metadata.json`` sidecar and preview files for every +known model from one layout to the other. Model files themselves NEVER move. +Paths inside the moved sidecar (``file_path``, ``file_name``, ``preview_url``) +are rewritten the same way :meth:`ModelScanner._update_metadata_paths` does. + +Intended flow (settings-first): + +1. The user switches ``sidecar_storage_mode`` (and optionally + ``sidecar_storage_path``) in settings. +2. The migration runs in the direction of the NEW mode with ``force=True``. + After the switch, files in the OLD layout are the source of truth; the + guard below would otherwise refuse to run because the active mode already + matches the migration target. + +Both orderings work because all path computations are mode-independent: the +alongside location is derived from the model path directly, and the mirror +location is resolved via ``get_configured_sidecar_root()``, which ignores the +active mode. + +Guards (pass ``force=True`` to bypass): + +- ``migrate_to_centralized`` refuses when centralized storage is already the + active, resolvable mode. +- ``migrate_to_alongside`` refuses when the active mode is ``alongside``. +""" + +from __future__ import annotations + +import errno +import json +import logging +import os +import shutil +from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Sequence, Tuple + +from ..service_registry import ServiceRegistry +from ..settings_manager import get_settings_manager +from ...utils.constants import PREVIEW_EXTENSIONS +from ...utils.file_utils import get_preview_extension +from ...utils.metadata_manager import MetadataManager +from ...utils.sidecar_paths import ( + METADATA_SUFFIX, + STORAGE_MODE_CENTRALIZED, + get_configured_sidecar_root, + get_sidecar_root, + get_storage_mode, + resolve_centralized_dir_for_dir, +) + + +class SidecarMigrationProgressReporter(Protocol): + """Protocol for progress reporters used during sidecar migration.""" + + async def on_progress(self, payload: Dict[str, Any]) -> None: + """Handle a sidecar migration progress update.""" + + +ScannerFactory = Callable[[], Awaitable[Any]] + +DIRECTION_TO_CENTRALIZED = "to_centralized" +DIRECTION_TO_ALONGSIDE = "to_alongside" + + +class SidecarMigrationUseCase: + """Move sidecars and previews between alongside and centralized layouts.""" + + def __init__( + self, + *, + scanner_factories: Sequence[Tuple[str, ScannerFactory]] | None = None, + settings_service=None, + logger: Optional[logging.Logger] = None, + ) -> None: + self._settings = settings_service or get_settings_manager() + self._scanner_factories: Tuple[Tuple[str, ScannerFactory], ...] = tuple( + scanner_factories + or ( + ("lora", ServiceRegistry.get_lora_scanner), + ("checkpoint", ServiceRegistry.get_checkpoint_scanner), + ("embedding", ServiceRegistry.get_embedding_scanner), + ("other", ServiceRegistry.get_other_scanner), + ) + ) + self._logger = logger or logging.getLogger(__name__) + + async def migrate_to_centralized( + self, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Move sidecars/previews from alongside the models into the mirror root.""" + + if ( + not force + and get_storage_mode() == STORAGE_MODE_CENTRALIZED + and get_sidecar_root() + ): + return self._refusal( + DIRECTION_TO_CENTRALIZED, + "sidecar storage is already centralized; pass force=true to migrate anyway", + ) + return await self._migrate( + direction=DIRECTION_TO_CENTRALIZED, + to_centralized=True, + progress_cb=progress_cb, + ) + + async def migrate_to_alongside( + self, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Move sidecars/previews from the mirror root back next to the models.""" + + if not force and get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return self._refusal( + DIRECTION_TO_ALONGSIDE, + "sidecar storage is already alongside; pass force=true to migrate anyway", + ) + return await self._migrate( + direction=DIRECTION_TO_ALONGSIDE, + to_centralized=False, + progress_cb=progress_cb, + ) + + @staticmethod + def _refusal(direction: str, message: str) -> Dict[str, Any]: + return { + "success": False, + "error": message, + "direction": direction, + "models_total": 0, + "models_processed": 0, + "models_moved": 0, + "moved": 0, + "skipped": 0, + "conflicts": 0, + "errors": [], + "error_count": 0, + } + + def _active_scanner_factories(self) -> Tuple[Tuple[str, ScannerFactory], ...]: + """Drop the opt-in other scanner while Other Models is disabled.""" + + if self._settings.is_other_models_enabled(): + return self._scanner_factories + return tuple(entry for entry in self._scanner_factories if entry[0] != "other") + + async def _collect_model_paths(self, errors: List[Dict[str, str]]) -> List[str]: + """Enumerate model file paths across every active scanner's cache.""" + + paths: List[str] = [] + for model_type, factory in self._active_scanner_factories(): + try: + scanner = await factory() + cache = await scanner.get_cached_data() + except Exception as exc: + self._logger.error( + "Sidecar migration: failed to enumerate %s models: %s", + model_type, + exc, + ) + errors.append({"model": model_type, "error": f"enumeration failed: {exc}"}) + continue + for entry in cache.raw_data: + file_path = entry.get("file_path") + if file_path: + paths.append(file_path) + return paths + + @staticmethod + def _move_file(src: str, dst: str) -> None: + """Move a file, tolerating EXDEV when the layouts span filesystems.""" + + os.makedirs(os.path.dirname(dst), exist_ok=True) + try: + os.rename(src, dst) + except OSError as exc: + if exc.errno != errno.EXDEV: + raise + shutil.copy2(src, dst) + os.remove(src) + + async def _migrate( + self, + *, + direction: str, + to_centralized: bool, + progress_cb: Optional[SidecarMigrationProgressReporter], + ) -> Dict[str, Any]: + root = get_configured_sidecar_root() + if not root: + return self._refusal( + direction, + "cannot resolve the centralized sidecar root", + ) + + errors: List[Dict[str, str]] = [] + model_paths = await self._collect_model_paths(errors) + + total = len(model_paths) + processed = 0 + models_moved = 0 + moved = 0 + skipped = 0 + conflicts = 0 + + async def emit(status: str, **extra: Any) -> None: + if progress_cb is None: + return + payload: Dict[str, Any] = { + "type": "sidecar_migration_progress", + "status": status, + "direction": direction, + "total": total, + "processed": processed, + "moved": moved, + "skipped": skipped, + "conflicts": conflicts, + "errors": len(errors), + } + payload.update(extra) + await progress_cb.on_progress(payload) + + await emit("started") + + for model_path in model_paths: + processed += 1 + current = os.path.basename(model_path) + try: + result = await self._migrate_model( + model_path, + root=root, + to_centralized=to_centralized, + ) + moved += result["moved"] + conflicts += result["conflicts"] + if result["skipped"]: + skipped += 1 + if result["moved"]: + models_moved += 1 + except Exception as exc: + self._logger.error( + "Sidecar migration failed for %s: %s", model_path, exc, exc_info=True + ) + errors.append({"model": current, "error": str(exc)}) + await emit("processing", current=current) + + await emit("completed") + + return { + "success": not errors, + "direction": direction, + "models_total": total, + "models_processed": processed, + "models_moved": models_moved, + "moved": moved, + "skipped": skipped, + "conflicts": conflicts, + "errors": errors, + "error_count": len(errors), + } + + async def _migrate_model( + self, + model_path: str, + *, + root: str, + to_centralized: bool, + ) -> Dict[str, int]: + """Migrate one model's sidecar + previews; return per-model counters.""" + + result = {"moved": 0, "conflicts": 0, "skipped": 0} + + model_path = os.path.abspath(model_path) + if not os.path.exists(model_path): + self._logger.warning( + "Sidecar migration: model file missing, skipping: %s", model_path + ) + result["skipped"] = 1 + return result + + model_dir = os.path.dirname(model_path) + mirror_dir = resolve_centralized_dir_for_dir(model_dir, sidecar_root=root) + if mirror_dir is None: + self._logger.warning( + "Sidecar migration: %s is outside configured model roots, skipping", + model_path, + ) + result["skipped"] = 1 + return result + + if to_centralized: + src_dir, dst_dir = model_dir, mirror_dir + else: + src_dir, dst_dir = mirror_dir, model_dir + + if os.path.normpath(src_dir) == os.path.normpath(dst_dir): + result["skipped"] = 1 + return result + + stem = os.path.splitext(os.path.basename(model_path))[0] + sidecar_name = stem + METADATA_SUFFIX + + moved_previews: List[str] = [] + for ext in PREVIEW_EXTENSIONS: + src = os.path.join(src_dir, stem + ext) + if not os.path.exists(src): + continue + dst = os.path.join(dst_dir, stem + ext) + if self._transfer(src, dst, result): + moved_previews.append(dst) + + sidecar_src = os.path.join(src_dir, sidecar_name) + sidecar_moved = False + sidecar_dst = os.path.join(dst_dir, sidecar_name) + if os.path.exists(sidecar_src): + sidecar_moved = self._transfer(sidecar_src, sidecar_dst, result) + + if sidecar_moved: + await self._rewrite_sidecar_paths(sidecar_dst, model_path, moved_previews) + + return result + + def _transfer(self, src: str, dst: str, result: Dict[str, int]) -> bool: + """Move ``src`` to ``dst`` with keep-newer conflict resolution. + + Returns True when the file was actually moved to the destination. On a + conflict the newer file wins: a newer source replaces the destination; + a newer (or equal) destination is kept and the source is deleted. + """ + + if os.path.exists(dst): + result["conflicts"] += 1 + if os.path.getmtime(src) > os.path.getmtime(dst): + self._logger.info( + "Sidecar migration: conflict at %s; source is newer, replacing", dst + ) + os.remove(dst) + else: + self._logger.info( + "Sidecar migration: conflict at %s; destination is newer, keeping it", + dst, + ) + os.remove(src) + return False + self._move_file(src, dst) + result["moved"] += 1 + return True + + async def _rewrite_sidecar_paths( + self, + sidecar_path: str, + model_path: str, + moved_previews: List[str], + ) -> None: + """Update path fields inside a moved sidecar, mirroring ModelScanner.""" + + with open(sidecar_path, "r", encoding="utf-8") as handle: + metadata = json.load(handle) + + stem = os.path.splitext(os.path.basename(model_path))[0] + metadata["file_path"] = model_path.replace(os.sep, "/") + metadata["file_name"] = stem + + if moved_previews and metadata.get("preview_url"): + recorded_ext = get_preview_extension(metadata["preview_url"]) + chosen = next( + ( + path + for path in moved_previews + if get_preview_extension(path) == recorded_ext + ), + moved_previews[0], + ) + metadata["preview_url"] = chosen.replace(os.sep, "/") + + await MetadataManager.save_metadata(sidecar_path, metadata) + + async def execute_with_error_handling( + self, + *, + direction: str, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + force: bool = False, + ) -> Dict[str, Any]: + """Wrapper providing progress notification on unexpected failures.""" + + try: + if direction == DIRECTION_TO_CENTRALIZED: + return await self.migrate_to_centralized(progress_cb, force=force) + if direction == DIRECTION_TO_ALONGSIDE: + return await self.migrate_to_alongside(progress_cb, force=force) + raise ValueError( + f"direction must be {DIRECTION_TO_CENTRALIZED!r} or {DIRECTION_TO_ALONGSIDE!r}" + ) + except Exception as exc: + if progress_cb is not None: + await progress_cb.on_progress( + { + "type": "sidecar_migration_progress", + "status": "error", + "direction": direction, + "error": str(exc), + } + ) + raise diff --git a/py/utils/metadata_manager.py b/py/utils/metadata_manager.py index 1b03a813..dbd38244 100644 --- a/py/utils/metadata_manager.py +++ b/py/utils/metadata_manager.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional, Type, Union, cast from .models import BaseModelMetadata, CheckpointMetadata, EmbeddingMetadata, LoraMetadata from .file_utils import normalize_path, find_preview_file, calculate_sha256, calculate_autov3 from .lora_metadata import extract_lora_metadata, extract_checkpoint_metadata -from .sidecar_paths import get_metadata_path, resolve_metadata_path +from .sidecar_paths import get_metadata_path, get_preview_dir, resolve_metadata_path logger = logging.getLogger(__name__) @@ -189,6 +189,10 @@ class MetadataManager: temp_path = f"{metadata_path}.tmp" try: + # Centralized sidecar mirrors may not exist yet (unlike the model's + # own directory in alongside mode, which always does). + os.makedirs(os.path.dirname(metadata_path), exist_ok=True) + # Convert to dict if needed if isinstance(metadata, BaseModelMetadata): metadata_dict = metadata.to_dict() @@ -251,10 +255,9 @@ class MetadataManager: try: base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) - + # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # Calculate file hash start_hash_time = time.perf_counter() @@ -378,15 +381,16 @@ class MetadataManager: # Check if preview exists at the current location preview_url = metadata.preview_url if preview_url: - # Get directory parts of both paths - file_dir = os.path.dirname(file_path) + # Get directory parts of both paths; the preview directory is the + # sidecar/preview dir (the model's own dir in alongside mode, the + # centralized mirror otherwise). + file_dir = get_preview_dir(file_path) preview_dir = os.path.dirname(preview_url) # Update preview if it doesn't exist OR if model and preview are in different directories if not os.path.exists(preview_url) or file_dir != preview_dir: base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) - new_preview_url = find_preview_file(base_name, dir_path) + new_preview_url = find_preview_file(base_name, file_dir) if new_preview_url: metadata.preview_url = normalize_path(new_preview_url) need_update = True diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py index 05d89e2e..d43f4b54 100644 --- a/py/utils/sidecar_paths.py +++ b/py/utils/sidecar_paths.py @@ -64,15 +64,8 @@ def is_centralized() -> bool: return get_storage_mode() == STORAGE_MODE_CENTRALIZED and bool(get_sidecar_root()) -def get_sidecar_root() -> str: - """Return the absolute root directory for centralized sidecar storage. - - Empty string when centralized storage is not usable (mode alongside or an - unresolvable configured path). - """ - - if get_storage_mode() != STORAGE_MODE_CENTRALIZED: - return "" +def _resolve_root_from_settings() -> str: + """Resolve the configured/default centralized root, ignoring the active mode.""" configured = _get_settings_value("sidecar_storage_path", "") if configured and isinstance(configured, str): @@ -90,6 +83,31 @@ def get_sidecar_root() -> str: return "" +def get_sidecar_root() -> str: + """Return the absolute root directory for centralized sidecar storage. + + Empty string when centralized storage is not usable (mode alongside or an + unresolvable configured path). + """ + + if get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return "" + + return _resolve_root_from_settings() + + +def get_configured_sidecar_root() -> str: + """Return the centralized sidecar root regardless of the active mode. + + Unlike :func:`get_sidecar_root`, this resolves the configured + ``sidecar_storage_path`` (or the ``/sidecars`` default) even + when the storage mode is ``alongside``. Migration tooling needs both + layouts at once and must not depend on which mode is currently active. + """ + + return _resolve_root_from_settings() + + def sanitize_path_component(name: str) -> str: """Return a filesystem-safe single path component.""" @@ -137,19 +155,38 @@ def resolve_centralized_dir(model_path: str) -> Optional[str]: or the path is not under any configured model root. """ - root = get_sidecar_root() + return resolve_centralized_dir_for_dir( + os.path.dirname(_normalize_for_match(model_path)) + ) + + +def resolve_centralized_dir_for_dir( + model_dir: str, *, sidecar_root: Optional[str] = None +) -> Optional[str]: + """Return the centralized mirror directory for a model *directory*. + + Same layout as :func:`resolve_centralized_dir`, but accepts the directory + itself. Used by folder-level operations (folder rename, mirror-tree walks) + that have no model file path to derive from. Passing a configured model + root returns the mirror base for that root. + + ``sidecar_root`` overrides the root lookup; pass + :func:`get_configured_sidecar_root` to resolve mirror paths independently + of the active storage mode (migration tooling). + """ + + root = sidecar_root if sidecar_root is not None else get_sidecar_root() if not root: return None - target = _normalize_for_match(model_path) - model_dir = os.path.dirname(target) + normalized_dir = _normalize_for_match(model_dir) best_root: Optional[str] = None for candidate in _iter_model_roots(): if not candidate: continue normalized = _normalize_for_match(candidate) - if model_dir == normalized or model_dir.startswith(normalized + os.sep): + if normalized_dir == normalized or normalized_dir.startswith(normalized + os.sep): if best_root is None or len(normalized) > len(best_root): best_root = normalized @@ -163,7 +200,7 @@ def resolve_centralized_dir(model_path: str) -> Optional[str]: except Exception: # pragma: no cover - defensive fallback library = "default" - rel_dir = os.path.relpath(model_dir, best_root) + rel_dir = os.path.relpath(normalized_dir, best_root) parts = [root, sanitize_path_component(library), sanitize_path_component(os.path.basename(best_root))] if rel_dir and rel_dir != os.curdir: parts.extend(sanitize_path_component(part) for part in rel_dir.split(os.sep) if part not in ("", os.curdir)) diff --git a/static/js/managers/SettingsManager.js b/static/js/managers/SettingsManager.js index 7ca0d22e..eb60c202 100644 --- a/static/js/managers/SettingsManager.js +++ b/static/js/managers/SettingsManager.js @@ -1192,6 +1192,9 @@ export class SettingsManager { this.updateExampleImagesOpenSettingsVisibility(); + // Load sidecar storage settings + this.loadSidecarStorageSettings(); + // Load download path templates this.loadDownloadPathTemplates(); @@ -1280,6 +1283,9 @@ export class SettingsManager { this.attachPathField('exampleImagesLocalRoot', { onAfterSelect: () => this.saveInputSetting('exampleImagesLocalRoot', 'example_images_local_root'), }); + this.attachPathField('sidecarStoragePath', { + onAfterSelect: () => this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'), + }); } loadDownloadBackendSettings() { @@ -3379,6 +3385,192 @@ export class SettingsManager { this.updateExampleImagesOpenSettingsVisibility(); } + loadSidecarStorageSettings() { + const currentMode = state.global.settings.sidecar_storage_mode === 'centralized' + ? 'centralized' + : 'alongside'; + + const modeSelect = document.getElementById('sidecarStorageMode'); + if (modeSelect) { + modeSelect.value = currentMode; + } + // Baseline used to detect a mode change in handleSidecarStorageModeChange + this._loadedSidecarStorageMode = currentMode; + + const pathInput = document.getElementById('sidecarStoragePath'); + if (pathInput) { + pathInput.value = state.global.settings.sidecar_storage_path || ''; + } + + this.updateSidecarStorageVisibility(); + } + + updateSidecarStorageVisibility() { + const modeSelect = document.getElementById('sidecarStorageMode'); + const pathSetting = document.getElementById('sidecarStoragePathSetting'); + if (!pathSetting) return; + + const mode = modeSelect ? modeSelect.value : state.global.settings.sidecar_storage_mode; + pathSetting.style.display = mode === 'centralized' ? 'block' : 'none'; + } + + async handleSidecarStorageModeChange() { + const modeSelect = document.getElementById('sidecarStorageMode'); + if (!modeSelect) return; + + const previousMode = this._loadedSidecarStorageMode || 'alongside'; + + await this.saveSelectSetting('sidecarStorageMode', 'sidecar_storage_mode'); + this.updateSidecarStorageVisibility(); + + const newMode = modeSelect.value; + this._loadedSidecarStorageMode = newMode; + + // Existing sidecars are not moved automatically; offer to migrate them. + if (newMode !== previousMode) { + const direction = newMode === 'centralized' ? 'to_centralized' : 'to_alongside'; + const confirmed = await this.confirmSidecarMigration(direction); + if (confirmed) { + await this.migrateSidecars(direction); + } else { + showToast('settings.sidecarStorage.migrationDeferred', {}, 'info'); + } + } + } + + // Entry point for the "Migrate Sidecars Now" button: the direction follows + // the currently saved storage mode. + async confirmAndMigrateSidecars() { + const direction = state.global.settings.sidecar_storage_mode === 'centralized' + ? 'to_centralized' + : 'to_alongside'; + const confirmed = await this.confirmSidecarMigration(direction); + if (confirmed) { + await this.migrateSidecars(direction); + } + } + + confirmSidecarMigration(direction) { + const modalElement = document.getElementById('sidecarMigrationConfirmModal'); + if (!modalElement) { + return Promise.resolve(false); + } + + const isToCentralized = direction === 'to_centralized'; + + const titleElement = modalElement.querySelector('[data-role="title"]'); + if (titleElement) { + titleElement.textContent = isToCentralized + ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') + : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); + } + + const messageElement = modalElement.querySelector('[data-role="message"]'); + if (messageElement) { + messageElement.textContent = isToCentralized + ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') + : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); + } + + const confirmButton = modalElement.querySelector('[data-action="confirm-sidecar-migration"]'); + const cancelButton = modalElement.querySelector('[data-action="cancel-sidecar-migration"]'); + if (!confirmButton || !cancelButton) { + return Promise.resolve(false); + } + + confirmButton.textContent = translate('modals.sidecarMigrationConfirm.confirmButton', {}, 'Migrate Now'); + + return new Promise((resolve) => { + let resolved = false; + + const cleanup = () => { + confirmButton.removeEventListener('click', handleConfirm); + cancelButton.removeEventListener('click', handleCancel); + document.removeEventListener('keydown', handleEscape, true); + }; + + const finalize = (proceed) => { + if (resolved) { + return; + } + resolved = true; + cleanup(); + modalElement.classList.remove('show'); + // Keep body.modal-open: the settings modal underneath is still open. + resolve(proceed); + }; + + const handleConfirm = (event) => { + event.preventDefault(); + finalize(true); + }; + + const handleCancel = (event) => { + event.preventDefault(); + finalize(false); + }; + + // Capture phase + stopPropagation so ESC never reaches the + // settings modal's own ESC handler underneath. + const handleEscape = (event) => { + if (event.key === 'Escape') { + event.stopPropagation(); + finalize(false); + } + }; + + confirmButton.addEventListener('click', handleConfirm); + cancelButton.addEventListener('click', handleCancel); + document.addEventListener('keydown', handleEscape, true); + + modalElement.classList.add('show'); + cancelButton.focus(); + }); + } + + async migrateSidecars(direction) { + const migrateBtn = document.getElementById('migrateSidecarsBtn'); + try { + if (migrateBtn) { + migrateBtn.disabled = true; + migrateBtn.textContent = translate('settings.sidecarStorage.migratingButton', {}, 'Migrating...'); + } + + state.loadingManager?.showSimpleLoading( + translate('settings.sidecarStorage.migrating', {}, 'Migrating sidecars...') + ); + + const response = await fetch('/api/lm/sidecars/migrate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + // The new mode is already saved by the time migration runs, + // so the backend guard requires force=true to confirm the + // "switch first, then migrate" flow. + body: JSON.stringify({ direction, force: true }), + }); + + const data = await response.json(); + if (!response.ok || data.success === false) { + throw new Error(data.error || 'Migration failed'); + } + + state.loadingManager?.hide(); + showToast('settings.sidecarStorage.migrateSuccess', {}, 'success'); + + // Reload so cards pick up metadata/preview paths from the new location + resetAndReload(true); + } catch (error) { + console.error('Error migrating sidecars:', error); + state.loadingManager?.hide(); + showToast('settings.sidecarStorage.migrateFailed', { message: error.message }, 'error'); + } finally { + if (migrateBtn) { + migrateBtn.disabled = false; + migrateBtn.textContent = translate('settings.sidecarStorage.migrateButton', {}, 'Migrate Sidecars Now'); + } + } + } + async loadMetadataArchiveSettings() { try { // Load current settings from state diff --git a/static/js/state/index.js b/static/js/state/index.js index 20eabd7c..f50e5ac8 100644 --- a/static/js/state/index.js +++ b/static/js/state/index.js @@ -62,6 +62,8 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({ download_skip_base_models: [], backup_auto_enabled: true, backup_retention_count: 5, + sidecar_storage_mode: 'alongside', + sidecar_storage_path: '', strip_lora_on_copy: false, use_new_license_icons: true, group_by_model: false, diff --git a/templates/components/modals/confirm_modals.html b/templates/components/modals/confirm_modals.html index cb114da2..6ca24b9d 100644 --- a/templates/components/modals/confirm_modals.html +++ b/templates/components/modals/confirm_modals.html @@ -96,6 +96,20 @@ + + + diff --git a/templates/components/modals/settings/library.html b/templates/components/modals/settings/library.html index 362c9324..89012f89 100644 --- a/templates/components/modals/settings/library.html +++ b/templates/components/modals/settings/library.html @@ -321,4 +321,60 @@ ('civitai_sqlite_archive', 'settings.metadataArchive.providerOrderCivitaiSqliteArchive'), ], 'settings.metadataArchive.providerOrderHelp') }} + + +
+ {{ sm.subsection_header('settings.sections.sidecarStorage') }} +
+
+
+ +
+
+ +
+
+
+ + + +
+
+
+ +
+
+ +
+
+
+
diff --git a/tests/frontend/managers/settingsManager.sidecarStorage.test.js b/tests/frontend/managers/settingsManager.sidecarStorage.test.js new file mode 100644 index 00000000..ae376076 --- /dev/null +++ b/tests/frontend/managers/settingsManager.sidecarStorage.test.js @@ -0,0 +1,284 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +vi.mock('../../../static/js/managers/ModalManager.js', () => ({ + modalManager: { + closeModal: vi.fn(), + }, +})); + +vi.mock('../../../static/js/utils/uiHelpers.js', () => ({ + showToast: vi.fn(), +})); + +vi.mock('../../../static/js/state/index.js', () => { + const settings = {}; + return { + state: { + global: { + settings, + }, + loadingManager: { + showSimpleLoading: vi.fn(), + hide: vi.fn(), + }, + }, + createDefaultSettings: () => ({ + language: 'en', + sidecar_storage_mode: 'alongside', + sidecar_storage_path: '', + }), + }; +}); + +vi.mock('../../../static/js/api/modelApiFactory.js', () => ({ + resetAndReload: vi.fn(), + getModelApiClient: vi.fn(), +})); + +vi.mock('../../../static/js/utils/constants.js', () => ({ + DOWNLOAD_PATH_TEMPLATES: {}, + DEFAULT_PATH_TEMPLATES: {}, + MAPPABLE_BASE_MODELS: [], + PATH_TEMPLATE_PLACEHOLDERS: {}, + FILENAME_TEMPLATE_PLACEHOLDERS: [], + DEFAULT_FILENAME_TEMPLATES: { lora: '', checkpoint: '', embedding: '' }, + DEFAULT_PRIORITY_TAG_CONFIG: { + lora: 'character, style', + checkpoint: 'base, guide', + embedding: 'hint', + }, + getMappableBaseModelsDynamic: () => [], +})); + +vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({ + translate: (_key, _params, fallback) => fallback ?? '', +})); + +vi.mock('../../../static/js/i18n/index.js', () => ({ + i18n: { + getCurrentLocale: () => 'en', + setLanguage: vi.fn().mockResolvedValue(), + }, +})); + +vi.mock('../../../static/js/components/shared/ModelCard.js', () => ({ + configureModelCardVideo: vi.fn(), +})); + +import { SettingsManager } from '../../../static/js/managers/SettingsManager.js'; +import { showToast } from '../../../static/js/utils/uiHelpers.js'; +import { resetAndReload } from '../../../static/js/api/modelApiFactory.js'; +import { state } from '../../../static/js/state/index.js'; + +const createManager = () => { + state.global.settings = {}; + const initSettingsSpy = vi + .spyOn(SettingsManager.prototype, 'initializeSettings') + .mockResolvedValue(); + const initializeSpy = vi + .spyOn(SettingsManager.prototype, 'initialize') + .mockImplementation(() => {}); + + const manager = new SettingsManager(); + + initSettingsSpy.mockRestore(); + initializeSpy.mockRestore(); + + return manager; +}; + +const appendSidecarControls = () => { + const select = document.createElement('select'); + select.id = 'sidecarStorageMode'; + ['alongside', 'centralized'].forEach((value) => { + const option = document.createElement('option'); + option.value = value; + select.appendChild(option); + }); + + const pathSetting = document.createElement('div'); + pathSetting.id = 'sidecarStoragePathSetting'; + pathSetting.style.display = 'none'; + + const pathInput = document.createElement('input'); + pathInput.id = 'sidecarStoragePath'; + + const migrateBtn = document.createElement('button'); + migrateBtn.id = 'migrateSidecarsBtn'; + + document.body.append(select, pathSetting, pathInput, migrateBtn); + return { select, pathSetting, pathInput, migrateBtn }; +}; + +const appendMigrationModal = () => { + const modal = document.createElement('div'); + modal.id = 'sidecarMigrationConfirmModal'; + modal.innerHTML = ` +

+

+ + `; + document.body.appendChild(modal); + return modal; +}; + +const mockFetchOk = (payload = { success: true }) => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(payload), + }); +}; + +beforeEach(() => { + document.body.innerHTML = ''; + vi.clearAllMocks(); +}); + +afterEach(() => { + delete global.fetch; +}); + +describe('SettingsManager sidecar storage', () => { + describe('loadSidecarStorageSettings', () => { + it('loads alongside mode and hides the centralized path input', () => { + const manager = createManager(); + const { select, pathSetting, pathInput } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'alongside', sidecar_storage_path: '/data/sidecars' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('alongside'); + expect(pathInput.value).toBe('/data/sidecars'); + expect(pathSetting.style.display).toBe('none'); + expect(manager._loadedSidecarStorageMode).toBe('alongside'); + }); + + it('loads centralized mode and shows the path input', () => { + const manager = createManager(); + const { select, pathSetting } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('centralized'); + expect(pathSetting.style.display).toBe('block'); + expect(manager._loadedSidecarStorageMode).toBe('centralized'); + }); + + it('falls back to alongside for unknown stored modes', () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'bogus' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('alongside'); + }); + }); + + describe('handleSidecarStorageModeChange', () => { + it('does not prompt for migration when the mode is unchanged', async () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside' }; + manager._loadedSidecarStorageMode = 'alongside'; + select.value = 'alongside'; + mockFetchOk(); + + await manager.handleSidecarStorageModeChange(); + + expect(global.fetch).not.toHaveBeenCalledWith( + '/api/lm/sidecars/migrate', + expect.anything() + ); + expect(showToast).toHaveBeenCalledWith( + 'toast.settings.settingsUpdated', + expect.anything(), + 'success' + ); + }); + + it('migrates to centralized after the user confirms the prompt', async () => { + const manager = createManager(); + const { select, pathSetting } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside' }; + manager._loadedSidecarStorageMode = 'alongside'; + select.value = 'centralized'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStorageModeChange(); + + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await changePromise; + + expect(state.global.settings.sidecar_storage_mode).toBe('centralized'); + expect(pathSetting.style.display).toBe('block'); + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ direction: 'to_centralized', force: true }), + })); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrateSuccess', {}, 'success'); + expect(resetAndReload).toHaveBeenCalledWith(true); + expect(modal.classList.contains('show')).toBe(false); + }); + + it('shows a deferred notice and skips migration when the user cancels', async () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + manager._loadedSidecarStorageMode = 'centralized'; + select.value = 'alongside'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStorageModeChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="cancel-sidecar-migration"]').click(); + await changePromise; + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrationDeferred', {}, 'info'); + }); + }); + + describe('confirmAndMigrateSidecars', () => { + it('derives the migration direction from the saved mode', async () => { + const manager = createManager(); + appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + mockFetchOk(); + + const confirmPromise = manager.confirmAndMigrateSidecars(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await confirmPromise; + + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + body: JSON.stringify({ direction: 'to_centralized', force: true }), + })); + }); + }); + + describe('migrateSidecars', () => { + it('surfaces backend failures as an error toast', async () => { + const manager = createManager(); + const { migrateBtn } = appendSidecarControls(); + mockFetchOk({ success: false, error: 'disk full' }); + + await manager.migrateSidecars('to_alongside'); + + expect(showToast).toHaveBeenCalledWith( + 'settings.sidecarStorage.migrateFailed', + { message: 'disk full' }, + 'error' + ); + expect(resetAndReload).not.toHaveBeenCalled(); + expect(migrateBtn.disabled).toBe(false); + }); + }); +}); diff --git a/tests/routes/test_misc_routes.py b/tests/routes/test_misc_routes.py index cb8406a8..8ea65248 100644 --- a/tests/routes/test_misc_routes.py +++ b/tests/routes/test_misc_routes.py @@ -23,6 +23,7 @@ from py.routes.handlers.misc_handlers import ( NodeRegistryHandler, ServiceRegistryAdapter, SettingsHandler, + SidecarMigrationHandler, _collect_comfyui_session_logs, _is_wsl, _wsl_to_windows_path, @@ -2557,3 +2558,86 @@ async def test_get_model_versions_status_supported_type_stays_interactive(): "hasBeenDownloaded": False, } ] + + +class DummySidecarMigrationUseCase: + def __init__(self, result): + self.result = result + self.calls = [] + + async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False): + self.calls.append({"direction": direction, "force": force}) + return self.result + + +def _sidecar_migration_handler(result): + use_case = DummySidecarMigrationUseCase(result) + handler = SidecarMigrationHandler( + use_case_factory=lambda: use_case, + progress_callback_factory=lambda: None, + ) + return handler, use_case + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_runs_to_centralized(): + result = {"success": True, "direction": "to_centralized", "moved": 3} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "to_centralized", "force": True}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert payload["moved"] == 3 + assert use_case.calls == [{"direction": "to_centralized", "force": True}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_rejects_bad_direction(): + handler, use_case = _sidecar_migration_handler({"success": True}) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "sideways"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert payload["success"] is False + assert use_case.calls == [] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_accepts_get_query_params(): + result = {"success": True, "direction": "to_alongside", "moved": 0} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest( # pyright: ignore[reportArgumentType] + query={"direction": "to_alongside", "force": "true"}, + method="GET", + ) + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert use_case.calls == [{"direction": "to_alongside", "force": True}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_guard_refusal_is_400(): + result = {"success": False, "error": "sidecar storage is already centralized"} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "to_centralized"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert payload["success"] is False + assert "already centralized" in payload["error"] + assert use_case.calls == [{"direction": "to_centralized", "force": False}] diff --git a/tests/services/test_centralized_sidecar_storage.py b/tests/services/test_centralized_sidecar_storage.py new file mode 100644 index 00000000..2d9164ab --- /dev/null +++ b/tests/services/test_centralized_sidecar_storage.py @@ -0,0 +1,361 @@ +"""Centralized sidecar storage: lifecycle flows (delete/move/rename/scans).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import AsyncMock + +import pytest + +from py.config import config +from py.services.checkpoint_scanner import CheckpointScanner +from py.services.model_lifecycle_service import ( + ModelLifecycleService, + delete_model_artifacts, +) +from py.services.pending_delete_service import PendingDeleteService +from py.services.settings_manager import get_settings_manager +from py.utils.metadata_manager import MetadataManager + + +def _normalize(path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def library_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Single checkpoint root under tmp_path; every other root emptied.""" + + root = tmp_path / "checkpoints" + root.mkdir() + for attr, value in ( + ("loras_roots", []), + ("base_models_roots", [str(root)]), + ("checkpoints_roots", [str(root)]), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + return root + + +@pytest.fixture +def centralized(library_root: Path, tmp_path: Path) -> Path: + """Enable centralized mode rooted at tmp_path/sidecars.""" + + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", str(sidecar_root)) + return sidecar_root + + +def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: + """Expected mirror directory for a library-relative path.""" + + library = get_settings_manager().get_active_library_name() + return sidecar_root.joinpath(library, "checkpoints", *rel) + + +def _write_sidecar( + mirror_dir: Path, + stem: str, + *, + file_path: str, + preview_name: str | None = None, + extra: Dict[str, Any] | None = None, +) -> Path: + mirror_dir.mkdir(parents=True, exist_ok=True) + payload: Dict[str, Any] = { + "file_name": stem, + "file_path": _normalize(file_path), + } + if preview_name: + payload["preview_url"] = _normalize(mirror_dir / preview_name) + (mirror_dir / preview_name).write_bytes(b"preview") + if extra: + payload.update(extra) + metadata_path = mirror_dir / f"{stem}.metadata.json" + metadata_path.write_text(json.dumps(payload), encoding="utf-8") + return metadata_path + + +@pytest.mark.asyncio +async def test_delete_model_artifacts_centralized( + library_root: Path, centralized: Path +): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") + + deleted = await delete_model_artifacts(str(library_root), "model") + + assert not model.exists() + assert not (mirror / "model.metadata.json").exists() + assert not (mirror / "model.preview.webp").exists() + assert any(path.endswith("model.safetensors") for path in deleted) + assert "model.metadata.json" in deleted + assert "model.preview.webp" in deleted + + +@pytest.mark.asyncio +async def test_delete_model_artifacts_alongside_still_siblings( + library_root: Path, +): + """Alongside mode (default) keeps deleting sidecars next to the model.""" + + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + sidecar = library_root / "model.metadata.json" + sidecar.write_text("{}", encoding="utf-8") + preview = library_root / "model.preview.webp" + preview.write_bytes(b"preview") + + await delete_model_artifacts(str(library_root), "model") + + assert not model.exists() + assert not sidecar.exists() + assert not preview.exists() + + +def test_enumerate_model_artifacts_centralized( + library_root: Path, centralized: Path +): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.png") + + service = PendingDeleteService.__new__(PendingDeleteService) + artifacts = service._enumerate_model_artifacts( + str(library_root), "model", ".safetensors" + ) + + assert artifacts == [ + os.path.abspath(str(model)), + os.path.abspath(str(mirror / "model.metadata.json")), + os.path.abspath(str(mirror / "model.preview.png")), + ] + + +class _RecordingScanner: + def __init__(self): + self.calls: List[tuple] = [] + self.model_type = "lora" + + async def update_single_model_cache(self, old_path, new_path, metadata): + self.calls.append((old_path, new_path, metadata)) + + +class _PassthroughMetadataManager: + async def save_metadata(self, path: str, metadata): + await MetadataManager.save_metadata(path, metadata) + return True + + +async def _json_metadata_loader(path: str) -> Dict[str, object]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +@pytest.mark.asyncio +async def test_rename_model_centralized(library_root: Path, centralized: Path): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") + + service = ModelLifecycleService( + scanner=_RecordingScanner(), + metadata_manager=_PassthroughMetadataManager(), + metadata_loader=_json_metadata_loader, + ) + + result = await service.rename_model( + file_path=_normalize(model), new_file_name="renamed" + ) + + assert result["success"] is True + renamed = library_root / "renamed.safetensors" + assert renamed.exists() + assert not model.exists() + + # Mirror sidecar/preview renamed with the model; nothing left alongside. + assert (mirror / "renamed.metadata.json").exists() + assert (mirror / "renamed.preview.webp").exists() + assert not (mirror / "model.metadata.json").exists() + assert not (mirror / "model.preview.webp").exists() + assert not (library_root / "renamed.metadata.json").exists() + assert not (library_root / "renamed.preview.webp").exists() + + saved = json.loads((mirror / "renamed.metadata.json").read_text()) + assert saved["file_name"] == "renamed" + assert saved["file_path"].endswith("renamed.safetensors") + assert saved["preview_url"].endswith("renamed.preview.webp") + assert str(mirror).replace(os.sep, "/") in saved["preview_url"] + + +@pytest.mark.asyncio +async def test_move_model_centralized( + library_root: Path, centralized: Path, monkeypatch: pytest.MonkeyPatch +): + source_dir = library_root / "old" + source_dir.mkdir() + model = source_dir / "model.safetensors" + model.write_bytes(b"weights") + target_dir = library_root / "new" + + old_mirror = _mirror_dir(centralized, "old") + _write_sidecar( + old_mirror, "model", file_path=model, preview_name="model.preview.webp" + ) + + scanner = CheckpointScanner() + monkeypatch.setattr( + scanner, "update_single_model_cache", AsyncMock(return_value=True) + ) + + result = await scanner.move_model(_normalize(model), _normalize(target_dir)) + + assert result is not None + moved_model = target_dir / "model.safetensors" + assert moved_model.exists() + assert not model.exists() + + new_mirror = _mirror_dir(centralized, "new") + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.webp").exists() + assert not (old_mirror / "model.metadata.json").exists() + assert not (old_mirror / "model.preview.webp").exists() + + saved = json.loads((new_mirror / "model.metadata.json").read_text()) + assert saved["file_path"].endswith("new/model.safetensors") + assert saved["preview_url"].endswith("model.preview.webp") + assert str(new_mirror).replace(os.sep, "/") in saved["preview_url"] + + +class _FakeCache: + def __init__(self, raw_data: List[Dict[str, Any]]): + self.raw_data = raw_data + self.all_folders: List[str] = [] + self.folders: List[str] = [] + + def remove_from_version_index(self, _item) -> None: + pass + + def rebuild_version_index(self) -> None: + pass + + async def resort(self) -> None: + pass + + +@pytest.mark.asyncio +async def test_rename_known_folder_centralized( + library_root: Path, centralized: Path, monkeypatch: pytest.MonkeyPatch +): + # The caller (ModelFileService.rename_folder) has already renamed the + # model directory on disk when the scanner is asked to re-key records. + old_dir = library_root / "oldfolder" + new_dir = library_root / "newfolder" + new_dir.mkdir() + model = new_dir / "model.safetensors" + model.write_bytes(b"weights") + + old_model_path = old_dir / "model.safetensors" + old_mirror = _mirror_dir(centralized, "oldfolder") + _write_sidecar( + old_mirror, "model", file_path=old_model_path, preview_name="model.preview.webp" + ) + + cache_entry: Dict[str, Any] = { + "file_path": _normalize(old_model_path), + "folder": "oldfolder", + "preview_url": _normalize(old_mirror / "model.preview.webp"), + "sha256": "", + } + scanner = CheckpointScanner() + scanner._cache = _FakeCache([cache_entry]) + monkeypatch.setattr(scanner, "_persist_current_cache", AsyncMock()) + + changed = await scanner.rename_known_folder( + "oldfolder", + "newfolder", + previous_path=str(old_dir), + new_path=str(new_dir), + ) + + assert changed is True + + new_mirror = _mirror_dir(centralized, "newfolder") + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.webp").exists() + assert not old_mirror.exists() + + saved = json.loads((new_mirror / "model.metadata.json").read_text()) + assert saved["file_path"] == _normalize(model) + assert saved["preview_url"] == _normalize(new_mirror / "model.preview.webp") + + assert cache_entry["file_path"] == _normalize(model) + assert cache_entry["folder"] == "newfolder" + assert cache_entry["preview_url"] == _normalize( + new_mirror / "model.preview.webp" + ) + + +@pytest.mark.asyncio +async def test_pending_models_mirror_walk(library_root: Path, centralized: Path): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar( + mirror, + "model", + file_path=model, + extra={"hash_status": "pending", "sha256": ""}, + ) + # Orphan sidecar: recorded model path is gone and no stem match exists. + _write_sidecar( + mirror, + "ghost", + file_path=library_root / "ghost.safetensors", + extra={"hash_status": "pending", "sha256": ""}, + ) + + scanner = CheckpointScanner() + pending = await scanner._find_pending_models_from_filesystem() + + assert len(pending) == 1 + assert pending[0]["file_path"] == _normalize(model) + assert pending[0]["hash_status"] == "pending" + + +@pytest.mark.asyncio +async def test_pending_models_mirror_walk_uses_stem_fallback( + library_root: Path, centralized: Path +): + """A stale recorded file_path falls back to probing by stem + extension.""" + + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar( + mirror, + "model", + file_path=library_root / "renamed-away.safetensors", + extra={"hash_status": "pending", "sha256": ""}, + ) + + scanner = CheckpointScanner() + pending = await scanner._find_pending_models_from_filesystem() + + assert len(pending) == 1 + assert pending[0]["file_path"] == _normalize(model) diff --git a/tests/services/use_cases/test_sidecar_migration_use_case.py b/tests/services/use_cases/test_sidecar_migration_use_case.py new file mode 100644 index 00000000..fca8c43f --- /dev/null +++ b/tests/services/use_cases/test_sidecar_migration_use_case.py @@ -0,0 +1,294 @@ +"""Sidecar migration use case: layout moves, conflicts, guards.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from py.config import config +from py.services.settings_manager import get_settings_manager +from py.services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase + + +def _normalize(path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def library_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Single lora root under tmp_path; every other root emptied.""" + + root = tmp_path / "loras" + root.mkdir() + for attr, value in ( + ("loras_roots", [str(root)]), + ("base_models_roots", []), + ("checkpoints_roots", []), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + return root + + +@pytest.fixture +def sidecar_root(tmp_path: Path) -> Path: + """Configured centralized root (mode-independent for migration).""" + + root = tmp_path / "sidecars" + get_settings_manager().set("sidecar_storage_path", str(root)) + return root + + +def _set_mode(mode: str) -> None: + get_settings_manager().set("sidecar_storage_mode", mode) + + +def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: + """Expected mirror directory for a library-relative path.""" + + library = get_settings_manager().get_active_library_name() + return sidecar_root.joinpath(library, "loras", *rel) + + +def _write_model(directory: Path, stem: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + model = directory / f"{stem}.safetensors" + model.write_bytes(b"weights") + return model + + +def _write_sidecar(directory: Path, stem: str, model: Path, *, preview_ext: str | None = ".preview.webp") -> Path: + directory.mkdir(parents=True, exist_ok=True) + payload: Dict[str, Any] = { + "file_name": stem, + "file_path": _normalize(model), + } + if preview_ext: + payload["preview_url"] = _normalize(directory / f"{stem}{preview_ext}") + sidecar = directory / f"{stem}.metadata.json" + sidecar.write_text(json.dumps(payload), encoding="utf-8") + return sidecar + + +class _FakeCache: + def __init__(self, raw_data: List[Dict[str, Any]]) -> None: + self.raw_data = raw_data + + +class _FakeScanner: + def __init__(self, raw_data: List[Dict[str, Any]]) -> None: + self._cache = _FakeCache(raw_data) + + async def get_cached_data(self) -> _FakeCache: + return self._cache + + +def _make_use_case(model_paths: List[str]) -> SidecarMigrationUseCase: + scanner = _FakeScanner([{"file_path": path} for path in model_paths]) + + async def scanner_factory() -> _FakeScanner: + return scanner + + return SidecarMigrationUseCase( + scanner_factories=(("lora", scanner_factory),), + settings_service=get_settings_manager(), + ) + + +class _ProgressRecorder: + def __init__(self) -> None: + self.payloads: List[Dict[str, Any]] = [] + + async def on_progress(self, payload: Dict[str, Any]) -> None: + self.payloads.append(payload) + + +@pytest.mark.asyncio +async def test_migrate_to_centralized_moves_sidecar_and_previews( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + model = _write_model(library_root / "sub", "model") + sidecar = _write_sidecar(library_root / "sub", "model", model) + preview = library_root / "sub" / "model.preview.webp" + preview.write_bytes(b"preview") + extra_preview = library_root / "sub" / "model.png" + extra_preview.write_bytes(b"extra") + + recorder = _ProgressRecorder() + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_centralized(recorder, force=True) + + assert summary["success"] is True + assert summary["direction"] == "to_centralized" + assert summary["moved"] == 3 + assert summary["models_moved"] == 1 + assert summary["skipped"] == 0 + assert summary["conflicts"] == 0 + assert summary["errors"] == [] + + mirror = _mirror_dir(sidecar_root, "sub") + assert not sidecar.exists() + assert not preview.exists() + assert not extra_preview.exists() + moved_sidecar = mirror / "model.metadata.json" + assert moved_sidecar.exists() + assert (mirror / "model.preview.webp").exists() + assert (mirror / "model.png").exists() + # Model files never move. + assert model.exists() + + metadata = json.loads(moved_sidecar.read_text(encoding="utf-8")) + assert metadata["file_path"] == _normalize(model) + assert metadata["file_name"] == "model" + # Recorded extension wins when rewriting preview_url. + assert metadata["preview_url"] == _normalize(mirror / "model.preview.webp") + + statuses = [payload["status"] for payload in recorder.payloads] + assert statuses[0] == "started" + assert statuses[-1] == "completed" + assert all(p["type"] == "sidecar_migration_progress" for p in recorder.payloads) + + +@pytest.mark.asyncio +async def test_migrate_to_alongside_reverses_layout( + library_root: Path, sidecar_root: Path +): + _set_mode("alongside") + model = _write_model(library_root / "sub", "model") + mirror = _mirror_dir(sidecar_root, "sub") + sidecar = _write_sidecar(mirror, "model", model) + preview = mirror / "model.preview.webp" + preview.write_bytes(b"preview") + + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_alongside(force=True) + + assert summary["success"] is True + assert summary["moved"] == 2 + + assert not sidecar.exists() + assert not preview.exists() + moved_sidecar = library_root / "sub" / "model.metadata.json" + assert moved_sidecar.exists() + assert (library_root / "sub" / "model.preview.webp").exists() + + metadata = json.loads(moved_sidecar.read_text(encoding="utf-8")) + assert metadata["file_path"] == _normalize(model) + assert metadata["preview_url"] == _normalize( + library_root / "sub" / "model.preview.webp" + ) + + +@pytest.mark.asyncio +async def test_migrate_conflict_keeps_newer_file( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + mirror = _mirror_dir(sidecar_root) + + # Model A: destination (mirror) sidecar is newer -> destination wins. + model_a = _write_model(library_root, "model_a") + src_a = _write_sidecar(library_root, "model_a", model_a, preview_ext=None) + dst_a = _write_sidecar(mirror, "model_a", model_a, preview_ext=None) + os.utime(src_a, (1000, 1000)) + os.utime(dst_a, (2000, 2000)) + + # Model B: source (alongside) sidecar is newer -> source replaces. + model_b = _write_model(library_root, "model_b") + src_b = _write_sidecar(library_root, "model_b", model_b, preview_ext=None) + dst_b = _write_sidecar(mirror, "model_b", model_b, preview_ext=None) + (mirror / "model_b.metadata.json").write_text( + json.dumps({"stale": True}), encoding="utf-8" + ) + os.utime(src_b, (3000, 3000)) + os.utime(dst_b, (2000, 2000)) + + use_case = _make_use_case([str(model_a), str(model_b)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["conflicts"] == 2 + assert summary["moved"] == 1 + + # A: destination kept, source deleted, content untouched. + assert not src_a.exists() + metadata_a = json.loads(dst_a.read_text(encoding="utf-8")) + assert metadata_a["file_name"] == "model_a" + + # B: newer source replaced the stale destination. + assert not src_b.exists() + metadata_b = json.loads(dst_b.read_text(encoding="utf-8")) + assert metadata_b.get("stale") is None + assert metadata_b["file_name"] == "model_b" + + +@pytest.mark.asyncio +async def test_migrate_missing_model_file_is_skipped( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + missing_model = library_root / "ghost.safetensors" + sidecar = _write_sidecar(library_root, "ghost", missing_model, preview_ext=None) + + use_case = _make_use_case([str(missing_model)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is True + assert summary["skipped"] == 1 + assert summary["moved"] == 0 + # Sidecar stays put when the model file is gone. + assert sidecar.exists() + + +@pytest.mark.asyncio +async def test_migrate_empty_library_is_noop( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + recorder = _ProgressRecorder() + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_centralized(recorder, force=True) + + assert summary["success"] is True + assert summary["models_total"] == 0 + assert summary["moved"] == 0 + statuses = [payload["status"] for payload in recorder.payloads] + assert statuses == ["started", "completed"] + + +@pytest.mark.asyncio +async def test_migrate_to_centralized_refuses_when_already_centralized( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_centralized() + + assert summary["success"] is False + assert "already centralized" in summary["error"] + assert summary["moved"] == 0 + + +@pytest.mark.asyncio +async def test_migrate_to_alongside_refuses_when_already_alongside( + library_root: Path, sidecar_root: Path +): + _set_mode("alongside") + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_alongside() + + assert summary["success"] is False + assert "already alongside" in summary["error"] + assert summary["moved"] == 0 diff --git a/tests/utils/test_sidecar_paths.py b/tests/utils/test_sidecar_paths.py new file mode 100644 index 00000000..f09966d9 --- /dev/null +++ b/tests/utils/test_sidecar_paths.py @@ -0,0 +1,250 @@ +"""Tests for py.utils.sidecar_paths (alongside + centralized storage modes).""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from py.services.settings_manager import get_settings_manager +from py.utils import sidecar_paths +from py.utils.sidecar_paths import ( + METADATA_SUFFIX, + get_configured_sidecar_root, + get_metadata_path, + get_preview_dir, + get_sidecar_dir, + get_sidecar_root, + get_storage_mode, + is_centralized, + is_metadata_path, + resolve_centralized_dir, + resolve_centralized_dir_for_dir, + resolve_metadata_path, + sanitize_path_component, +) + + +def _normalize(path: Path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def model_roots(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: + """Point every config model root the sidecar module reads at tmp_path.""" + + from py.config import config + + loras = tmp_path / "loras" + checkpoints = tmp_path / "checkpoints" + loras.mkdir() + checkpoints.mkdir() + + for attr, value in ( + ("loras_roots", [str(loras)]), + ("base_models_roots", [str(checkpoints)]), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + + return {"loras": loras, "checkpoints": checkpoints} + + +@pytest.fixture +def centralized(model_roots: dict, tmp_path: Path) -> Path: + """Enable centralized mode rooted at tmp_path/sidecars; returns the root.""" + + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", str(sidecar_root)) + return sidecar_root + + +class TestAlongsideMode: + def test_default_mode_is_alongside(self): + assert get_storage_mode() == "alongside" + assert not is_centralized() + assert get_sidecar_root() == "" + + def test_metadata_path_next_to_model(self, tmp_path: Path): + model = tmp_path / "sub" / "model.safetensors" + assert get_metadata_path(str(model)) == os.path.join( + str(tmp_path), "sub", "model" + METADATA_SUFFIX + ) + + def test_preview_and_sidecar_dir_are_model_dir(self, tmp_path: Path): + model = tmp_path / "sub" / "model.safetensors" + expected = os.path.dirname(os.path.abspath(str(model))) + assert get_sidecar_dir(str(model)) == expected + assert get_preview_dir(str(model)) == expected + + +class TestPathPredicates: + def test_is_metadata_path(self): + assert is_metadata_path("/x/model.metadata.json") + assert not is_metadata_path("/x/model.safetensors") + assert not is_metadata_path("/x/model.metadata.json.bak") + + def test_resolve_metadata_path_passthrough_for_sidecar(self): + sidecar = "/x/model.metadata.json" + assert resolve_metadata_path(sidecar) == sidecar + + def test_resolve_metadata_path_derives_for_model(self, tmp_path: Path): + model = tmp_path / "model.safetensors" + assert resolve_metadata_path(str(model)) == get_metadata_path(str(model)) + + +class TestSanitizePathComponent: + def test_special_characters_replaced(self): + assert sanitize_path_component("foo bar/baz:qux") == "foo_bar_baz_qux" + + def test_safe_characters_kept(self): + assert sanitize_path_component("Flux-1.dev_v2") == "Flux-1.dev_v2" + + def test_empty_falls_back_to_underscore(self): + assert sanitize_path_component("") == "_" + assert sanitize_path_component(None) == "_" + + +class TestCentralizedMode: + def test_mirror_layout(self, model_roots: dict, centralized: Path): + model = model_roots["loras"] / "styles" / "anime" / "model.safetensors" + library = get_settings_manager().get_active_library_name() + + metadata_path = get_metadata_path(str(model)) + + expected = os.path.join( + str(centralized), library, "loras", "styles", "anime", "model" + METADATA_SUFFIX + ) + assert metadata_path == expected + assert get_preview_dir(str(model)) == os.path.dirname(expected) + assert is_centralized() + + def test_longest_root_wins(self, model_roots: dict, centralized: Path, monkeypatch: pytest.MonkeyPatch): + from py.config import config + + nested = model_roots["loras"] / "nested" + nested.mkdir() + monkeypatch.setattr( + config, + "loras_roots", + [str(model_roots["loras"]), str(nested)], + raising=False, + ) + library = get_settings_manager().get_active_library_name() + + model = nested / "model.safetensors" + assert get_metadata_path(str(model)) == os.path.join( + str(centralized), library, "nested", "model" + METADATA_SUFFIX + ) + + def test_outside_roots_falls_back_to_alongside( + self, model_roots: dict, centralized: Path, tmp_path: Path + ): + outside = tmp_path / "elsewhere" / "model.safetensors" + + assert resolve_centralized_dir(str(outside)) is None + assert get_sidecar_dir(str(outside)) == os.path.dirname( + os.path.abspath(str(outside)) + ) + assert get_metadata_path(str(outside)) == os.path.join( + str(tmp_path), "elsewhere", "model" + METADATA_SUFFIX + ) + + def test_resolve_centralized_dir_for_dir_matches_model_resolution( + self, model_roots: dict, centralized: Path + ): + model_dir = model_roots["checkpoints"] / "sub" + model = model_dir / "model.safetensors" + + assert resolve_centralized_dir_for_dir(str(model_dir)) == resolve_centralized_dir( + str(model) + ) + + def test_resolve_centralized_dir_for_dir_root_maps_to_mirror_base( + self, model_roots: dict, centralized: Path + ): + library = get_settings_manager().get_active_library_name() + + assert resolve_centralized_dir_for_dir(str(model_roots["loras"])) == os.path.join( + str(centralized), library, "loras" + ) + + def test_empty_path_uses_default_sidecar_root(self, model_roots: dict, tmp_path: Path): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", "") + + root = get_sidecar_root() + assert root + assert root.endswith(os.sep + "sidecars") + assert is_centralized() + + +class TestModeIndependentResolution: + """Migration tooling resolves the mirror layout regardless of active mode.""" + + def test_configured_root_resolves_in_alongside_mode(self, tmp_path: Path): + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + settings.set("sidecar_storage_path", str(sidecar_root)) + + assert get_sidecar_root() == "" + assert get_configured_sidecar_root() == os.path.abspath(str(sidecar_root)) + + def test_configured_root_defaults_to_settings_dir(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + settings.set("sidecar_storage_path", "") + + root = get_configured_sidecar_root() + assert root + assert root.endswith(os.sep + "sidecars") + + def test_resolve_centralized_dir_for_dir_with_explicit_root( + self, model_roots: dict, tmp_path: Path + ): + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + library = settings.get_active_library_name() + + model_dir = model_roots["loras"] / "sub" + + # Alongside mode: no root resolves without the override. + assert resolve_centralized_dir_for_dir(str(model_dir)) is None + assert resolve_centralized_dir_for_dir( + str(model_dir), sidecar_root=str(sidecar_root) + ) == os.path.join(str(sidecar_root), library, "loras", "sub") + + +class TestSettingsValidation: + def test_invalid_mode_falls_back_to_alongside(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "bogus") + assert settings.get("sidecar_storage_mode") == "alongside" + + def test_mode_is_normalized(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", " Centralized ") + assert settings.get("sidecar_storage_mode") == "centralized" + + def test_path_is_normalized_to_absolute(self, tmp_path: Path): + settings = get_settings_manager() + settings.set("sidecar_storage_path", str(tmp_path / "sidecars")) + assert settings.get("sidecar_storage_path") == os.path.abspath( + str(tmp_path / "sidecars") + ) + + def test_non_string_path_becomes_empty(self): + settings = get_settings_manager() + settings.set("sidecar_storage_path", None) + assert settings.get("sidecar_storage_path") == "" From 16430aef214d79d1d8a9e1ceef5281499e6497a0 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 26 Sep 2026 11:32:18 +0800 Subject: [PATCH 3/4] fix: reconcile scanner caches after sidecar migration Sandbox E2E showed that after a migration the list API kept serving pre-migration preview_url values; the first request to a stale URL made the preview route's stale-URL cleanup wipe the reference from the cache entirely, recoverable only by a full rebuild rescan. The use case now records each migrated model's final preview location (from the destination directory, covering conflict-keep cases), updates the owning scanner's cache entries via ModelCache.update_preview_url, and persists the cache. Per-scanner reconcile failures are logged and skipped; per-model migration errors no longer prevent reconciliation of the healthy models. Verified end-to-end in a sandboxed standalone server: after to_centralized and to_alongside migrations the list endpoint immediately returns the correct preview URLs with no rescan, previews serve with HTTP 200 in both layouts, and the mirror tree is empty after migrating back. --- .../use_cases/sidecar_migration_use_case.py | 143 +++++++++++++----- .../test_sidecar_migration_use_case.py | 97 +++++++++++- 2 files changed, 203 insertions(+), 37 deletions(-) diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py index 9f50b6ff..4bb3756b 100644 --- a/py/services/use_cases/sidecar_migration_use_case.py +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -11,6 +11,8 @@ This use case moves the ``.metadata.json`` sidecar and preview files for every known model from one layout to the other. Model files themselves NEVER move. Paths inside the moved sidecar (``file_path``, ``file_name``, ``preview_url``) are rewritten the same way :meth:`ModelScanner._update_metadata_paths` does. +After the move, scanner caches are reconciled so the list API immediately +serves the new preview locations instead of stale pre-migration URLs. Intended flow (settings-first): @@ -45,7 +47,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Seq from ..service_registry import ServiceRegistry from ..settings_manager import get_settings_manager from ...utils.constants import PREVIEW_EXTENSIONS -from ...utils.file_utils import get_preview_extension +from ...utils.file_utils import find_preview_file, get_preview_extension from ...utils.metadata_manager import MetadataManager from ...utils.sidecar_paths import ( METADATA_SUFFIX, @@ -157,10 +159,12 @@ class SidecarMigrationUseCase: return self._scanner_factories return tuple(entry for entry in self._scanner_factories if entry[0] != "other") - async def _collect_model_paths(self, errors: List[Dict[str, str]]) -> List[str]: - """Enumerate model file paths across every active scanner's cache.""" + async def _collect_model_paths( + self, errors: List[Dict[str, str]] + ) -> List[Tuple[Any, List[str]]]: + """Enumerate model file paths grouped by the scanner that owns them.""" - paths: List[str] = [] + groups: List[Tuple[Any, List[str]]] = [] for model_type, factory in self._active_scanner_factories(): try: scanner = await factory() @@ -173,11 +177,13 @@ class SidecarMigrationUseCase: ) errors.append({"model": model_type, "error": f"enumeration failed: {exc}"}) continue - for entry in cache.raw_data: - file_path = entry.get("file_path") - if file_path: - paths.append(file_path) - return paths + paths = [ + entry["file_path"] + for entry in cache.raw_data + if entry.get("file_path") + ] + groups.append((scanner, paths)) + return groups @staticmethod def _move_file(src: str, dst: str) -> None: @@ -207,14 +213,17 @@ class SidecarMigrationUseCase: ) errors: List[Dict[str, str]] = [] - model_paths = await self._collect_model_paths(errors) + scanner_groups = await self._collect_model_paths(errors) - total = len(model_paths) + total = sum(len(paths) for _, paths in scanner_groups) processed = 0 models_moved = 0 moved = 0 skipped = 0 conflicts = 0 + # (file_path, final preview path at the destination layout), grouped + # by scanner so caches can be reconciled after the move. + preview_updates: List[Tuple[Any, List[Tuple[str, str]]]] = [] async def emit(status: str, **extra: Any) -> None: if progress_cb is None: @@ -235,27 +244,34 @@ class SidecarMigrationUseCase: await emit("started") - for model_path in model_paths: - processed += 1 - current = os.path.basename(model_path) - try: - result = await self._migrate_model( - model_path, - root=root, - to_centralized=to_centralized, - ) - moved += result["moved"] - conflicts += result["conflicts"] - if result["skipped"]: - skipped += 1 - if result["moved"]: - models_moved += 1 - except Exception as exc: - self._logger.error( - "Sidecar migration failed for %s: %s", model_path, exc, exc_info=True - ) - errors.append({"model": current, "error": str(exc)}) - await emit("processing", current=current) + for scanner, model_paths in scanner_groups: + updates: List[Tuple[str, str]] = [] + for model_path in model_paths: + processed += 1 + current = os.path.basename(model_path) + try: + result = await self._migrate_model( + model_path, + root=root, + to_centralized=to_centralized, + ) + moved += result["moved"] + conflicts += result["conflicts"] + if result["skipped"]: + skipped += 1 + else: + updates.append((model_path, result["preview_url"])) + if result["moved"]: + models_moved += 1 + except Exception as exc: + self._logger.error( + "Sidecar migration failed for %s: %s", model_path, exc, exc_info=True + ) + errors.append({"model": current, "error": str(exc)}) + await emit("processing", current=current) + preview_updates.append((scanner, updates)) + + await self._reconcile_scanner_caches(preview_updates) await emit("completed") @@ -278,10 +294,14 @@ class SidecarMigrationUseCase: *, root: str, to_centralized: bool, - ) -> Dict[str, int]: - """Migrate one model's sidecar + previews; return per-model counters.""" + ) -> Dict[str, Any]: + """Migrate one model's sidecar + previews; return per-model counters. - result = {"moved": 0, "conflicts": 0, "skipped": 0} + ``preview_url`` in the result is the model's final preview path in the + destination layout ("" when none), used to reconcile scanner caches. + """ + + result: Dict[str, Any] = {"moved": 0, "conflicts": 0, "skipped": 0, "preview_url": ""} model_path = os.path.abspath(model_path) if not os.path.exists(model_path): @@ -331,9 +351,60 @@ class SidecarMigrationUseCase: if sidecar_moved: await self._rewrite_sidecar_paths(sidecar_dst, model_path, moved_previews) + # Ground truth from the destination directory: covers conflict-keep + # and partial moves, not just the previews transferred in this run. + final_preview = find_preview_file(stem, dst_dir) + if final_preview: + result["preview_url"] = final_preview.replace(os.sep, "/") + return result - def _transfer(self, src: str, dst: str, result: Dict[str, int]) -> bool: + async def _reconcile_scanner_caches( + self, preview_updates: List[Tuple[Any, List[Tuple[str, str]]]] + ) -> None: + """Point scanner cache entries at the post-migration preview locations. + + Without this the list API keeps serving pre-migration ``preview_url`` + values whose files no longer exist; hitting one triggers the preview + route's stale-URL cleanup, which would wipe the reference for good. + A failing scanner is logged and skipped — the on-disk migration has + already succeeded, and a full rescan repairs the cache. + """ + + for scanner, updates in preview_updates: + if not updates: + continue + try: + cache = await scanner.get_cached_data() + changed = False + for file_path, preview_url in updates: + entry = next( + (item for item in cache.raw_data if item.get("file_path") == file_path), + None, + ) + if entry is None: + continue + if entry.get("preview_url", "") == preview_url: + continue + if hasattr(cache, "update_preview_url"): + await cache.update_preview_url( + file_path, + preview_url, + entry.get("preview_nsfw_level", 0), + ) + else: # pragma: no cover - minimal cache doubles + entry["preview_url"] = preview_url + changed = True + if changed and hasattr(scanner, "_persist_current_cache"): + await scanner._persist_current_cache() + except Exception as exc: + self._logger.error( + "Sidecar migration: failed to reconcile scanner cache: %s", + exc, + exc_info=True, + ) + + def _transfer(self, src: str, dst: str, result: Dict[str, Any]) -> bool: """Move ``src`` to ``dst`` with keep-newer conflict resolution. Returns True when the file was actually moved to the destination. On a diff --git a/tests/services/use_cases/test_sidecar_migration_use_case.py b/tests/services/use_cases/test_sidecar_migration_use_case.py index fca8c43f..4251329b 100644 --- a/tests/services/use_cases/test_sidecar_migration_use_case.py +++ b/tests/services/use_cases/test_sidecar_migration_use_case.py @@ -83,14 +83,28 @@ class _FakeCache: def __init__(self, raw_data: List[Dict[str, Any]]) -> None: self.raw_data = raw_data + async def update_preview_url( + self, file_path: str, preview_url: str, preview_nsfw_level: int + ) -> bool: + for item in self.raw_data: + if item["file_path"] == file_path: + item["preview_url"] = preview_url + item["preview_nsfw_level"] = preview_nsfw_level + return True + return False + class _FakeScanner: def __init__(self, raw_data: List[Dict[str, Any]]) -> None: self._cache = _FakeCache(raw_data) + self.persist_calls = 0 async def get_cached_data(self) -> _FakeCache: return self._cache + async def _persist_current_cache(self) -> None: + self.persist_calls += 1 + def _make_use_case(model_paths: List[str]) -> SidecarMigrationUseCase: scanner = _FakeScanner([{"file_path": path} for path in model_paths]) @@ -98,10 +112,12 @@ def _make_use_case(model_paths: List[str]) -> SidecarMigrationUseCase: async def scanner_factory() -> _FakeScanner: return scanner - return SidecarMigrationUseCase( + use_case = SidecarMigrationUseCase( scanner_factories=(("lora", scanner_factory),), settings_service=get_settings_manager(), ) + use_case._test_scanner = scanner # expose for cache-reconcile assertions + return use_case class _ProgressRecorder: @@ -158,6 +174,11 @@ async def test_migrate_to_centralized_moves_sidecar_and_previews( assert statuses[-1] == "completed" assert all(p["type"] == "sidecar_migration_progress" for p in recorder.payloads) + # Scanner cache was reconciled to the mirror preview and persisted. + entry = use_case._test_scanner._cache.raw_data[0] + assert entry["preview_url"] == _normalize(mirror / "model.preview.webp") + assert use_case._test_scanner.persist_calls == 1 + @pytest.mark.asyncio async def test_migrate_to_alongside_reverses_layout( @@ -188,6 +209,12 @@ async def test_migrate_to_alongside_reverses_layout( library_root / "sub" / "model.preview.webp" ) + entry = use_case._test_scanner._cache.raw_data[0] + assert entry["preview_url"] == _normalize( + library_root / "sub" / "model.preview.webp" + ) + assert use_case._test_scanner.persist_calls == 1 + @pytest.mark.asyncio async def test_migrate_conflict_keeps_newer_file( @@ -292,3 +319,71 @@ async def test_migrate_to_alongside_refuses_when_already_alongside( assert summary["success"] is False assert "already alongside" in summary["error"] assert summary["moved"] == 0 + + +def _make_use_case_with_entries(entries: List[Dict[str, Any]]) -> SidecarMigrationUseCase: + scanner = _FakeScanner(entries) + + async def scanner_factory() -> _FakeScanner: + return scanner + + use_case = SidecarMigrationUseCase( + scanner_factories=(("lora", scanner_factory),), + settings_service=get_settings_manager(), + ) + use_case._test_scanner = scanner + return use_case + + +@pytest.mark.asyncio +async def test_migrate_reconcile_clears_stale_preview_when_none_remains( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + model = _write_model(library_root, "model") + _write_sidecar(library_root, "model", model, preview_ext=None) + + stale_url = _normalize(library_root / "model.preview.webp") + entries = [ + {"file_path": str(model), "preview_url": stale_url, "preview_nsfw_level": 4} + ] + use_case = _make_use_case_with_entries(entries) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is True + entry = use_case._test_scanner._cache.raw_data[0] + # No preview exists in either layout: the stale reference is cleared. + assert entry["preview_url"] == "" + assert use_case._test_scanner.persist_calls == 1 + + +@pytest.mark.asyncio +async def test_migrate_reconcile_survives_per_model_errors( + library_root: Path, sidecar_root: Path, monkeypatch: pytest.MonkeyPatch +): + _set_mode("centralized") + model_ok = _write_model(library_root, "ok") + _write_sidecar(library_root, "ok", model_ok) + (library_root / "ok.preview.webp").write_bytes(b"preview") + model_bad = _write_model(library_root, "bad") + _write_sidecar(library_root, "bad", model_bad, preview_ext=None) + + use_case = _make_use_case([str(model_ok), str(model_bad)]) + original = use_case._migrate_model + + async def failing_migrate(model_path: str, **kwargs): + if os.path.basename(model_path) == "bad.safetensors": + raise RuntimeError("boom") + return await original(model_path, **kwargs) + + monkeypatch.setattr(use_case, "_migrate_model", failing_migrate) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is False + assert summary["error_count"] == 1 + # The healthy model's cache entry is still reconciled and persisted. + ok_entry = use_case._test_scanner._cache.raw_data[0] + assert ok_entry["preview_url"] == _normalize( + _mirror_dir(sidecar_root) / "ok.preview.webp" + ) + assert use_case._test_scanner.persist_calls == 1 From a6fca8612fcc526575eb0a381055186e626803dd Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 26 Sep 2026 12:24:17 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20injec?= =?UTF-8?q?tive=20mirror=20roots,=20root=20relocation,=20full=20preview=20?= =?UTF-8?q?coverage,=20EXDEV-safe=20rollback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review on #1124: - P1: mirror layout root component is now - (sha256 of the normalized root path), so two roots sharing a basename no longer map to the same mirror directory and overwrite each other's sidecars - P1: changing sidecar_storage_path while centralized no longer strands assets in the old root — new relocate_root migration direction moves the whole mirror tree, rewrites preview_url prefixes inside sidecars, reconciles scanner caches, and prunes the emptied old tree; the settings UI detects the path change and offers the relocation - P2: migration enumerates the same preview candidates as find_preview_file — case-insensitive variants (model.WEBP) and the legacy .example.0.jpeg suffix — instead of exact lowercase PREVIEW_EXTENSIONS only - P2: _rollback_model_staging restores staged files with the EXDEV-tolerant mover, so a failed undoable-delete staging no longer strands a cross-filesystem centralized sidecar copy Tests: same-basename root injectivity, mixed-case/example preview migration, relocate_root happy path + guards + route 400, frontend relocation prompt flow. Verified end-to-end in a sandboxed standalone server: uppercase/legacy previews migrate, root relocation moves the tree and the list API serves the new locations immediately without a rescan. --- docs/metadata-json-schema.md | 5 +- locales/de.json | 6 +- locales/en.json | 6 +- locales/es.json | 6 +- locales/fr.json | 6 +- locales/he.json | 6 +- locales/ja.json | 6 +- locales/ko.json | 6 +- locales/ru.json | 6 +- locales/zh-CN.json | 6 +- locales/zh-TW.json | 6 +- py/routes/handlers/misc_handlers.py | 11 +- py/services/pending_delete_service.py | 5 +- .../use_cases/sidecar_migration_use_case.py | 208 +++++++++++++++++- py/utils/sidecar_paths.py | 22 +- static/js/managers/SettingsManager.js | 51 ++++- .../components/modals/settings/library.html | 2 +- .../settingsManager.sidecarStorage.test.js | 56 +++++ tests/routes/test_misc_routes.py | 47 +++- .../test_centralized_sidecar_storage.py | 24 +- .../test_sidecar_migration_use_case.py | 129 ++++++++++- tests/utils/test_sidecar_paths.py | 43 +++- 22 files changed, 592 insertions(+), 71 deletions(-) diff --git a/docs/metadata-json-schema.md b/docs/metadata-json-schema.md index 00672a6a..cb98d24b 100644 --- a/docs/metadata-json-schema.md +++ b/docs/metadata-json-schema.md @@ -23,12 +23,13 @@ By default, `.metadata.json` sidecars and preview images live **alongside** thei In centralized mode, sidecars and previews mirror the library-relative directory structure: ``` -////.metadata.json +////.metadata.json ``` -- `` is the active library name, `` the basename of the model root containing the file, and `` the model's directory relative to that root. Each component is sanitized to filesystem-safe characters. +- `` is the active library name and `` the model's directory relative to the model root containing the file. `` combines the root's basename with a short hash of its full path so two roots sharing a basename (e.g. `/mnt/a/loras` and `/mnt/b/loras`) never collide. Each component is sanitized to filesystem-safe characters. - `.civitai.info` files always stay next to the model file, in both modes. - Changing the mode does **not** move existing files automatically — run the migration (`POST /api/lm/sidecars/migrate` with `{"direction": "to_centralized" | "to_alongside"}`, or the "Migrate Sidecars Now" button in settings). +- Changing `sidecar_storage_path` while centralized likewise needs a root relocation: `{"direction": "relocate_root", "old_root": ""}` moves the whole mirror tree to the new root (the settings UI offers this automatically). - All sidecar/preview path derivation goes through the helpers in `py/utils/sidecar_paths.py`; never construct paths inline. --- diff --git a/locales/de.json b/locales/de.json index ca6b8990..98ddd37f 100644 --- a/locales/de.json +++ b/locales/de.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "App-Proxy aktivieren", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Tags zu mehreren Modellen hinzufügen", diff --git a/locales/en.json b/locales/en.json index 40eff655..2a7b5af3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -809,7 +809,8 @@ "migrateFailed": "Sidecar migration failed: {message}", "migrationDeferred": "Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Enable App-level Proxy", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "Move sidecars to centralized storage?", "titleToAlongside": "Move sidecars back next to model files?", - "confirmButton": "Migrate Now" + "confirmButton": "Migrate Now", + "titleRelocateRoot": "Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Add Tags to Multiple Models", diff --git a/locales/es.json b/locales/es.json index 54654810..b4e0356e 100644 --- a/locales/es.json +++ b/locales/es.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Habilitar proxy a nivel de aplicación", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Añadir etiquetas a múltiples modelos", diff --git a/locales/fr.json b/locales/fr.json index 792745be..2a4008db 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Activer le proxy au niveau de l'application", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Ajouter des tags à plusieurs modèles", diff --git a/locales/he.json b/locales/he.json index 672b8464..51afcf6b 100644 --- a/locales/he.json +++ b/locales/he.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "הפעל פרוקסי ברמת האפליקציה", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "הוסף תגיות למספר מודלים", diff --git a/locales/ja.json b/locales/ja.json index 6de03706..61f7982f 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "アプリレベルのプロキシを有効化", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "複数モデルにタグを追加", diff --git a/locales/ko.json b/locales/ko.json index cbfb665b..af342656 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "앱 수준 프록시 활성화", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "여러 모델에 태그 추가", diff --git a/locales/ru.json b/locales/ru.json index 9e26faf1..0613fe82 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Включить прокси на уровне приложения", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Добавить теги к нескольким моделям", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 53728dd3..106fa7d6 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "启用应用级代理", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "批量添加标签", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 2770a105..f425fa3a 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "啟用應用程式代理", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "新增標籤到多個模型", diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index a365daf3..27616aa4 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -4141,7 +4141,7 @@ class NodeRegistryHandler: class SidecarMigrationHandler: """Migrate sidecar metadata and previews between storage layouts.""" - _VALID_DIRECTIONS = ("to_centralized", "to_alongside") + _VALID_DIRECTIONS = ("to_centralized", "to_alongside", "relocate_root") def __init__( self, @@ -4168,12 +4168,18 @@ class SidecarMigrationHandler: return web.json_response( { "success": False, - "error": "direction must be 'to_centralized' or 'to_alongside'", + "error": "direction must be 'to_centralized', 'to_alongside' or 'relocate_root'", }, status=400, ) force = params.get("force") in (True, 1, "true", "1") + old_root = str(params.get("old_root") or "").strip() + if direction == "relocate_root" and not old_root: + return web.json_response( + {"success": False, "error": "old_root is required for relocate_root"}, + status=400, + ) use_case = self._use_case_factory() progress_cb = self._progress_callback_factory() @@ -4181,6 +4187,7 @@ class SidecarMigrationHandler: direction=direction, progress_cb=progress_cb, force=force, + old_root=old_root, ) status = 200 if result.get("success") else 400 return web.json_response(result, status=status) diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index c57cf6e4..8d579715 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -777,7 +777,10 @@ class PendingDeleteService: if not os.path.exists(staged_path): continue try: - os.rename(staged_path, original_path) + # EXDEV-tolerant: centralized sidecars may have been copied + # across filesystems into staging, so plain os.rename would + # fail here and strand the only copy. + self._restore_file(staged_path, original_path) except OSError as exc: # pragma: no cover - best-effort rollback logger.warning( "Failed to roll back staged file %s -> %s: %s", diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py index 4bb3756b..04e78994 100644 --- a/py/services/use_cases/sidecar_migration_use_case.py +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -70,6 +70,27 @@ ScannerFactory = Callable[[], Awaitable[Any]] DIRECTION_TO_CENTRALIZED = "to_centralized" DIRECTION_TO_ALONGSIDE = "to_alongside" +DIRECTION_RELOCATE_ROOT = "relocate_root" + +# Same candidate set find_preview_file recognizes: every PREVIEW_EXTENSIONS +# suffix plus the legacy ".example.0.jpeg" (issue #225). +_PREVIEW_CANDIDATE_EXTENSIONS = tuple(PREVIEW_EXTENSIONS) + (".example.0.jpeg",) + + +def _enumerate_preview_names(directory: str, stem: str) -> List[str]: + """Return preview filenames for ``stem`` present in ``directory``. + + Case-insensitive full-name match against the preview candidate set, so + files like ``model.WEBP`` or ``model.Png`` placed by external tools are + migrated along with the exact-case variants. + """ + + targets = {f"{stem.lower()}{ext}" for ext in _PREVIEW_CANDIDATE_EXTENSIONS} + try: + entries = os.listdir(directory) + except OSError: + return [] + return [entry for entry in entries if entry.lower() in targets] class SidecarMigrationUseCase: @@ -136,6 +157,179 @@ class SidecarMigrationUseCase: progress_cb=progress_cb, ) + async def migrate_root( + self, + old_root: str, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Relocate the whole mirror tree from a previous root to the configured one. + + Used after ``sidecar_storage_path`` changes while centralized storage + is active: without it, every asset under the old root would silently + disappear from the application. Moves every file keeping the + root-relative structure, rewrites the ``preview_url`` prefix inside + moved sidecars, reconciles scanner caches, and prunes the emptied old + tree. Keep-newer conflict resolution matches :meth:`_transfer`. + """ + + if not force and get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "sidecar storage is not centralized; pass force=true to relocate anyway", + ) + new_root = get_configured_sidecar_root() + if not new_root: + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "cannot resolve the centralized sidecar root", + ) + old = ( + os.path.abspath(os.path.expanduser(old_root.strip())) + if isinstance(old_root, str) and old_root.strip() + else "" + ) + if not old: + return self._refusal(DIRECTION_RELOCATE_ROOT, "old_root is required") + if os.path.normpath(old) == os.path.normpath(new_root): + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "old_root matches the configured sidecar root", + ) + + files: List[Tuple[str, str]] = [] + if os.path.isdir(old): + for dirpath, _dirnames, filenames in os.walk(old): + rel = os.path.relpath(dirpath, old) + target_dir = new_root if rel == os.curdir else os.path.join(new_root, rel) + for filename in filenames: + files.append( + (os.path.join(dirpath, filename), os.path.join(target_dir, filename)) + ) + + errors: List[Dict[str, str]] = [] + counters: Dict[str, Any] = {"moved": 0, "conflicts": 0} + moved_sidecars: List[str] = [] + + async def emit(status: str, **extra: Any) -> None: + if progress_cb is None: + return + payload: Dict[str, Any] = { + "type": "sidecar_migration_progress", + "status": status, + "direction": DIRECTION_RELOCATE_ROOT, + "total": len(files), + "processed": extra.pop("processed", 0), + "moved": counters["moved"], + "skipped": 0, + "conflicts": counters["conflicts"], + "errors": len(errors), + } + payload.update(extra) + await progress_cb.on_progress(payload) + + await emit("started") + + for index, (src, dst) in enumerate(files, start=1): + try: + if self._transfer(src, dst, counters) and src.endswith(METADATA_SUFFIX): + moved_sidecars.append(dst) + except Exception as exc: + self._logger.error( + "Sidecar root relocation failed for %s: %s", src, exc, exc_info=True + ) + errors.append({"model": os.path.basename(src), "error": str(exc)}) + await emit("processing", processed=index, current=os.path.basename(src)) + + old_prefix = old.replace(os.sep, "/").rstrip("/") + "/" + new_prefix = new_root.replace(os.sep, "/").rstrip("/") + "/" + for sidecar in moved_sidecars: + self._rewrite_root_prefix(sidecar, old_prefix, new_prefix) + await self._reconcile_root_prefix(old_prefix, new_prefix) + + # Prune the emptied old tree, best-effort. + if os.path.isdir(old): + for dirpath, dirnames, filenames in os.walk(old, topdown=False): + if filenames: + continue + for dirname in dirnames: + try: + os.rmdir(os.path.join(dirpath, dirname)) + except OSError: + pass + try: + os.rmdir(dirpath) + except OSError: + pass + + await emit("completed") + + return { + "success": not errors, + "direction": DIRECTION_RELOCATE_ROOT, + "models_total": len(files), + "models_processed": len(files), + "models_moved": 0, + "moved": counters["moved"], + "skipped": 0, + "conflicts": counters["conflicts"], + "errors": errors, + "error_count": len(errors), + } + + def _rewrite_root_prefix( + self, sidecar_path: str, old_prefix: str, new_prefix: str + ) -> None: + """Repoint preview_url inside a relocated sidecar from old to new root.""" + + try: + with open(sidecar_path, "r", encoding="utf-8") as handle: + metadata = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + self._logger.warning( + "Sidecar root relocation: cannot read %s: %s", sidecar_path, exc + ) + return + + preview_url = metadata.get("preview_url") + if not isinstance(preview_url, str) or not preview_url.startswith(old_prefix): + return + metadata["preview_url"] = new_prefix + preview_url[len(old_prefix):] + try: + with open(sidecar_path, "w", encoding="utf-8") as handle: + json.dump(metadata, handle, ensure_ascii=False, indent=2) + except OSError as exc: + self._logger.warning( + "Sidecar root relocation: cannot rewrite %s: %s", sidecar_path, exc + ) + + async def _reconcile_root_prefix(self, old_prefix: str, new_prefix: str) -> None: + """Rewrite old-root preview URLs in every scanner cache after relocation.""" + + for model_type, factory in self._active_scanner_factories(): + try: + scanner = await factory() + cache = await scanner.get_cached_data() + changed = False + for item in cache.raw_data: + preview_url = item.get("preview_url") + if ( + isinstance(preview_url, str) + and preview_url.startswith(old_prefix) + ): + item["preview_url"] = new_prefix + preview_url[len(old_prefix):] + changed = True + if changed and hasattr(scanner, "_persist_current_cache"): + await scanner._persist_current_cache() + except Exception as exc: + self._logger.error( + "Sidecar root relocation: failed to reconcile %s cache: %s", + model_type, + exc, + exc_info=True, + ) + @staticmethod def _refusal(direction: str, message: str) -> Dict[str, Any]: return { @@ -334,11 +528,9 @@ class SidecarMigrationUseCase: sidecar_name = stem + METADATA_SUFFIX moved_previews: List[str] = [] - for ext in PREVIEW_EXTENSIONS: - src = os.path.join(src_dir, stem + ext) - if not os.path.exists(src): - continue - dst = os.path.join(dst_dir, stem + ext) + for preview_name in _enumerate_preview_names(src_dir, stem): + src = os.path.join(src_dir, preview_name) + dst = os.path.join(dst_dir, preview_name) if self._transfer(src, dst, result): moved_previews.append(dst) @@ -465,6 +657,7 @@ class SidecarMigrationUseCase: direction: str, progress_cb: Optional[SidecarMigrationProgressReporter] = None, force: bool = False, + old_root: Optional[str] = None, ) -> Dict[str, Any]: """Wrapper providing progress notification on unexpected failures.""" @@ -473,8 +666,11 @@ class SidecarMigrationUseCase: return await self.migrate_to_centralized(progress_cb, force=force) if direction == DIRECTION_TO_ALONGSIDE: return await self.migrate_to_alongside(progress_cb, force=force) + if direction == DIRECTION_RELOCATE_ROOT: + return await self.migrate_root(old_root or "", progress_cb, force=force) raise ValueError( - f"direction must be {DIRECTION_TO_CENTRALIZED!r} or {DIRECTION_TO_ALONGSIDE!r}" + f"direction must be {DIRECTION_TO_CENTRALIZED!r}, " + f"{DIRECTION_TO_ALONGSIDE!r} or {DIRECTION_RELOCATE_ROOT!r}" ) except Exception as exc: if progress_cb is not None: diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py index d43f4b54..ad03f6de 100644 --- a/py/utils/sidecar_paths.py +++ b/py/utils/sidecar_paths.py @@ -12,7 +12,7 @@ setting: - ``centralized``: sidecars and previews live under a configurable root (``sidecar_storage_path`` setting, default ``/sidecars``), mirroring the library-relative directory structure: - ``////.metadata.json``. + ``////.metadata.json``. All helpers are pure path computations: no directory scans and no file I/O on the hot path. Settings lookups go through ``SettingsManager.get`` (a dict @@ -21,6 +21,7 @@ read); config roots come from the already-initialized ``config`` singleton. from __future__ import annotations +import hashlib import logging import os import re @@ -145,10 +146,25 @@ def _normalize_for_match(path: str) -> str: return os.path.normpath(os.path.abspath(path)) +def root_mirror_component(root_path: str) -> str: + """Return the mirror path component identifying a model root. + + ``-`` where the hash is a short digest of the + normalized absolute root path. Two roots sharing a basename (e.g. + ``/mnt/a/loras`` and ``/mnt/b/loras``) would otherwise map to the same + mirror directory and overwrite each other's sidecars. + """ + + normalized = _normalize_for_match(root_path) + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8] + return f"{sanitize_path_component(os.path.basename(normalized))}-{digest}" + + def resolve_centralized_dir(model_path: str) -> Optional[str]: """Return the centralized mirror directory for ``model_path``. - The mirror layout is ``///`` + The mirror layout is + ``///`` where ``rel_dir`` is the model's directory relative to the model root that contains it. The longest matching root wins so nested roots resolve to the most specific mirror. Returns ``None`` when centralized storage is inactive @@ -201,7 +217,7 @@ def resolve_centralized_dir_for_dir( library = "default" rel_dir = os.path.relpath(normalized_dir, best_root) - parts = [root, sanitize_path_component(library), sanitize_path_component(os.path.basename(best_root))] + parts = [root, sanitize_path_component(library), root_mirror_component(best_root)] if rel_dir and rel_dir != os.curdir: parts.extend(sanitize_path_component(part) for part in rel_dir.split(os.sep) if part not in ("", os.curdir)) return os.path.join(*parts) diff --git a/static/js/managers/SettingsManager.js b/static/js/managers/SettingsManager.js index eb60c202..55e373a9 100644 --- a/static/js/managers/SettingsManager.js +++ b/static/js/managers/SettingsManager.js @@ -1284,7 +1284,7 @@ export class SettingsManager { onAfterSelect: () => this.saveInputSetting('exampleImagesLocalRoot', 'example_images_local_root'), }); this.attachPathField('sidecarStoragePath', { - onAfterSelect: () => this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'), + onAfterSelect: () => this.handleSidecarStoragePathChange(), }); } @@ -3396,6 +3396,8 @@ export class SettingsManager { } // Baseline used to detect a mode change in handleSidecarStorageModeChange this._loadedSidecarStorageMode = currentMode; + // Baseline used to detect a root change in handleSidecarStoragePathChange + this._loadedSidecarStoragePath = state.global.settings.sidecar_storage_path || ''; const pathInput = document.getElementById('sidecarStoragePath'); if (pathInput) { @@ -3438,6 +3440,30 @@ export class SettingsManager { } } + // Path change while centralized storage is active: the assets under the + // previous root do not move by themselves, so offer a root relocation. + async handleSidecarStoragePathChange() { + const pathInput = document.getElementById('sidecarStoragePath'); + if (!pathInput) return; + + const previousPath = this._loadedSidecarStoragePath || ''; + + await this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'); + + const newPath = pathInput.value.trim(); + this._loadedSidecarStoragePath = newPath; + + const centralized = state.global.settings.sidecar_storage_mode === 'centralized'; + if (centralized && previousPath && previousPath !== newPath) { + const confirmed = await this.confirmSidecarMigration('relocate_root'); + if (confirmed) { + await this.migrateSidecars('relocate_root', { old_root: previousPath }); + } else { + showToast('settings.sidecarStorage.migrationDeferred', {}, 'info'); + } + } + } + // Entry point for the "Migrate Sidecars Now" button: the direction follows // the currently saved storage mode. async confirmAndMigrateSidecars() { @@ -3457,19 +3483,24 @@ export class SettingsManager { } const isToCentralized = direction === 'to_centralized'; + const isRelocate = direction === 'relocate_root'; const titleElement = modalElement.querySelector('[data-role="title"]'); if (titleElement) { - titleElement.textContent = isToCentralized - ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') - : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); + titleElement.textContent = isRelocate + ? translate('modals.sidecarMigrationConfirm.titleRelocateRoot', {}, 'Move sidecars to the new storage directory?') + : isToCentralized + ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') + : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); } const messageElement = modalElement.querySelector('[data-role="message"]'); if (messageElement) { - messageElement.textContent = isToCentralized - ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') - : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); + messageElement.textContent = isRelocate + ? translate('settings.sidecarStorage.confirmRelocateRoot', {}, 'The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?') + : isToCentralized + ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') + : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); } const confirmButton = modalElement.querySelector('[data-action="confirm-sidecar-migration"]'); @@ -3528,7 +3559,7 @@ export class SettingsManager { }); } - async migrateSidecars(direction) { + async migrateSidecars(direction, extraBody = {}) { const migrateBtn = document.getElementById('migrateSidecarsBtn'); try { if (migrateBtn) { @@ -3543,10 +3574,10 @@ export class SettingsManager { const response = await fetch('/api/lm/sidecars/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - // The new mode is already saved by the time migration runs, + // The new mode/path is already saved by the time migration runs, // so the backend guard requires force=true to confirm the // "switch first, then migrate" flow. - body: JSON.stringify({ direction, force: true }), + body: JSON.stringify({ direction, force: true, ...extraBody }), }); const data = await response.json(); diff --git a/templates/components/modals/settings/library.html b/templates/components/modals/settings/library.html index 89012f89..aa5424ef 100644 --- a/templates/components/modals/settings/library.html +++ b/templates/components/modals/settings/library.html @@ -354,7 +354,7 @@
diff --git a/tests/frontend/managers/settingsManager.sidecarStorage.test.js b/tests/frontend/managers/settingsManager.sidecarStorage.test.js index ae376076..3f20d3f7 100644 --- a/tests/frontend/managers/settingsManager.sidecarStorage.test.js +++ b/tests/frontend/managers/settingsManager.sidecarStorage.test.js @@ -281,4 +281,60 @@ describe('SettingsManager sidecar storage', () => { expect(migrateBtn.disabled).toBe(false); }); }); + + describe('handleSidecarStoragePathChange', () => { + it('offers root relocation when the path changes in centralized mode', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStoragePathChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await changePromise; + + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + body: JSON.stringify({ direction: 'relocate_root', force: true, old_root: '/old/root' }), + })); + expect(manager._loadedSidecarStoragePath).toBe('/new/root'); + }); + + it('does not prompt when the path changes in alongside mode', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + await manager.handleSidecarStoragePathChange(); + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + }); + + it('shows a deferred notice when relocation is cancelled', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStoragePathChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="cancel-sidecar-migration"]').click(); + await changePromise; + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrationDeferred', {}, 'info'); + }); + }); }); diff --git a/tests/routes/test_misc_routes.py b/tests/routes/test_misc_routes.py index 8ea65248..e693be48 100644 --- a/tests/routes/test_misc_routes.py +++ b/tests/routes/test_misc_routes.py @@ -2565,8 +2565,8 @@ class DummySidecarMigrationUseCase: self.result = result self.calls = [] - async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False): - self.calls.append({"direction": direction, "force": force}) + async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False, old_root=None): + self.calls.append({"direction": direction, "force": force, "old_root": old_root}) return self.result @@ -2592,7 +2592,7 @@ async def test_sidecar_migration_handler_runs_to_centralized(): assert response.status == 200 assert payload["success"] is True assert payload["moved"] == 3 - assert use_case.calls == [{"direction": "to_centralized", "force": True}] + assert use_case.calls == [{"direction": "to_centralized", "force": True, "old_root": ""}] @pytest.mark.asyncio @@ -2624,7 +2624,7 @@ async def test_sidecar_migration_handler_accepts_get_query_params(): assert response.status == 200 assert payload["success"] is True - assert use_case.calls == [{"direction": "to_alongside", "force": True}] + assert use_case.calls == [{"direction": "to_alongside", "force": True, "old_root": ""}] @pytest.mark.asyncio @@ -2640,4 +2640,41 @@ async def test_sidecar_migration_handler_guard_refusal_is_400(): assert response.status == 400 assert payload["success"] is False assert "already centralized" in payload["error"] - assert use_case.calls == [{"direction": "to_centralized", "force": False}] + assert use_case.calls == [{"direction": "to_centralized", "force": False, "old_root": ""}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_relocate_root_passes_old_root(): + result = {"success": True, "direction": "relocate_root", "moved": 5} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest( # pyright: ignore[reportArgumentType] + json_data={ + "direction": "relocate_root", + "old_root": "/old/sidecars", + "force": True, + } + ) + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert use_case.calls == [ + {"direction": "relocate_root", "force": True, "old_root": "/old/sidecars"} + ] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_relocate_root_requires_old_root(): + handler, use_case = _sidecar_migration_handler({"success": True}) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "relocate_root"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert "old_root" in payload["error"] + assert use_case.calls == [] diff --git a/tests/services/test_centralized_sidecar_storage.py b/tests/services/test_centralized_sidecar_storage.py index 2d9164ab..2c40d26e 100644 --- a/tests/services/test_centralized_sidecar_storage.py +++ b/tests/services/test_centralized_sidecar_storage.py @@ -19,6 +19,7 @@ from py.services.model_lifecycle_service import ( from py.services.pending_delete_service import PendingDeleteService from py.services.settings_manager import get_settings_manager from py.utils.metadata_manager import MetadataManager +from py.utils.sidecar_paths import root_mirror_component def _normalize(path) -> str: @@ -57,11 +58,12 @@ def centralized(library_root: Path, tmp_path: Path) -> Path: return sidecar_root -def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: +def _mirror_dir(library_root: Path, sidecar_root: Path, *rel: str) -> Path: """Expected mirror directory for a library-relative path.""" library = get_settings_manager().get_active_library_name() - return sidecar_root.joinpath(library, "checkpoints", *rel) + component = root_mirror_component(str(library_root)) + return sidecar_root.joinpath(library, component, *rel) def _write_sidecar( @@ -93,7 +95,7 @@ async def test_delete_model_artifacts_centralized( ): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") deleted = await delete_model_artifacts(str(library_root), "model") @@ -131,7 +133,7 @@ def test_enumerate_model_artifacts_centralized( ): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.png") service = PendingDeleteService.__new__(PendingDeleteService) @@ -170,7 +172,7 @@ async def _json_metadata_loader(path: str) -> Dict[str, object]: async def test_rename_model_centralized(library_root: Path, centralized: Path): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") service = ModelLifecycleService( @@ -213,7 +215,7 @@ async def test_move_model_centralized( model.write_bytes(b"weights") target_dir = library_root / "new" - old_mirror = _mirror_dir(centralized, "old") + old_mirror = _mirror_dir(library_root, centralized, "old") _write_sidecar( old_mirror, "model", file_path=model, preview_name="model.preview.webp" ) @@ -230,7 +232,7 @@ async def test_move_model_centralized( assert moved_model.exists() assert not model.exists() - new_mirror = _mirror_dir(centralized, "new") + new_mirror = _mirror_dir(library_root, centralized, "new") assert (new_mirror / "model.metadata.json").exists() assert (new_mirror / "model.preview.webp").exists() assert not (old_mirror / "model.metadata.json").exists() @@ -271,7 +273,7 @@ async def test_rename_known_folder_centralized( model.write_bytes(b"weights") old_model_path = old_dir / "model.safetensors" - old_mirror = _mirror_dir(centralized, "oldfolder") + old_mirror = _mirror_dir(library_root, centralized, "oldfolder") _write_sidecar( old_mirror, "model", file_path=old_model_path, preview_name="model.preview.webp" ) @@ -295,7 +297,7 @@ async def test_rename_known_folder_centralized( assert changed is True - new_mirror = _mirror_dir(centralized, "newfolder") + new_mirror = _mirror_dir(library_root, centralized, "newfolder") assert (new_mirror / "model.metadata.json").exists() assert (new_mirror / "model.preview.webp").exists() assert not old_mirror.exists() @@ -315,7 +317,7 @@ async def test_rename_known_folder_centralized( async def test_pending_models_mirror_walk(library_root: Path, centralized: Path): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar( mirror, "model", @@ -346,7 +348,7 @@ async def test_pending_models_mirror_walk_uses_stem_fallback( model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar( mirror, "model", diff --git a/tests/services/use_cases/test_sidecar_migration_use_case.py b/tests/services/use_cases/test_sidecar_migration_use_case.py index 4251329b..7db41409 100644 --- a/tests/services/use_cases/test_sidecar_migration_use_case.py +++ b/tests/services/use_cases/test_sidecar_migration_use_case.py @@ -12,6 +12,7 @@ import pytest from py.config import config from py.services.settings_manager import get_settings_manager from py.services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase +from py.utils.sidecar_paths import root_mirror_component def _normalize(path) -> str: @@ -52,11 +53,12 @@ def _set_mode(mode: str) -> None: get_settings_manager().set("sidecar_storage_mode", mode) -def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: +def _mirror_dir(library_root: Path, sidecar_root: Path, *rel: str) -> Path: """Expected mirror directory for a library-relative path.""" library = get_settings_manager().get_active_library_name() - return sidecar_root.joinpath(library, "loras", *rel) + component = root_mirror_component(str(library_root)) + return sidecar_root.joinpath(library, component, *rel) def _write_model(directory: Path, stem: str) -> Path: @@ -152,7 +154,7 @@ async def test_migrate_to_centralized_moves_sidecar_and_previews( assert summary["conflicts"] == 0 assert summary["errors"] == [] - mirror = _mirror_dir(sidecar_root, "sub") + mirror = _mirror_dir(library_root, sidecar_root, "sub") assert not sidecar.exists() assert not preview.exists() assert not extra_preview.exists() @@ -186,7 +188,7 @@ async def test_migrate_to_alongside_reverses_layout( ): _set_mode("alongside") model = _write_model(library_root / "sub", "model") - mirror = _mirror_dir(sidecar_root, "sub") + mirror = _mirror_dir(library_root, sidecar_root, "sub") sidecar = _write_sidecar(mirror, "model", model) preview = mirror / "model.preview.webp" preview.write_bytes(b"preview") @@ -221,7 +223,7 @@ async def test_migrate_conflict_keeps_newer_file( library_root: Path, sidecar_root: Path ): _set_mode("centralized") - mirror = _mirror_dir(sidecar_root) + mirror = _mirror_dir(library_root, sidecar_root) # Model A: destination (mirror) sidecar is newer -> destination wins. model_a = _write_model(library_root, "model_a") @@ -384,6 +386,121 @@ async def test_migrate_reconcile_survives_per_model_errors( # The healthy model's cache entry is still reconciled and persisted. ok_entry = use_case._test_scanner._cache.raw_data[0] assert ok_entry["preview_url"] == _normalize( - _mirror_dir(sidecar_root) / "ok.preview.webp" + _mirror_dir(library_root, sidecar_root) / "ok.preview.webp" ) assert use_case._test_scanner.persist_calls == 1 + + +@pytest.mark.asyncio +async def test_migrate_covers_mixed_case_and_example_previews( + library_root: Path, sidecar_root: Path +): + """Previews like model.WEBP / model.example.0.jpeg migrate too (#225 compat).""" + + _set_mode("centralized") + model = _write_model(library_root, "model") + _write_sidecar(library_root, "model", model, preview_ext=".preview.WEBP") + (library_root / "model.preview.WEBP").write_bytes(b"preview") + (library_root / "model.example.0.jpeg").write_bytes(b"example") + + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is True + assert summary["moved"] == 3 # sidecar + 2 previews + + mirror = _mirror_dir(library_root, sidecar_root) + assert (mirror / "model.preview.WEBP").exists() + assert (mirror / "model.example.0.jpeg").exists() + assert not (library_root / "model.preview.WEBP").exists() + assert not (library_root / "model.example.0.jpeg").exists() + + metadata = json.loads((mirror / "model.metadata.json").read_text(encoding="utf-8")) + assert metadata["preview_url"] == _normalize(mirror / "model.preview.WEBP") + + +@pytest.mark.asyncio +async def test_migrate_root_relocates_tree_and_reconciles( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + library = get_settings_manager().get_active_library_name() + component = root_mirror_component(str(library_root)) + + # Assets under the OLD root, mirroring the layout. + old_root = tmp_path / "old_sidecars" + old_mirror = old_root / library / component / "sub" + old_mirror.mkdir(parents=True) + model = _write_model(library_root / "sub", "model") + payload = { + "file_name": "model", + "file_path": _normalize(model), + "preview_url": _normalize(old_mirror / "model.preview.png"), + } + (old_mirror / "model.metadata.json").write_text(json.dumps(payload), encoding="utf-8") + (old_mirror / "model.preview.png").write_bytes(b"preview") + + entries = [ + { + "file_path": str(model), + "preview_url": _normalize(old_mirror / "model.preview.png"), + "preview_nsfw_level": 2, + } + ] + use_case = _make_use_case_with_entries(entries) + summary = await use_case.migrate_root(str(old_root), force=True) + + assert summary["success"] is True + assert summary["moved"] == 2 + + new_mirror = sidecar_root / library / component / "sub" + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.png").exists() + + # Sidecar preview_url rewritten onto the new root. + migrated = json.loads((new_mirror / "model.metadata.json").read_text(encoding="utf-8")) + assert migrated["preview_url"] == _normalize(new_mirror / "model.preview.png") + # Model path fields untouched — model files never move. + assert migrated["file_path"] == _normalize(model) + + # Scanner cache preview URLs repointed and persisted. + entry = use_case._test_scanner._cache.raw_data[0] + assert entry["preview_url"] == _normalize(new_mirror / "model.preview.png") + assert use_case._test_scanner.persist_calls == 1 + + # Emptied old tree pruned. + assert not old_root.exists() + + +@pytest.mark.asyncio +async def test_migrate_root_guards( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_root("") + assert summary["success"] is False + assert "old_root is required" in summary["error"] + + summary = await use_case.migrate_root(str(sidecar_root)) + assert summary["success"] is False + assert "matches the configured" in summary["error"] + + _set_mode("alongside") + summary = await use_case.migrate_root(str(tmp_path / "old_sidecars")) + assert summary["success"] is False + assert "not centralized" in summary["error"] + + +@pytest.mark.asyncio +async def test_migrate_root_missing_old_tree_is_noop( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_root(str(tmp_path / "nonexistent"), force=True) + + assert summary["success"] is True + assert summary["moved"] == 0 diff --git a/tests/utils/test_sidecar_paths.py b/tests/utils/test_sidecar_paths.py index f09966d9..a5a4f6dd 100644 --- a/tests/utils/test_sidecar_paths.py +++ b/tests/utils/test_sidecar_paths.py @@ -22,6 +22,7 @@ from py.utils.sidecar_paths import ( resolve_centralized_dir, resolve_centralized_dir_for_dir, resolve_metadata_path, + root_mirror_component, sanitize_path_component, ) @@ -117,16 +118,48 @@ class TestCentralizedMode: def test_mirror_layout(self, model_roots: dict, centralized: Path): model = model_roots["loras"] / "styles" / "anime" / "model.safetensors" library = get_settings_manager().get_active_library_name() + root_component = root_mirror_component(str(model_roots["loras"])) metadata_path = get_metadata_path(str(model)) expected = os.path.join( - str(centralized), library, "loras", "styles", "anime", "model" + METADATA_SUFFIX + str(centralized), library, root_component, "styles", "anime", "model" + METADATA_SUFFIX ) assert metadata_path == expected assert get_preview_dir(str(model)) == os.path.dirname(expected) assert is_centralized() + def test_same_basename_roots_get_distinct_mirrors( + self, model_roots: dict, centralized: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + from py.config import config + + # Two roots sharing the basename "loras" must not share a mirror dir. + other_parent = tmp_path / "elsewhere" + other_root = other_parent / "loras" + other_root.mkdir(parents=True) + monkeypatch.setattr( + config, + "loras_roots", + [str(model_roots["loras"]), str(other_root)], + raising=False, + ) + + model_a = model_roots["loras"] / "model.safetensors" + model_b = other_root / "model.safetensors" + + dir_a = resolve_centralized_dir(str(model_a)) + dir_b = resolve_centralized_dir(str(model_b)) + assert dir_a is not None and dir_b is not None + assert dir_a != dir_b + assert root_mirror_component(str(model_roots["loras"])) != root_mirror_component( + str(other_root) + ) + # Same root always maps to the same component (stable hash). + assert root_mirror_component(str(model_roots["loras"])) == root_mirror_component( + str(model_roots["loras"]) + os.sep + ) + def test_longest_root_wins(self, model_roots: dict, centralized: Path, monkeypatch: pytest.MonkeyPatch): from py.config import config @@ -142,7 +175,7 @@ class TestCentralizedMode: model = nested / "model.safetensors" assert get_metadata_path(str(model)) == os.path.join( - str(centralized), library, "nested", "model" + METADATA_SUFFIX + str(centralized), library, root_mirror_component(str(nested)), "model" + METADATA_SUFFIX ) def test_outside_roots_falls_back_to_alongside( @@ -174,7 +207,7 @@ class TestCentralizedMode: library = get_settings_manager().get_active_library_name() assert resolve_centralized_dir_for_dir(str(model_roots["loras"])) == os.path.join( - str(centralized), library, "loras" + str(centralized), library, root_mirror_component(str(model_roots["loras"])) ) def test_empty_path_uses_default_sidecar_root(self, model_roots: dict, tmp_path: Path): @@ -223,7 +256,9 @@ class TestModeIndependentResolution: assert resolve_centralized_dir_for_dir(str(model_dir)) is None assert resolve_centralized_dir_for_dir( str(model_dir), sidecar_root=str(sidecar_root) - ) == os.path.join(str(sidecar_root), library, "loras", "sub") + ) == os.path.join( + str(sidecar_root), library, root_mirror_component(str(model_roots["loras"])), "sub" + ) class TestSettingsValidation: