diff --git a/docs/metadata-json-schema.md b/docs/metadata-json-schema.md index 00672a6a..cb98d24b 100644 --- a/docs/metadata-json-schema.md +++ b/docs/metadata-json-schema.md @@ -23,12 +23,13 @@ By default, `.metadata.json` sidecars and preview images live **alongside** thei In centralized mode, sidecars and previews mirror the library-relative directory structure: ``` -////.metadata.json +////.metadata.json ``` -- `` is the active library name, `` the basename of the model root containing the file, and `` the model's directory relative to that root. Each component is sanitized to filesystem-safe characters. +- `` is the active library name and `` the model's directory relative to the model root containing the file. `` combines the root's basename with a short hash of its full path so two roots sharing a basename (e.g. `/mnt/a/loras` and `/mnt/b/loras`) never collide. Each component is sanitized to filesystem-safe characters. - `.civitai.info` files always stay next to the model file, in both modes. - Changing the mode does **not** move existing files automatically — run the migration (`POST /api/lm/sidecars/migrate` with `{"direction": "to_centralized" | "to_alongside"}`, or the "Migrate Sidecars Now" button in settings). +- Changing `sidecar_storage_path` while centralized likewise needs a root relocation: `{"direction": "relocate_root", "old_root": ""}` moves the whole mirror tree to the new root (the settings UI offers this automatically). - All sidecar/preview path derivation goes through the helpers in `py/utils/sidecar_paths.py`; never construct paths inline. --- diff --git a/locales/de.json b/locales/de.json index ca6b8990..98ddd37f 100644 --- a/locales/de.json +++ b/locales/de.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "App-Proxy aktivieren", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Tags zu mehreren Modellen hinzufügen", diff --git a/locales/en.json b/locales/en.json index 40eff655..2a7b5af3 100644 --- a/locales/en.json +++ b/locales/en.json @@ -809,7 +809,8 @@ "migrateFailed": "Sidecar migration failed: {message}", "migrationDeferred": "Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Enable App-level Proxy", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "Move sidecars to centralized storage?", "titleToAlongside": "Move sidecars back next to model files?", - "confirmButton": "Migrate Now" + "confirmButton": "Migrate Now", + "titleRelocateRoot": "Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Add Tags to Multiple Models", diff --git a/locales/es.json b/locales/es.json index 54654810..b4e0356e 100644 --- a/locales/es.json +++ b/locales/es.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Habilitar proxy a nivel de aplicación", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Añadir etiquetas a múltiples modelos", diff --git a/locales/fr.json b/locales/fr.json index 792745be..2a4008db 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Activer le proxy au niveau de l'application", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Ajouter des tags à plusieurs modèles", diff --git a/locales/he.json b/locales/he.json index 672b8464..51afcf6b 100644 --- a/locales/he.json +++ b/locales/he.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "הפעל פרוקסי ברמת האפליקציה", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "הוסף תגיות למספר מודלים", diff --git a/locales/ja.json b/locales/ja.json index 6de03706..61f7982f 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "アプリレベルのプロキシを有効化", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "複数モデルにタグを追加", diff --git a/locales/ko.json b/locales/ko.json index cbfb665b..af342656 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "앱 수준 프록시 활성화", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "여러 모델에 태그 추가", diff --git a/locales/ru.json b/locales/ru.json index 9e26faf1..0613fe82 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "Включить прокси на уровне приложения", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "Добавить теги к нескольким моделям", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 53728dd3..106fa7d6 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "启用应用级代理", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "批量添加标签", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 2770a105..f425fa3a 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -809,7 +809,8 @@ "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", - "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmRelocateRoot": "[TODO: Translate] The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?" }, "proxySettings": { "enableProxy": "啟用應用程式代理", @@ -1662,7 +1663,8 @@ "sidecarMigrationConfirm": { "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", - "confirmButton": "[TODO: Translate] Migrate Now" + "confirmButton": "[TODO: Translate] Migrate Now", + "titleRelocateRoot": "[TODO: Translate] Move sidecars to the new storage directory?" }, "bulkAddTags": { "title": "新增標籤到多個模型", diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index a365daf3..27616aa4 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -4141,7 +4141,7 @@ class NodeRegistryHandler: class SidecarMigrationHandler: """Migrate sidecar metadata and previews between storage layouts.""" - _VALID_DIRECTIONS = ("to_centralized", "to_alongside") + _VALID_DIRECTIONS = ("to_centralized", "to_alongside", "relocate_root") def __init__( self, @@ -4168,12 +4168,18 @@ class SidecarMigrationHandler: return web.json_response( { "success": False, - "error": "direction must be 'to_centralized' or 'to_alongside'", + "error": "direction must be 'to_centralized', 'to_alongside' or 'relocate_root'", }, status=400, ) force = params.get("force") in (True, 1, "true", "1") + old_root = str(params.get("old_root") or "").strip() + if direction == "relocate_root" and not old_root: + return web.json_response( + {"success": False, "error": "old_root is required for relocate_root"}, + status=400, + ) use_case = self._use_case_factory() progress_cb = self._progress_callback_factory() @@ -4181,6 +4187,7 @@ class SidecarMigrationHandler: direction=direction, progress_cb=progress_cb, force=force, + old_root=old_root, ) status = 200 if result.get("success") else 400 return web.json_response(result, status=status) diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index c57cf6e4..8d579715 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -777,7 +777,10 @@ class PendingDeleteService: if not os.path.exists(staged_path): continue try: - os.rename(staged_path, original_path) + # EXDEV-tolerant: centralized sidecars may have been copied + # across filesystems into staging, so plain os.rename would + # fail here and strand the only copy. + self._restore_file(staged_path, original_path) except OSError as exc: # pragma: no cover - best-effort rollback logger.warning( "Failed to roll back staged file %s -> %s: %s", diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py index 4bb3756b..04e78994 100644 --- a/py/services/use_cases/sidecar_migration_use_case.py +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -70,6 +70,27 @@ ScannerFactory = Callable[[], Awaitable[Any]] DIRECTION_TO_CENTRALIZED = "to_centralized" DIRECTION_TO_ALONGSIDE = "to_alongside" +DIRECTION_RELOCATE_ROOT = "relocate_root" + +# Same candidate set find_preview_file recognizes: every PREVIEW_EXTENSIONS +# suffix plus the legacy ".example.0.jpeg" (issue #225). +_PREVIEW_CANDIDATE_EXTENSIONS = tuple(PREVIEW_EXTENSIONS) + (".example.0.jpeg",) + + +def _enumerate_preview_names(directory: str, stem: str) -> List[str]: + """Return preview filenames for ``stem`` present in ``directory``. + + Case-insensitive full-name match against the preview candidate set, so + files like ``model.WEBP`` or ``model.Png`` placed by external tools are + migrated along with the exact-case variants. + """ + + targets = {f"{stem.lower()}{ext}" for ext in _PREVIEW_CANDIDATE_EXTENSIONS} + try: + entries = os.listdir(directory) + except OSError: + return [] + return [entry for entry in entries if entry.lower() in targets] class SidecarMigrationUseCase: @@ -136,6 +157,179 @@ class SidecarMigrationUseCase: progress_cb=progress_cb, ) + async def migrate_root( + self, + old_root: str, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Relocate the whole mirror tree from a previous root to the configured one. + + Used after ``sidecar_storage_path`` changes while centralized storage + is active: without it, every asset under the old root would silently + disappear from the application. Moves every file keeping the + root-relative structure, rewrites the ``preview_url`` prefix inside + moved sidecars, reconciles scanner caches, and prunes the emptied old + tree. Keep-newer conflict resolution matches :meth:`_transfer`. + """ + + if not force and get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "sidecar storage is not centralized; pass force=true to relocate anyway", + ) + new_root = get_configured_sidecar_root() + if not new_root: + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "cannot resolve the centralized sidecar root", + ) + old = ( + os.path.abspath(os.path.expanduser(old_root.strip())) + if isinstance(old_root, str) and old_root.strip() + else "" + ) + if not old: + return self._refusal(DIRECTION_RELOCATE_ROOT, "old_root is required") + if os.path.normpath(old) == os.path.normpath(new_root): + return self._refusal( + DIRECTION_RELOCATE_ROOT, + "old_root matches the configured sidecar root", + ) + + files: List[Tuple[str, str]] = [] + if os.path.isdir(old): + for dirpath, _dirnames, filenames in os.walk(old): + rel = os.path.relpath(dirpath, old) + target_dir = new_root if rel == os.curdir else os.path.join(new_root, rel) + for filename in filenames: + files.append( + (os.path.join(dirpath, filename), os.path.join(target_dir, filename)) + ) + + errors: List[Dict[str, str]] = [] + counters: Dict[str, Any] = {"moved": 0, "conflicts": 0} + moved_sidecars: List[str] = [] + + async def emit(status: str, **extra: Any) -> None: + if progress_cb is None: + return + payload: Dict[str, Any] = { + "type": "sidecar_migration_progress", + "status": status, + "direction": DIRECTION_RELOCATE_ROOT, + "total": len(files), + "processed": extra.pop("processed", 0), + "moved": counters["moved"], + "skipped": 0, + "conflicts": counters["conflicts"], + "errors": len(errors), + } + payload.update(extra) + await progress_cb.on_progress(payload) + + await emit("started") + + for index, (src, dst) in enumerate(files, start=1): + try: + if self._transfer(src, dst, counters) and src.endswith(METADATA_SUFFIX): + moved_sidecars.append(dst) + except Exception as exc: + self._logger.error( + "Sidecar root relocation failed for %s: %s", src, exc, exc_info=True + ) + errors.append({"model": os.path.basename(src), "error": str(exc)}) + await emit("processing", processed=index, current=os.path.basename(src)) + + old_prefix = old.replace(os.sep, "/").rstrip("/") + "/" + new_prefix = new_root.replace(os.sep, "/").rstrip("/") + "/" + for sidecar in moved_sidecars: + self._rewrite_root_prefix(sidecar, old_prefix, new_prefix) + await self._reconcile_root_prefix(old_prefix, new_prefix) + + # Prune the emptied old tree, best-effort. + if os.path.isdir(old): + for dirpath, dirnames, filenames in os.walk(old, topdown=False): + if filenames: + continue + for dirname in dirnames: + try: + os.rmdir(os.path.join(dirpath, dirname)) + except OSError: + pass + try: + os.rmdir(dirpath) + except OSError: + pass + + await emit("completed") + + return { + "success": not errors, + "direction": DIRECTION_RELOCATE_ROOT, + "models_total": len(files), + "models_processed": len(files), + "models_moved": 0, + "moved": counters["moved"], + "skipped": 0, + "conflicts": counters["conflicts"], + "errors": errors, + "error_count": len(errors), + } + + def _rewrite_root_prefix( + self, sidecar_path: str, old_prefix: str, new_prefix: str + ) -> None: + """Repoint preview_url inside a relocated sidecar from old to new root.""" + + try: + with open(sidecar_path, "r", encoding="utf-8") as handle: + metadata = json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + self._logger.warning( + "Sidecar root relocation: cannot read %s: %s", sidecar_path, exc + ) + return + + preview_url = metadata.get("preview_url") + if not isinstance(preview_url, str) or not preview_url.startswith(old_prefix): + return + metadata["preview_url"] = new_prefix + preview_url[len(old_prefix):] + try: + with open(sidecar_path, "w", encoding="utf-8") as handle: + json.dump(metadata, handle, ensure_ascii=False, indent=2) + except OSError as exc: + self._logger.warning( + "Sidecar root relocation: cannot rewrite %s: %s", sidecar_path, exc + ) + + async def _reconcile_root_prefix(self, old_prefix: str, new_prefix: str) -> None: + """Rewrite old-root preview URLs in every scanner cache after relocation.""" + + for model_type, factory in self._active_scanner_factories(): + try: + scanner = await factory() + cache = await scanner.get_cached_data() + changed = False + for item in cache.raw_data: + preview_url = item.get("preview_url") + if ( + isinstance(preview_url, str) + and preview_url.startswith(old_prefix) + ): + item["preview_url"] = new_prefix + preview_url[len(old_prefix):] + changed = True + if changed and hasattr(scanner, "_persist_current_cache"): + await scanner._persist_current_cache() + except Exception as exc: + self._logger.error( + "Sidecar root relocation: failed to reconcile %s cache: %s", + model_type, + exc, + exc_info=True, + ) + @staticmethod def _refusal(direction: str, message: str) -> Dict[str, Any]: return { @@ -334,11 +528,9 @@ class SidecarMigrationUseCase: sidecar_name = stem + METADATA_SUFFIX moved_previews: List[str] = [] - for ext in PREVIEW_EXTENSIONS: - src = os.path.join(src_dir, stem + ext) - if not os.path.exists(src): - continue - dst = os.path.join(dst_dir, stem + ext) + for preview_name in _enumerate_preview_names(src_dir, stem): + src = os.path.join(src_dir, preview_name) + dst = os.path.join(dst_dir, preview_name) if self._transfer(src, dst, result): moved_previews.append(dst) @@ -465,6 +657,7 @@ class SidecarMigrationUseCase: direction: str, progress_cb: Optional[SidecarMigrationProgressReporter] = None, force: bool = False, + old_root: Optional[str] = None, ) -> Dict[str, Any]: """Wrapper providing progress notification on unexpected failures.""" @@ -473,8 +666,11 @@ class SidecarMigrationUseCase: return await self.migrate_to_centralized(progress_cb, force=force) if direction == DIRECTION_TO_ALONGSIDE: return await self.migrate_to_alongside(progress_cb, force=force) + if direction == DIRECTION_RELOCATE_ROOT: + return await self.migrate_root(old_root or "", progress_cb, force=force) raise ValueError( - f"direction must be {DIRECTION_TO_CENTRALIZED!r} or {DIRECTION_TO_ALONGSIDE!r}" + f"direction must be {DIRECTION_TO_CENTRALIZED!r}, " + f"{DIRECTION_TO_ALONGSIDE!r} or {DIRECTION_RELOCATE_ROOT!r}" ) except Exception as exc: if progress_cb is not None: diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py index d43f4b54..ad03f6de 100644 --- a/py/utils/sidecar_paths.py +++ b/py/utils/sidecar_paths.py @@ -12,7 +12,7 @@ setting: - ``centralized``: sidecars and previews live under a configurable root (``sidecar_storage_path`` setting, default ``/sidecars``), mirroring the library-relative directory structure: - ``////.metadata.json``. + ``////.metadata.json``. All helpers are pure path computations: no directory scans and no file I/O on the hot path. Settings lookups go through ``SettingsManager.get`` (a dict @@ -21,6 +21,7 @@ read); config roots come from the already-initialized ``config`` singleton. from __future__ import annotations +import hashlib import logging import os import re @@ -145,10 +146,25 @@ def _normalize_for_match(path: str) -> str: return os.path.normpath(os.path.abspath(path)) +def root_mirror_component(root_path: str) -> str: + """Return the mirror path component identifying a model root. + + ``-`` where the hash is a short digest of the + normalized absolute root path. Two roots sharing a basename (e.g. + ``/mnt/a/loras`` and ``/mnt/b/loras``) would otherwise map to the same + mirror directory and overwrite each other's sidecars. + """ + + normalized = _normalize_for_match(root_path) + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8] + return f"{sanitize_path_component(os.path.basename(normalized))}-{digest}" + + def resolve_centralized_dir(model_path: str) -> Optional[str]: """Return the centralized mirror directory for ``model_path``. - The mirror layout is ``///`` + The mirror layout is + ``///`` where ``rel_dir`` is the model's directory relative to the model root that contains it. The longest matching root wins so nested roots resolve to the most specific mirror. Returns ``None`` when centralized storage is inactive @@ -201,7 +217,7 @@ def resolve_centralized_dir_for_dir( library = "default" rel_dir = os.path.relpath(normalized_dir, best_root) - parts = [root, sanitize_path_component(library), sanitize_path_component(os.path.basename(best_root))] + parts = [root, sanitize_path_component(library), root_mirror_component(best_root)] if rel_dir and rel_dir != os.curdir: parts.extend(sanitize_path_component(part) for part in rel_dir.split(os.sep) if part not in ("", os.curdir)) return os.path.join(*parts) diff --git a/static/js/managers/SettingsManager.js b/static/js/managers/SettingsManager.js index eb60c202..55e373a9 100644 --- a/static/js/managers/SettingsManager.js +++ b/static/js/managers/SettingsManager.js @@ -1284,7 +1284,7 @@ export class SettingsManager { onAfterSelect: () => this.saveInputSetting('exampleImagesLocalRoot', 'example_images_local_root'), }); this.attachPathField('sidecarStoragePath', { - onAfterSelect: () => this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'), + onAfterSelect: () => this.handleSidecarStoragePathChange(), }); } @@ -3396,6 +3396,8 @@ export class SettingsManager { } // Baseline used to detect a mode change in handleSidecarStorageModeChange this._loadedSidecarStorageMode = currentMode; + // Baseline used to detect a root change in handleSidecarStoragePathChange + this._loadedSidecarStoragePath = state.global.settings.sidecar_storage_path || ''; const pathInput = document.getElementById('sidecarStoragePath'); if (pathInput) { @@ -3438,6 +3440,30 @@ export class SettingsManager { } } + // Path change while centralized storage is active: the assets under the + // previous root do not move by themselves, so offer a root relocation. + async handleSidecarStoragePathChange() { + const pathInput = document.getElementById('sidecarStoragePath'); + if (!pathInput) return; + + const previousPath = this._loadedSidecarStoragePath || ''; + + await this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'); + + const newPath = pathInput.value.trim(); + this._loadedSidecarStoragePath = newPath; + + const centralized = state.global.settings.sidecar_storage_mode === 'centralized'; + if (centralized && previousPath && previousPath !== newPath) { + const confirmed = await this.confirmSidecarMigration('relocate_root'); + if (confirmed) { + await this.migrateSidecars('relocate_root', { old_root: previousPath }); + } else { + showToast('settings.sidecarStorage.migrationDeferred', {}, 'info'); + } + } + } + // Entry point for the "Migrate Sidecars Now" button: the direction follows // the currently saved storage mode. async confirmAndMigrateSidecars() { @@ -3457,19 +3483,24 @@ export class SettingsManager { } const isToCentralized = direction === 'to_centralized'; + const isRelocate = direction === 'relocate_root'; const titleElement = modalElement.querySelector('[data-role="title"]'); if (titleElement) { - titleElement.textContent = isToCentralized - ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') - : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); + titleElement.textContent = isRelocate + ? translate('modals.sidecarMigrationConfirm.titleRelocateRoot', {}, 'Move sidecars to the new storage directory?') + : isToCentralized + ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') + : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); } const messageElement = modalElement.querySelector('[data-role="message"]'); if (messageElement) { - messageElement.textContent = isToCentralized - ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') - : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); + messageElement.textContent = isRelocate + ? translate('settings.sidecarStorage.confirmRelocateRoot', {}, 'The centralized storage directory changed, but existing sidecars and preview images are still in the previous directory. Move them to the new directory now?') + : isToCentralized + ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') + : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); } const confirmButton = modalElement.querySelector('[data-action="confirm-sidecar-migration"]'); @@ -3528,7 +3559,7 @@ export class SettingsManager { }); } - async migrateSidecars(direction) { + async migrateSidecars(direction, extraBody = {}) { const migrateBtn = document.getElementById('migrateSidecarsBtn'); try { if (migrateBtn) { @@ -3543,10 +3574,10 @@ export class SettingsManager { const response = await fetch('/api/lm/sidecars/migrate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - // The new mode is already saved by the time migration runs, + // The new mode/path is already saved by the time migration runs, // so the backend guard requires force=true to confirm the // "switch first, then migrate" flow. - body: JSON.stringify({ direction, force: true }), + body: JSON.stringify({ direction, force: true, ...extraBody }), }); const data = await response.json(); diff --git a/templates/components/modals/settings/library.html b/templates/components/modals/settings/library.html index 89012f89..aa5424ef 100644 --- a/templates/components/modals/settings/library.html +++ b/templates/components/modals/settings/library.html @@ -354,7 +354,7 @@
diff --git a/tests/frontend/managers/settingsManager.sidecarStorage.test.js b/tests/frontend/managers/settingsManager.sidecarStorage.test.js index ae376076..3f20d3f7 100644 --- a/tests/frontend/managers/settingsManager.sidecarStorage.test.js +++ b/tests/frontend/managers/settingsManager.sidecarStorage.test.js @@ -281,4 +281,60 @@ describe('SettingsManager sidecar storage', () => { expect(migrateBtn.disabled).toBe(false); }); }); + + describe('handleSidecarStoragePathChange', () => { + it('offers root relocation when the path changes in centralized mode', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStoragePathChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await changePromise; + + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + body: JSON.stringify({ direction: 'relocate_root', force: true, old_root: '/old/root' }), + })); + expect(manager._loadedSidecarStoragePath).toBe('/new/root'); + }); + + it('does not prompt when the path changes in alongside mode', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + await manager.handleSidecarStoragePathChange(); + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + }); + + it('shows a deferred notice when relocation is cancelled', async () => { + const manager = createManager(); + const { pathInput } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized', sidecar_storage_path: '/old/root' }; + manager._loadedSidecarStoragePath = '/old/root'; + pathInput.value = '/new/root'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStoragePathChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="cancel-sidecar-migration"]').click(); + await changePromise; + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrationDeferred', {}, 'info'); + }); + }); }); diff --git a/tests/routes/test_misc_routes.py b/tests/routes/test_misc_routes.py index 8ea65248..e693be48 100644 --- a/tests/routes/test_misc_routes.py +++ b/tests/routes/test_misc_routes.py @@ -2565,8 +2565,8 @@ class DummySidecarMigrationUseCase: self.result = result self.calls = [] - async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False): - self.calls.append({"direction": direction, "force": force}) + async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False, old_root=None): + self.calls.append({"direction": direction, "force": force, "old_root": old_root}) return self.result @@ -2592,7 +2592,7 @@ async def test_sidecar_migration_handler_runs_to_centralized(): assert response.status == 200 assert payload["success"] is True assert payload["moved"] == 3 - assert use_case.calls == [{"direction": "to_centralized", "force": True}] + assert use_case.calls == [{"direction": "to_centralized", "force": True, "old_root": ""}] @pytest.mark.asyncio @@ -2624,7 +2624,7 @@ async def test_sidecar_migration_handler_accepts_get_query_params(): assert response.status == 200 assert payload["success"] is True - assert use_case.calls == [{"direction": "to_alongside", "force": True}] + assert use_case.calls == [{"direction": "to_alongside", "force": True, "old_root": ""}] @pytest.mark.asyncio @@ -2640,4 +2640,41 @@ async def test_sidecar_migration_handler_guard_refusal_is_400(): assert response.status == 400 assert payload["success"] is False assert "already centralized" in payload["error"] - assert use_case.calls == [{"direction": "to_centralized", "force": False}] + assert use_case.calls == [{"direction": "to_centralized", "force": False, "old_root": ""}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_relocate_root_passes_old_root(): + result = {"success": True, "direction": "relocate_root", "moved": 5} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest( # pyright: ignore[reportArgumentType] + json_data={ + "direction": "relocate_root", + "old_root": "/old/sidecars", + "force": True, + } + ) + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert use_case.calls == [ + {"direction": "relocate_root", "force": True, "old_root": "/old/sidecars"} + ] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_relocate_root_requires_old_root(): + handler, use_case = _sidecar_migration_handler({"success": True}) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "relocate_root"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert "old_root" in payload["error"] + assert use_case.calls == [] diff --git a/tests/services/test_centralized_sidecar_storage.py b/tests/services/test_centralized_sidecar_storage.py index 2d9164ab..2c40d26e 100644 --- a/tests/services/test_centralized_sidecar_storage.py +++ b/tests/services/test_centralized_sidecar_storage.py @@ -19,6 +19,7 @@ from py.services.model_lifecycle_service import ( from py.services.pending_delete_service import PendingDeleteService from py.services.settings_manager import get_settings_manager from py.utils.metadata_manager import MetadataManager +from py.utils.sidecar_paths import root_mirror_component def _normalize(path) -> str: @@ -57,11 +58,12 @@ def centralized(library_root: Path, tmp_path: Path) -> Path: return sidecar_root -def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: +def _mirror_dir(library_root: Path, sidecar_root: Path, *rel: str) -> Path: """Expected mirror directory for a library-relative path.""" library = get_settings_manager().get_active_library_name() - return sidecar_root.joinpath(library, "checkpoints", *rel) + component = root_mirror_component(str(library_root)) + return sidecar_root.joinpath(library, component, *rel) def _write_sidecar( @@ -93,7 +95,7 @@ async def test_delete_model_artifacts_centralized( ): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") deleted = await delete_model_artifacts(str(library_root), "model") @@ -131,7 +133,7 @@ def test_enumerate_model_artifacts_centralized( ): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.png") service = PendingDeleteService.__new__(PendingDeleteService) @@ -170,7 +172,7 @@ async def _json_metadata_loader(path: str) -> Dict[str, object]: async def test_rename_model_centralized(library_root: Path, centralized: Path): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") service = ModelLifecycleService( @@ -213,7 +215,7 @@ async def test_move_model_centralized( model.write_bytes(b"weights") target_dir = library_root / "new" - old_mirror = _mirror_dir(centralized, "old") + old_mirror = _mirror_dir(library_root, centralized, "old") _write_sidecar( old_mirror, "model", file_path=model, preview_name="model.preview.webp" ) @@ -230,7 +232,7 @@ async def test_move_model_centralized( assert moved_model.exists() assert not model.exists() - new_mirror = _mirror_dir(centralized, "new") + new_mirror = _mirror_dir(library_root, centralized, "new") assert (new_mirror / "model.metadata.json").exists() assert (new_mirror / "model.preview.webp").exists() assert not (old_mirror / "model.metadata.json").exists() @@ -271,7 +273,7 @@ async def test_rename_known_folder_centralized( model.write_bytes(b"weights") old_model_path = old_dir / "model.safetensors" - old_mirror = _mirror_dir(centralized, "oldfolder") + old_mirror = _mirror_dir(library_root, centralized, "oldfolder") _write_sidecar( old_mirror, "model", file_path=old_model_path, preview_name="model.preview.webp" ) @@ -295,7 +297,7 @@ async def test_rename_known_folder_centralized( assert changed is True - new_mirror = _mirror_dir(centralized, "newfolder") + new_mirror = _mirror_dir(library_root, centralized, "newfolder") assert (new_mirror / "model.metadata.json").exists() assert (new_mirror / "model.preview.webp").exists() assert not old_mirror.exists() @@ -315,7 +317,7 @@ async def test_rename_known_folder_centralized( async def test_pending_models_mirror_walk(library_root: Path, centralized: Path): model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar( mirror, "model", @@ -346,7 +348,7 @@ async def test_pending_models_mirror_walk_uses_stem_fallback( model = library_root / "model.safetensors" model.write_bytes(b"weights") - mirror = _mirror_dir(centralized) + mirror = _mirror_dir(library_root, centralized) _write_sidecar( mirror, "model", diff --git a/tests/services/use_cases/test_sidecar_migration_use_case.py b/tests/services/use_cases/test_sidecar_migration_use_case.py index 4251329b..7db41409 100644 --- a/tests/services/use_cases/test_sidecar_migration_use_case.py +++ b/tests/services/use_cases/test_sidecar_migration_use_case.py @@ -12,6 +12,7 @@ import pytest from py.config import config from py.services.settings_manager import get_settings_manager from py.services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase +from py.utils.sidecar_paths import root_mirror_component def _normalize(path) -> str: @@ -52,11 +53,12 @@ def _set_mode(mode: str) -> None: get_settings_manager().set("sidecar_storage_mode", mode) -def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: +def _mirror_dir(library_root: Path, sidecar_root: Path, *rel: str) -> Path: """Expected mirror directory for a library-relative path.""" library = get_settings_manager().get_active_library_name() - return sidecar_root.joinpath(library, "loras", *rel) + component = root_mirror_component(str(library_root)) + return sidecar_root.joinpath(library, component, *rel) def _write_model(directory: Path, stem: str) -> Path: @@ -152,7 +154,7 @@ async def test_migrate_to_centralized_moves_sidecar_and_previews( assert summary["conflicts"] == 0 assert summary["errors"] == [] - mirror = _mirror_dir(sidecar_root, "sub") + mirror = _mirror_dir(library_root, sidecar_root, "sub") assert not sidecar.exists() assert not preview.exists() assert not extra_preview.exists() @@ -186,7 +188,7 @@ async def test_migrate_to_alongside_reverses_layout( ): _set_mode("alongside") model = _write_model(library_root / "sub", "model") - mirror = _mirror_dir(sidecar_root, "sub") + mirror = _mirror_dir(library_root, sidecar_root, "sub") sidecar = _write_sidecar(mirror, "model", model) preview = mirror / "model.preview.webp" preview.write_bytes(b"preview") @@ -221,7 +223,7 @@ async def test_migrate_conflict_keeps_newer_file( library_root: Path, sidecar_root: Path ): _set_mode("centralized") - mirror = _mirror_dir(sidecar_root) + mirror = _mirror_dir(library_root, sidecar_root) # Model A: destination (mirror) sidecar is newer -> destination wins. model_a = _write_model(library_root, "model_a") @@ -384,6 +386,121 @@ async def test_migrate_reconcile_survives_per_model_errors( # The healthy model's cache entry is still reconciled and persisted. ok_entry = use_case._test_scanner._cache.raw_data[0] assert ok_entry["preview_url"] == _normalize( - _mirror_dir(sidecar_root) / "ok.preview.webp" + _mirror_dir(library_root, sidecar_root) / "ok.preview.webp" ) assert use_case._test_scanner.persist_calls == 1 + + +@pytest.mark.asyncio +async def test_migrate_covers_mixed_case_and_example_previews( + library_root: Path, sidecar_root: Path +): + """Previews like model.WEBP / model.example.0.jpeg migrate too (#225 compat).""" + + _set_mode("centralized") + model = _write_model(library_root, "model") + _write_sidecar(library_root, "model", model, preview_ext=".preview.WEBP") + (library_root / "model.preview.WEBP").write_bytes(b"preview") + (library_root / "model.example.0.jpeg").write_bytes(b"example") + + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is True + assert summary["moved"] == 3 # sidecar + 2 previews + + mirror = _mirror_dir(library_root, sidecar_root) + assert (mirror / "model.preview.WEBP").exists() + assert (mirror / "model.example.0.jpeg").exists() + assert not (library_root / "model.preview.WEBP").exists() + assert not (library_root / "model.example.0.jpeg").exists() + + metadata = json.loads((mirror / "model.metadata.json").read_text(encoding="utf-8")) + assert metadata["preview_url"] == _normalize(mirror / "model.preview.WEBP") + + +@pytest.mark.asyncio +async def test_migrate_root_relocates_tree_and_reconciles( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + library = get_settings_manager().get_active_library_name() + component = root_mirror_component(str(library_root)) + + # Assets under the OLD root, mirroring the layout. + old_root = tmp_path / "old_sidecars" + old_mirror = old_root / library / component / "sub" + old_mirror.mkdir(parents=True) + model = _write_model(library_root / "sub", "model") + payload = { + "file_name": "model", + "file_path": _normalize(model), + "preview_url": _normalize(old_mirror / "model.preview.png"), + } + (old_mirror / "model.metadata.json").write_text(json.dumps(payload), encoding="utf-8") + (old_mirror / "model.preview.png").write_bytes(b"preview") + + entries = [ + { + "file_path": str(model), + "preview_url": _normalize(old_mirror / "model.preview.png"), + "preview_nsfw_level": 2, + } + ] + use_case = _make_use_case_with_entries(entries) + summary = await use_case.migrate_root(str(old_root), force=True) + + assert summary["success"] is True + assert summary["moved"] == 2 + + new_mirror = sidecar_root / library / component / "sub" + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.png").exists() + + # Sidecar preview_url rewritten onto the new root. + migrated = json.loads((new_mirror / "model.metadata.json").read_text(encoding="utf-8")) + assert migrated["preview_url"] == _normalize(new_mirror / "model.preview.png") + # Model path fields untouched — model files never move. + assert migrated["file_path"] == _normalize(model) + + # Scanner cache preview URLs repointed and persisted. + entry = use_case._test_scanner._cache.raw_data[0] + assert entry["preview_url"] == _normalize(new_mirror / "model.preview.png") + assert use_case._test_scanner.persist_calls == 1 + + # Emptied old tree pruned. + assert not old_root.exists() + + +@pytest.mark.asyncio +async def test_migrate_root_guards( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_root("") + assert summary["success"] is False + assert "old_root is required" in summary["error"] + + summary = await use_case.migrate_root(str(sidecar_root)) + assert summary["success"] is False + assert "matches the configured" in summary["error"] + + _set_mode("alongside") + summary = await use_case.migrate_root(str(tmp_path / "old_sidecars")) + assert summary["success"] is False + assert "not centralized" in summary["error"] + + +@pytest.mark.asyncio +async def test_migrate_root_missing_old_tree_is_noop( + library_root: Path, sidecar_root: Path, tmp_path: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_root(str(tmp_path / "nonexistent"), force=True) + + assert summary["success"] is True + assert summary["moved"] == 0 diff --git a/tests/utils/test_sidecar_paths.py b/tests/utils/test_sidecar_paths.py index f09966d9..a5a4f6dd 100644 --- a/tests/utils/test_sidecar_paths.py +++ b/tests/utils/test_sidecar_paths.py @@ -22,6 +22,7 @@ from py.utils.sidecar_paths import ( resolve_centralized_dir, resolve_centralized_dir_for_dir, resolve_metadata_path, + root_mirror_component, sanitize_path_component, ) @@ -117,16 +118,48 @@ class TestCentralizedMode: def test_mirror_layout(self, model_roots: dict, centralized: Path): model = model_roots["loras"] / "styles" / "anime" / "model.safetensors" library = get_settings_manager().get_active_library_name() + root_component = root_mirror_component(str(model_roots["loras"])) metadata_path = get_metadata_path(str(model)) expected = os.path.join( - str(centralized), library, "loras", "styles", "anime", "model" + METADATA_SUFFIX + str(centralized), library, root_component, "styles", "anime", "model" + METADATA_SUFFIX ) assert metadata_path == expected assert get_preview_dir(str(model)) == os.path.dirname(expected) assert is_centralized() + def test_same_basename_roots_get_distinct_mirrors( + self, model_roots: dict, centralized: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + from py.config import config + + # Two roots sharing the basename "loras" must not share a mirror dir. + other_parent = tmp_path / "elsewhere" + other_root = other_parent / "loras" + other_root.mkdir(parents=True) + monkeypatch.setattr( + config, + "loras_roots", + [str(model_roots["loras"]), str(other_root)], + raising=False, + ) + + model_a = model_roots["loras"] / "model.safetensors" + model_b = other_root / "model.safetensors" + + dir_a = resolve_centralized_dir(str(model_a)) + dir_b = resolve_centralized_dir(str(model_b)) + assert dir_a is not None and dir_b is not None + assert dir_a != dir_b + assert root_mirror_component(str(model_roots["loras"])) != root_mirror_component( + str(other_root) + ) + # Same root always maps to the same component (stable hash). + assert root_mirror_component(str(model_roots["loras"])) == root_mirror_component( + str(model_roots["loras"]) + os.sep + ) + def test_longest_root_wins(self, model_roots: dict, centralized: Path, monkeypatch: pytest.MonkeyPatch): from py.config import config @@ -142,7 +175,7 @@ class TestCentralizedMode: model = nested / "model.safetensors" assert get_metadata_path(str(model)) == os.path.join( - str(centralized), library, "nested", "model" + METADATA_SUFFIX + str(centralized), library, root_mirror_component(str(nested)), "model" + METADATA_SUFFIX ) def test_outside_roots_falls_back_to_alongside( @@ -174,7 +207,7 @@ class TestCentralizedMode: library = get_settings_manager().get_active_library_name() assert resolve_centralized_dir_for_dir(str(model_roots["loras"])) == os.path.join( - str(centralized), library, "loras" + str(centralized), library, root_mirror_component(str(model_roots["loras"])) ) def test_empty_path_uses_default_sidecar_root(self, model_roots: dict, tmp_path: Path): @@ -223,7 +256,9 @@ class TestModeIndependentResolution: assert resolve_centralized_dir_for_dir(str(model_dir)) is None assert resolve_centralized_dir_for_dir( str(model_dir), sidecar_root=str(sidecar_root) - ) == os.path.join(str(sidecar_root), library, "loras", "sub") + ) == os.path.join( + str(sidecar_root), library, root_mirror_component(str(model_roots["loras"])), "sub" + ) class TestSettingsValidation: