From f5e983eaaab67e0df7866f5822b561bf993cf630 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 26 Sep 2026 10:00:58 +0800 Subject: [PATCH] feat: optional centralized storage for sidecar metadata and previews (#1045) Add an opt-in 'centralized' sidecar storage mode alongside the default 'alongside' layout. In centralized mode, .metadata.json sidecars and preview assets live under a configurable root (sidecar_storage_path, default /sidecars), mirroring the library-relative directory structure: ////. Backend: - settings: sidecar_storage_mode / sidecar_storage_path with validation; changing either refreshes the preview allowlist - config: centralized root added to preview-serving allowlist - lifecycle: delete / move / rename / folder-rename / folder-delete and undoable-delete staging all operate on the mirror tree in centralized mode (model files themselves never move); EXDEV-tolerant cross- filesystem moves - scanners: pending-hash filesystem scan walks the mirror tree in centralized mode; preview discovery reads from the sidecar dir; .civitai.info stays co-located in both modes - migration: SidecarMigrationUseCase moves sidecars+previews between layouts both directions (keep-newer conflict resolution, preview_url rewriting, WebSocket progress), exposed as POST+GET /api/lm/sidecars/migrate with a mode guard (force=true for the settings-first flow) Frontend: - settings modal: sidecar storage section (mode select + path input with browse/validation), mode-change confirmation offering immediate migration (force=true), and a 'Migrate Sidecars Now' action - i18n keys synced to all locales ([TODO: Translate] placeholders) Docs: metadata-json-schema.md gains a storage-location section; AGENTS.md records the sidecar_paths helper convention. --- AGENTS.md | 7 + docs/metadata-json-schema.md | 22 + locales/de.json | 27 ++ locales/en.json | 27 ++ locales/es.json | 27 ++ locales/fr.json | 27 ++ locales/he.json | 27 ++ locales/ja.json | 27 ++ locales/ko.json | 27 ++ locales/ru.json | 27 ++ locales/zh-CN.json | 27 ++ locales/zh-TW.json | 27 ++ py/config.py | 20 + py/metadata_ops/__init__.py | 3 + py/routes/handlers/misc_handlers.py | 57 +++ py/routes/misc_route_registrar.py | 7 + py/routes/misc_routes.py | 3 + py/services/checkpoint_scanner.py | 9 +- py/services/download_manager.py | 6 + py/services/model_file_service.py | 16 + py/services/model_lifecycle_service.py | 51 ++- py/services/model_scanner.py | 194 ++++++-- py/services/other_scanner.py | 9 +- py/services/pending_delete_service.py | 33 +- py/services/preview_asset_service.py | 6 + py/services/settings_manager.py | 31 +- py/services/use_cases/__init__.py | 6 + .../use_cases/sidecar_migration_use_case.py | 418 ++++++++++++++++++ py/utils/metadata_manager.py | 20 +- py/utils/sidecar_paths.py | 65 ++- static/js/managers/SettingsManager.js | 192 ++++++++ static/js/state/index.js | 2 + .../components/modals/confirm_modals.html | 14 + .../components/modals/settings/library.html | 56 +++ .../settingsManager.sidecarStorage.test.js | 284 ++++++++++++ tests/routes/test_misc_routes.py | 84 ++++ .../test_centralized_sidecar_storage.py | 361 +++++++++++++++ .../test_sidecar_migration_use_case.py | 294 ++++++++++++ tests/utils/test_sidecar_paths.py | 250 +++++++++++ 39 files changed, 2704 insertions(+), 86 deletions(-) create mode 100644 py/services/use_cases/sidecar_migration_use_case.py create mode 100644 tests/frontend/managers/settingsManager.sidecarStorage.test.js create mode 100644 tests/services/test_centralized_sidecar_storage.py create mode 100644 tests/services/use_cases/test_sidecar_migration_use_case.py create mode 100644 tests/utils/test_sidecar_paths.py diff --git a/AGENTS.md b/AGENTS.md index 09d2b032..87561abe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -266,6 +266,13 @@ If a cross-layer issue ever needs a live server, the sandboxed helpers live in them during migration/import. Never write, modify, or delete them, and never propose doing so as a fix — LoRA Manager's own metadata lives in the `.metadata.json` sidecar it owns. +- **Sidecar/preview path derivation must go through `py/utils/sidecar_paths.py`** + helpers (never inline `splitext + ".metadata.json"`): the centralized storage + mode (`sidecar_storage_mode` / `sidecar_storage_path` settings) relocates + `.metadata.json` files and preview images under a mirror tree, so any + hand-built path is wrong in that mode. `.civitai.info` stays co-located with + the model file in both modes. The new settings keys live only in + `DEFAULT_SETTINGS` — `settings.json.example` stays minimal (see below). - **`settings.json.example` must stay minimal**: only `use_portable_settings`, `civitai_api_key`, and the four core `folder_paths` keys (`loras`, `checkpoints`, `unet`, `embeddings`). Do NOT add optional/default keys diff --git a/docs/metadata-json-schema.md b/docs/metadata-json-schema.md index 235733c6..00672a6a 100644 --- a/docs/metadata-json-schema.md +++ b/docs/metadata-json-schema.md @@ -11,6 +11,28 @@ This document defines the complete schema for `.metadata.json` files used by Lor --- +## Storage Location (Alongside vs Centralized) + +By default, `.metadata.json` sidecars and preview images live **alongside** their model files. An optional centralized mode stores them under a single root directory instead. Two settings control this (Settings → Library → Sidecar Storage): + +| Setting | Values | Default | +|---------|--------|---------| +| `sidecar_storage_mode` | `"alongside"` \| `"centralized"` | `"alongside"` | +| `sidecar_storage_path` | Absolute path string; empty = `/sidecars` | `""` | + +In centralized mode, sidecars and previews mirror the library-relative directory structure: + +``` +////.metadata.json +``` + +- `` is the active library name, `` the basename of the model root containing the file, and `` the model's directory relative to that root. Each component is sanitized to filesystem-safe characters. +- `.civitai.info` files always stay next to the model file, in both modes. +- Changing the mode does **not** move existing files automatically — run the migration (`POST /api/lm/sidecars/migrate` with `{"direction": "to_centralized" | "to_alongside"}`, or the "Migrate Sidecars Now" button in settings). +- All sidecar/preview path derivation goes through the helpers in `py/utils/sidecar_paths.py`; never construct paths inline. + +--- + ## Base Fields (All Model Types) These fields are present in all model metadata files. diff --git a/locales/de.json b/locales/de.json index c2a4a3bd..ca6b8990 100644 --- a/locales/de.json +++ b/locales/de.json @@ -383,6 +383,7 @@ "exampleImages": "Beispielbilder", "autoOrganize": "Auto-Organisierung", "metadata": "Metadaten", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Proxy-Einstellungen" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "App-Proxy aktivieren", "enableProxyHelp": "Aktivieren Sie benutzerdefinierte Proxy-Einstellungen für diese Anwendung. Überschreibt die System-Proxy-Einstellungen.", @@ -1637,6 +1659,11 @@ "titleRevert": "Ursprüngliche Dateinamen wiederherstellen?", "revertButton": "Ursprüngliche Dateinamen wiederherstellen" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Tags zu mehreren Modellen hinzufügen", "description": "Tags hinzufügen zu", diff --git a/locales/en.json b/locales/en.json index de13e421..40eff655 100644 --- a/locales/en.json +++ b/locales/en.json @@ -383,6 +383,7 @@ "exampleImages": "Example Images", "autoOrganize": "Auto-organize", "metadata": "Metadata", + "sidecarStorage": "Sidecar Storage", "proxySettings": "Proxy Settings" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "Sidecar Storage Mode", + "modeHelp": "Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "Alongside model files (default)", + "centralized": "Centralized storage" + }, + "path": "Centralized Storage Path", + "pathHelp": "Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "Empty = /sidecars", + "management": "Sidecar Migration", + "managementHelp": "Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "Migrate Sidecars Now", + "migratingButton": "Migrating...", + "migrating": "Migrating sidecars...", + "migrateSuccess": "Sidecar migration completed successfully", + "migrateFailed": "Sidecar migration failed: {message}", + "migrationDeferred": "Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Enable App-level Proxy", "enableProxyHelp": "Enable custom proxy settings for this application, overriding system proxy settings", @@ -1637,6 +1659,11 @@ "titleRevert": "Restore original filenames?", "revertButton": "Restore Original Filenames" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "Move sidecars to centralized storage?", + "titleToAlongside": "Move sidecars back next to model files?", + "confirmButton": "Migrate Now" + }, "bulkAddTags": { "title": "Add Tags to Multiple Models", "description": "Add tags to", diff --git a/locales/es.json b/locales/es.json index b8a2a25b..54654810 100644 --- a/locales/es.json +++ b/locales/es.json @@ -383,6 +383,7 @@ "exampleImages": "Imágenes de ejemplo", "autoOrganize": "Organización automática", "metadata": "Metadatos", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Configuración de proxy" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Habilitar proxy a nivel de aplicación", "enableProxyHelp": "Habilita la configuración de proxy personalizada para esta aplicación, sobrescribiendo la configuración de proxy del sistema", @@ -1637,6 +1659,11 @@ "titleRevert": "¿Restaurar los nombres de archivo originales?", "revertButton": "Restaurar nombres de archivo originales" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Añadir etiquetas a múltiples modelos", "description": "Añadir etiquetas a", diff --git a/locales/fr.json b/locales/fr.json index adabe9a7..792745be 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -383,6 +383,7 @@ "exampleImages": "Images d'exemple", "autoOrganize": "Organisation automatique", "metadata": "Métadonnées", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Paramètres du proxy" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Activer le proxy au niveau de l'application", "enableProxyHelp": "Activer les paramètres de proxy personnalisés pour cette application, remplaçant les paramètres de proxy système", @@ -1637,6 +1659,11 @@ "titleRevert": "Restaurer les noms de fichier d'origine ?", "revertButton": "Restaurer les noms de fichier d'origine" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Ajouter des tags à plusieurs modèles", "description": "Ajouter des tags à", diff --git a/locales/he.json b/locales/he.json index 203e0ca4..672b8464 100644 --- a/locales/he.json +++ b/locales/he.json @@ -383,6 +383,7 @@ "exampleImages": "תמונות דוגמה", "autoOrganize": "ארגון אוטומטי", "metadata": "מטא-נתונים", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "הגדרות פרוקסי" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "הפעל פרוקסי ברמת האפליקציה", "enableProxyHelp": "אפשר הגדרות פרוקסי מותאמות אישית עבור יישום זה, במקום הגדרות הפרוקסי של המערכת", @@ -1637,6 +1659,11 @@ "titleRevert": "לשחזר שמות קבצים מקוריים?", "revertButton": "שחזר שמות קבצים מקוריים" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "הוסף תגיות למספר מודלים", "description": "הוסף תגיות ל-", diff --git a/locales/ja.json b/locales/ja.json index 8f99755c..6de03706 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -383,6 +383,7 @@ "exampleImages": "例画像", "autoOrganize": "自動整理", "metadata": "メタデータ", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "プロキシ設定" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "アプリレベルのプロキシを有効化", "enableProxyHelp": "このアプリケーション専用のカスタムプロキシ設定を有効にします（システムのプロキシ設定を上書きします）", @@ -1637,6 +1659,11 @@ "titleRevert": "元のファイル名を復元しますか？", "revertButton": "元のファイル名を復元" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "複数モデルにタグを追加", "description": "タグを追加するモデル：", diff --git a/locales/ko.json b/locales/ko.json index 35e558f9..cbfb665b 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -383,6 +383,7 @@ "exampleImages": "예시 이미지", "autoOrganize": "자동 정리", "metadata": "메타데이터", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "프록시 설정" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "앱 수준 프록시 활성화", "enableProxyHelp": "이 애플리케이션에 대한 사용자 지정 프록시 설정을 활성화하여 시스템 프록시 설정을 무시합니다", @@ -1637,6 +1659,11 @@ "titleRevert": "원본 파일명을 복원하시겠습니까?", "revertButton": "원본 파일명 복원" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "여러 모델에 태그 추가", "description": "다음에 태그를 추가합니다:", diff --git a/locales/ru.json b/locales/ru.json index 2d8e8552..9e26faf1 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -383,6 +383,7 @@ "exampleImages": "Примеры изображений", "autoOrganize": "Автоорганизация", "metadata": "Метаданные", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "Настройки прокси" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "Включить прокси на уровне приложения", "enableProxyHelp": "Включить пользовательские настройки прокси для этого приложения, переопределяя системные настройки прокси", @@ -1637,6 +1659,11 @@ "titleRevert": "Восстановить исходные имена файлов?", "revertButton": "Восстановить исходные имена файлов" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "Добавить теги к нескольким моделям", "description": "Добавить теги к", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 84a730b7..53728dd3 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -383,6 +383,7 @@ "exampleImages": "示例图片", "autoOrganize": "自动整理", "metadata": "元数据", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "代理设置" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "启用应用级代理", "enableProxyHelp": "为此应用启用自定义代理设置，覆盖系统代理设置", @@ -1637,6 +1659,11 @@ "titleRevert": "恢复原始文件名？", "revertButton": "恢复原始文件名" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "批量添加标签", "description": "为多个模型添加标签", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 3e536578..2770a105 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -383,6 +383,7 @@ "exampleImages": "範例圖片", "autoOrganize": "自動整理", "metadata": "中繼資料", + "sidecarStorage": "[TODO: Translate] Sidecar Storage", "proxySettings": "代理設定" }, "nav": { @@ -789,6 +790,27 @@ "providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB", "providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive" }, + "sidecarStorage": { + "mode": "[TODO: Translate] Sidecar Storage Mode", + "modeHelp": "[TODO: Translate] Choose where .metadata.json sidecars and preview images are stored: next to each model file, or in a single centralized directory that mirrors your library structure. .civitai.info files always stay next to the model file.", + "modeOptions": { + "alongside": "[TODO: Translate] Alongside model files (default)", + "centralized": "[TODO: Translate] Centralized storage" + }, + "path": "[TODO: Translate] Centralized Storage Path", + "pathHelp": "[TODO: Translate] Root directory for centralized sidecar storage. Leave empty to use the default location (/sidecars).", + "pathPlaceholder": "[TODO: Translate] Empty = /sidecars", + "management": "[TODO: Translate] Sidecar Migration", + "managementHelp": "[TODO: Translate] Move existing .metadata.json sidecars and preview images between alongside and centralized storage, matching the currently selected mode. Changing the mode does not move existing files automatically.", + "migrateButton": "[TODO: Translate] Migrate Sidecars Now", + "migratingButton": "[TODO: Translate] Migrating...", + "migrating": "[TODO: Translate] Migrating sidecars...", + "migrateSuccess": "[TODO: Translate] Sidecar migration completed successfully", + "migrateFailed": "[TODO: Translate] Sidecar migration failed: {message}", + "migrationDeferred": "[TODO: Translate] Existing sidecars were not moved. You can migrate them later from Settings → Library → Sidecar Storage.", + "confirmToCentralized": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the \"Migrate Sidecars Now\" button.", + "confirmToAlongside": "[TODO: Translate] The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the \"Migrate Sidecars Now\" button." + }, "proxySettings": { "enableProxy": "啟用應用程式代理", "enableProxyHelp": "啟用此應用程式的自訂代理設定，將覆蓋系統代理設定", @@ -1637,6 +1659,11 @@ "titleRevert": "要還原原始檔案名稱嗎？", "revertButton": "還原原始檔案名稱" }, + "sidecarMigrationConfirm": { + "titleToCentralized": "[TODO: Translate] Move sidecars to centralized storage?", + "titleToAlongside": "[TODO: Translate] Move sidecars back next to model files?", + "confirmButton": "[TODO: Translate] Migrate Now" + }, "bulkAddTags": { "title": "新增標籤到多個模型", "description": "新增標籤到", diff --git a/py/config.py b/py/config.py index 3c5465c4..8ec4b9af 100644 --- a/py/config.py +++ b/py/config.py @@ -891,6 +891,17 @@ class Config: if self.recipes_path: preview_roots.update(self._expand_preview_root(self.recipes_path)) + # Centralized sidecar storage holds preview assets outside the model + # roots; allow serving them when the mode is active. + try: + from .utils.sidecar_paths import get_sidecar_root # Local import to avoid circular dependency + + sidecar_root = get_sidecar_root() + except Exception: # pragma: no cover - defensive fallback + sidecar_root = "" + if sidecar_root: + preview_roots.update(self._expand_preview_root(sidecar_root)) + for target, link in self._path_mappings.items(): preview_roots.update(self._expand_preview_root(target)) preview_roots.update(self._expand_preview_root(link)) @@ -1494,6 +1505,15 @@ class Config: self.other_roots = self._init_other_paths() self._rebuild_preview_roots() + def refresh_preview_roots(self) -> None: + """Rebuild the preview allowlist after path-affecting settings change. + + Called when ``sidecar_storage_mode`` / ``sidecar_storage_path`` are + updated so centralized preview assets become servable (or stop being + servable) without a restart. + """ + self._rebuild_preview_roots() + def get_other_models_availability(self) -> Dict[str, Any]: """Report the other-model folders the host can actually expose. diff --git a/py/metadata_ops/__init__.py b/py/metadata_ops/__init__.py index 40781ff4..15b0f1d9 100644 --- a/py/metadata_ops/__init__.py +++ b/py/metadata_ops/__init__.py @@ -179,6 +179,9 @@ async def download_preview( base_name = os.path.splitext(os.path.basename(model_path))[0] preview_dir = get_preview_dir(model_path) + # Centralized mirrors may not exist yet (unlike the model's own directory + # in alongside mode). + os.makedirs(preview_dir, exist_ok=True) output_path = os.path.join(preview_dir, base_name + ".webp") downloader = await get_downloader() diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index 8e156f6a..a365daf3 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -45,6 +45,8 @@ from ...services.llm_service import ( get_provider_model_ids, ) from ...services.cache_health_monitor import CacheHealthMonitor, CacheHealthStatus +from ...services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase +from ...services.websocket_progress_callback import WebSocketBroadcastCallback from ...utils.models import BaseModelMetadata from ...utils.constants import ( CIVITAI_USER_MODEL_TYPES, @@ -4136,6 +4138,57 @@ class NodeRegistryHandler: return web.json_response({"success": False, "error": str(exc)}, status=500) +class SidecarMigrationHandler: + """Migrate sidecar metadata and previews between storage layouts.""" + + _VALID_DIRECTIONS = ("to_centralized", "to_alongside") + + def __init__( + self, + *, + use_case_factory: Callable[[], SidecarMigrationUseCase] = SidecarMigrationUseCase, + progress_callback_factory: Callable[[], Any] = WebSocketBroadcastCallback, + ) -> None: + self._use_case_factory = use_case_factory + self._progress_callback_factory = progress_callback_factory + + async def migrate_sidecars(self, request: web.Request) -> web.Response: + """Run a sidecar migration; accepts POST JSON or GET query params.""" + try: + if request.method == "GET": + params: Mapping[str, Any] = request.query + else: + try: + params = await request.json() + except Exception: # empty/invalid body: fall back to query + params = request.query + + direction = str(params.get("direction") or "").strip() + if direction not in self._VALID_DIRECTIONS: + return web.json_response( + { + "success": False, + "error": "direction must be 'to_centralized' or 'to_alongside'", + }, + status=400, + ) + + force = params.get("force") in (True, 1, "true", "1") + + use_case = self._use_case_factory() + progress_cb = self._progress_callback_factory() + result = await use_case.execute_with_error_handling( + direction=direction, + progress_cb=progress_cb, + force=force, + ) + status = 200 if result.get("success") else 400 + return web.json_response(result, status=status) + except Exception as exc: + logger.error("Sidecar migration failed: %s", exc, exc_info=True) + return web.json_response({"success": False, "error": str(exc)}, status=500) + + class MiscHandlerSet: """Aggregate handlers into a lookup compatible with the registrar.""" @@ -4162,6 +4215,7 @@ class MiscHandlerSet: model_source_handler: Any = None, agent_handler: Any = None, download_routing: Any = None, + sidecar_migration: Any = None, ) -> None: self.health = health self.settings = settings @@ -4183,6 +4237,7 @@ class MiscHandlerSet: self.model_source_handler = model_source_handler self.agent_handler = agent_handler self.download_routing = download_routing + self.sidecar_migration = sidecar_migration def to_route_mapping( self, @@ -4249,6 +4304,8 @@ class MiscHandlerSet: "cancel_agent_skill": self.agent_handler.cancel_agent_skill, # Download routing handler "get_download_routing": self.download_routing.get_download_routing, + # Sidecar migration handler + "migrate_sidecars": self.sidecar_migration.migrate_sidecars, # Base model handlers "get_base_models": self.base_model.get_base_models, "refresh_base_models": self.base_model.refresh_base_models, diff --git a/py/routes/misc_route_registrar.py b/py/routes/misc_route_registrar.py index 0c4dde79..ccfa440a 100644 --- a/py/routes/misc_route_registrar.py +++ b/py/routes/misc_route_registrar.py @@ -113,6 +113,13 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = ( RouteDefinition( "POST", "/api/lm/download/routing", "get_download_routing" ), + # Sidecar storage layout migration (GET supported for the extension) + RouteDefinition( + "POST", "/api/lm/sidecars/migrate", "migrate_sidecars" + ), + RouteDefinition( + "GET", "/api/lm/sidecars/migrate", "migrate_sidecars" + ), RouteDefinition( "POST", "/api/lm/download-model-source", "download_model_source" ), diff --git a/py/routes/misc_routes.py b/py/routes/misc_routes.py index 8db36ce4..03b980a0 100644 --- a/py/routes/misc_routes.py +++ b/py/routes/misc_routes.py @@ -32,6 +32,7 @@ from .handlers.misc_handlers import ( NodeRegistry, NodeRegistryHandler, SettingsHandler, + SidecarMigrationHandler, SupportersHandler, TrainedWordsHandler, UsageStatsHandler, @@ -142,6 +143,7 @@ class MiscRoutes: model_source_handler = ModelSourceHandler() agent_handler = AgentHandler() download_routing = DownloadRoutingHandler() + sidecar_migration = SidecarMigrationHandler() return self._handler_set_factory( health=health, @@ -164,6 +166,7 @@ class MiscRoutes: model_source_handler=model_source_handler, agent_handler=agent_handler, download_routing=download_routing, + sidecar_migration=sidecar_migration, ) diff --git a/py/services/checkpoint_scanner.py b/py/services/checkpoint_scanner.py index 02402798..d5842499 100644 --- a/py/services/checkpoint_scanner.py +++ b/py/services/checkpoint_scanner.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional from ..utils.models import CheckpointMetadata from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3 from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_preview_dir, is_centralized from ..config import config from .model_scanner import ModelScanner, _is_excluded_dir from .model_hash_index import ModelHashIndex @@ -61,10 +62,9 @@ class CheckpointScanner(ModelScanner): return None base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # AutoV3 reads only the safetensors header, so it is cheap even for # large checkpoints; record the checked state at creation time ("" = @@ -322,6 +322,11 @@ class CheckpointScanner(ModelScanner): async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]: """Scan filesystem for checkpoint metadata files with pending hash status.""" + # Centralized mode stores sidecars in the mirror tree, not next to the + # models; walk the mirror instead of the model folders. + if is_centralized(): + return self._find_pending_models_in_sidecar_mirror() + pending_models = [] for root_path in self.get_model_roots(): diff --git a/py/services/download_manager.py b/py/services/download_manager.py index 52518ec7..b2cf40ec 100644 --- a/py/services/download_manager.py +++ b/py/services/download_manager.py @@ -2465,6 +2465,10 @@ class DownloadManager: # Download preview image if available images = version_info.get("images", []) if images: + # Centralized preview mirrors may not exist yet (unlike the + # model's own directory in alongside mode). + os.makedirs(get_preview_dir(save_path), exist_ok=True) + if progress_callback: await progress_callback( 1 @@ -3067,10 +3071,12 @@ class DownloadManager: first_target = targets[0] if preview_path != first_target: + os.makedirs(os.path.dirname(first_target), exist_ok=True) os.replace(preview_path, first_target) source_path = first_target for target in targets[1:]: + os.makedirs(os.path.dirname(target), exist_ok=True) shutil.copyfile(source_path, target) return targets diff --git a/py/services/model_file_service.py b/py/services/model_file_service.py index 659805cd..8fef00a7 100644 --- a/py/services/model_file_service.py +++ b/py/services/model_file_service.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from ..utils.utils import calculate_relative_path_for_model, remove_empty_dirs from ..utils.constants import AUTO_ORGANIZE_BATCH_SIZE, MODEL_FILE_EXTENSIONS +from ..utils.sidecar_paths import is_centralized, resolve_centralized_dir_for_dir from ..services.settings_manager import get_settings_manager from ..services.model_lifecycle_service import _require_path_in_library_roots from ..services.pending_delete_service import PENDING_DELETE_DIR_NAME @@ -631,6 +632,21 @@ class ModelMoveService: shutil.rmtree(absolute_path) + # Centralized mode: prune the folder's mirror subtree when it no + # longer holds any sidecar files (per-model deletes already + # removed their sidecars, so only empty directories are expected; + # a non-empty mirror keeps its orphan sidecars). + if is_centralized(): + mirror_dir = resolve_centralized_dir_for_dir(absolute_path) + if mirror_dir and os.path.isdir(mirror_dir): + for root, _dirs, files in os.walk(mirror_dir, topdown=False): + if files: + continue + try: + os.rmdir(root) + except OSError: # pragma: no cover - best-effort cleanup + pass + await self._forget_folder(relative_folder) return { diff --git a/py/services/model_lifecycle_service.py b/py/services/model_lifecycle_service.py index bb59f97f..f5c8fb4b 100644 --- a/py/services/model_lifecycle_service.py +++ b/py/services/model_lifecycle_service.py @@ -11,7 +11,7 @@ from ..services.service_registry import ServiceRegistry from ..services.pending_delete_service import get_pending_delete_service from ..utils.constants import PREVIEW_EXTENSIONS from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path +from ..utils.sidecar_paths import get_metadata_path, get_preview_dir, get_sidecar_dir logger = logging.getLogger(__name__) @@ -42,19 +42,22 @@ async def load_local_metadata(metadata_path: str) -> Dict[str, Any]: async def delete_model_artifacts( target_dir: str, file_name: str, main_extension: str | None = None ) -> List[str]: - """Delete the primary model artefacts within ``target_dir``.""" + """Delete the primary model artefacts within ``target_dir``. + + Sidecars and previews are taken from the model's sidecar directory — the + model's own directory in alongside mode, the centralized mirror otherwise. + """ main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [ - main_file, - os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), - ] + model_path = os.path.join(target_dir, main_file) + sidecar_dir = get_sidecar_dir(model_path) + patterns = [os.path.basename(get_metadata_path(model_path))] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") deleted: List[str] = [] - main_path = os.path.join(target_dir, main_file).replace(os.sep, "/") + main_path = model_path.replace(os.sep, "/") if os.path.exists(main_path): os.remove(main_path) @@ -62,8 +65,8 @@ async def delete_model_artifacts( else: logger.warning("Model file not found: %s", main_file) - for pattern in patterns[1:]: - path = os.path.join(target_dir, pattern) + for pattern in patterns: + path = os.path.join(sidecar_dir, pattern) if os.path.exists(path): try: os.remove(path) @@ -389,17 +392,21 @@ class ModelLifecycleService: raise ValueError("A file with this name already exists") metadata_filename = os.path.basename(get_metadata_path(file_path)) - patterns = [ - f"{old_file_name}{old_extension}", - metadata_filename, - f"{metadata_filename}.bak", + # Sidecars/previews live in the sidecar dir (the model's own dir in + # alongside mode, the centralized mirror otherwise); the model file + # itself always stays in target_dir. + sidecar_dir = get_sidecar_dir(file_path) + patterns: List[tuple[str, str]] = [ + (target_dir, f"{old_file_name}{old_extension}"), + (sidecar_dir, metadata_filename), + (sidecar_dir, f"{metadata_filename}.bak"), ] for ext in PREVIEW_EXTENSIONS: - patterns.append(f"{old_file_name}{ext}") + patterns.append((sidecar_dir, f"{old_file_name}{ext}")) existing_files: List[tuple[str, str]] = [] - for pattern in patterns: - path = os.path.join(target_dir, pattern) + for pattern_dir, pattern in patterns: + path = os.path.join(pattern_dir, pattern) if os.path.exists(path): existing_files.append((path, pattern)) @@ -418,9 +425,9 @@ class ModelLifecycleService: for old_path, pattern in existing_files: ext = self._get_multipart_ext(pattern) - new_path = os.path.join(target_dir, f"{new_file_name}{ext}").replace( - os.sep, "/" - ) + new_path = os.path.join( + os.path.dirname(old_path), f"{new_file_name}{ext}" + ).replace(os.sep, "/") os.rename(old_path, new_path) renamed_files.append(new_path) @@ -437,9 +444,9 @@ class ModelLifecycleService: if metadata.get("preview_url"): old_preview = str(metadata["preview_url"]) ext = self._get_multipart_ext(old_preview) - new_preview = os.path.join(target_dir, f"{new_file_name}{ext}").replace( - os.sep, "/" - ) + new_preview = os.path.join( + get_preview_dir(new_file_path), f"{new_file_name}{ext}" + ).replace(os.sep, "/") metadata["preview_url"] = new_preview await self._metadata_manager.save_metadata(new_file_path, metadata) diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index a8525165..c884c9de 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -11,7 +11,13 @@ from ..utils.models import BaseModelMetadata, autov3_from_civitai_files from ..config import config from ..utils.file_utils import find_preview_file, get_preview_extension, calculate_sha256, calculate_autov3 from ..utils.metadata_manager import MetadataManager -from ..utils.sidecar_paths import get_metadata_path, get_preview_dir +from ..utils.sidecar_paths import ( + get_metadata_path, + get_preview_dir, + get_sidecar_dir, + is_centralized, + resolve_centralized_dir_for_dir, +) from ..utils.civitai_utils import resolve_license_info from .model_cache import ModelCache from .model_hash_index import ModelHashIndex @@ -1614,6 +1620,25 @@ class ModelScanner: old_abs_prefix = f"{str(previous_path).replace(chr(92), '/').rstrip('/')}/" new_abs_prefix = f"{str(new_path).replace(chr(92), '/').rstrip('/')}/" + # Centralized sidecar mode: sidecars/previews live in the mirror tree, + # not under the renamed model directory, so the mirror subtree must + # move too and mirror-prefixed preview URLs need their own rekey. + old_mirror_dir: Optional[str] = None + new_mirror_dir: Optional[str] = None + if is_centralized(): + old_mirror_dir = resolve_centralized_dir_for_dir(str(previous_path)) + new_mirror_dir = resolve_centralized_dir_for_dir(str(new_path)) + old_mirror_prefix = ( + f"{old_mirror_dir.replace(chr(92), '/').rstrip('/')}/" + if old_mirror_dir + else "" + ) + new_mirror_prefix = ( + f"{new_mirror_dir.replace(chr(92), '/').rstrip('/')}/" + if new_mirror_dir + else "" + ) + cache = self._cache if cache is None: return False @@ -1671,8 +1696,24 @@ class ModelScanner: item["preview_url"] = self._rekey_path( item["preview_url"], old_abs_prefix, new_abs_prefix ) + if old_mirror_prefix: + item["preview_url"] = self._rekey_path( + item["preview_url"], old_mirror_prefix, new_mirror_prefix + ) touched.append(item) + if old_mirror_dir and new_mirror_dir and os.path.isdir(old_mirror_dir): + try: + os.makedirs(os.path.dirname(new_mirror_dir), exist_ok=True) + shutil.move(old_mirror_dir, new_mirror_dir) + except Exception as exc: # pragma: no cover - defensive + logger.warning( + "Failed to move centralized sidecar mirror %s -> %s: %s", + old_mirror_dir, + new_mirror_dir, + exc, + ) + if touched: changed = True await self._rewrite_sidecar_paths(touched) @@ -1701,7 +1742,9 @@ class ModelScanner: async def _rewrite_sidecar_paths(self, entries: List[Dict[str, Any]]) -> None: """Point each model's metadata sidecar at its new location. - Sidecars travel with the renamed directory, so only the recorded + In alongside mode sidecars travel with the renamed directory; in + centralized mode the mirror subtree has already been moved by the + caller (:meth:`rename_known_folder`). Either way only the recorded ``file_path``/``preview_url`` inside them need rewriting. Failures are logged and skipped — a stale sidecar is repaired by the next metadata refresh, and must not abort the rename. @@ -1720,6 +1763,94 @@ class ModelScanner: "Failed to rewrite metadata sidecar %s: %s", metadata_path, exc ) + def _find_pending_models_in_sidecar_mirror(self) -> List[Dict[str, Any]]: + """Mirror-tree counterpart of the alongside pending-hash filesystem scan. + + Centralized mode stores ``.metadata.json`` sidecars in the mirror + tree, so walking the model folders finds nothing. Each mirror base is + resolved from a configured model root; a sidecar's recorded + ``file_path`` locates its model, with a stem-based probe under the + mapped model root as fallback (mirror path components are sanitized, + so reverse mapping is best-effort). Orphan sidecars whose model file + no longer exists are skipped, matching the alongside scan. + """ + + pending_models: List[Dict[str, Any]] = [] + + for root_path in self.get_model_roots(): + mirror_base = resolve_centralized_dir_for_dir(root_path) + if not mirror_base or not os.path.isdir(mirror_base): + continue + + for dirpath, dirnames, filenames in os.walk(mirror_base): + dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)] + for filename in filenames: + if not filename.endswith(".metadata.json"): + continue + + metadata_path = os.path.join(dirpath, filename) + try: + with open(metadata_path, "r", encoding="utf-8") as f: + data = json.load(f) + + # Check if hash is pending + hash_status = data.get("hash_status", "completed") + sha256 = data.get("sha256", "") + + if hash_status != "completed" or not sha256: + # Find corresponding model file: prefer the + # sidecar's recorded path, then probe by stem + # under the mapped model root. + model_path = None + recorded_path = data.get("file_path") + if ( + isinstance(recorded_path, str) + and recorded_path + and os.path.exists(recorded_path) + ): + model_path = recorded_path + else: + model_name = filename.replace(".metadata.json", "") + rel_dir = os.path.relpath(dirpath, mirror_base) + candidate_dir = ( + root_path + if rel_dir == os.curdir + else os.path.join(root_path, rel_dir) + ) + for ext in self.file_extensions: + potential_path = os.path.join( + candidate_dir, model_name + ext + ) + if os.path.exists(potential_path): + model_path = potential_path + break + + if model_path: + pending_models.append( + { + "file_path": model_path.replace(os.sep, "/"), + "hash_status": hash_status, + "sha256": sha256, + **{ + k: v + for k, v in data.items() + if k + not in [ + "file_path", + "hash_status", + "sha256", + ] + }, + } + ) + except (json.JSONDecodeError, Exception) as e: + logger.debug( + f"Error reading metadata file {metadata_path}: {e}" + ) + continue + + return pending_models + def _schedule_all_folders_backfill(self) -> None: """Kick off a one-shot background folder walk if none is running.""" if self._all_folders_backfill_running: @@ -1890,7 +2021,7 @@ class ModelScanner: file_info['name'] = os.path.basename(file_path) metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path) - metadata.preview_url = find_preview_file(local_stem, os.path.dirname(file_path)) + metadata.preview_url = find_preview_file(local_stem, get_preview_dir(file_path)) await MetadataManager.save_metadata(file_path, metadata) logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)") except Exception as e: @@ -2327,38 +2458,51 @@ class ModelScanner: # Move all associated files with the same base name source_metadata = None moved_metadata_path = None - - # Find all files with the same base name in the source directory + + # Associated files (sidecar metadata, previews) sit next to the + # model in alongside mode and in the mirror tree in centralized + # mode; collect from every directory that holds them. + source_sidecar_dir = get_sidecar_dir(source_path) + target_sidecar_dir = get_sidecar_dir(target_file) + associated_dirs = [(source_dir, target_path)] + if os.path.normpath(source_sidecar_dir) != os.path.normpath(source_dir): + associated_dirs.append((source_sidecar_dir, target_sidecar_dir)) + + # Find all files with the same base name in the source directories files_to_move = [] - try: - for file in os.listdir(source_dir): - if file.startswith(base_name + ".") and file != os.path.basename(source_path): - source_file_path = os.path.join(source_dir, file) - # Generate new filename with the same base name as the model file - file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png") - new_associated_filename = f"{final_base_name}{file_suffix}" - target_associated_path = os.path.join(target_path, new_associated_filename) - - # Store metadata file path for special handling - if file == os.path.basename(get_metadata_path(source_path)): - source_metadata = source_file_path - moved_metadata_path = target_associated_path - else: - files_to_move.append((source_file_path, target_associated_path)) - except Exception as e: - logger.error(f"Error listing files in {source_dir}: {e}") - + metadata_filename = os.path.basename(get_metadata_path(source_path)) + for assoc_source_dir, assoc_target_dir in associated_dirs: + try: + for file in os.listdir(assoc_source_dir): + if file.startswith(base_name + ".") and file != os.path.basename(source_path): + source_file_path = os.path.join(assoc_source_dir, file) + # Generate new filename with the same base name as the model file + file_suffix = file[len(base_name):] # Get the part after base_name (e.g., ".metadata.json", ".preview.png") + new_associated_filename = f"{final_base_name}{file_suffix}" + target_associated_path = os.path.join(assoc_target_dir, new_associated_filename) + + # Store metadata file path for special handling + if file == metadata_filename: + source_metadata = source_file_path + moved_metadata_path = target_associated_path + else: + files_to_move.append((source_file_path, target_associated_path)) + except Exception as e: + logger.error(f"Error listing files in {assoc_source_dir}: {e}") + # Move all associated files metadata = None for source_file, target_file_path in files_to_move: try: + os.makedirs(os.path.dirname(target_file_path), exist_ok=True) shutil.move(source_file, target_file_path) except Exception as e: logger.error(f"Error moving associated file {source_file}: {e}") - + # Handle metadata file specially to update paths if source_metadata and moved_metadata_path and os.path.exists(source_metadata): try: + os.makedirs(os.path.dirname(moved_metadata_path), exist_ok=True) shutil.move(source_metadata, moved_metadata_path) metadata = await self._update_metadata_paths(moved_metadata_path, target_file) except Exception as e: @@ -2826,7 +2970,7 @@ class ModelScanner: if not file_path: return None - dir_path = os.path.dirname(file_path) + dir_path = get_preview_dir(file_path) base_name = os.path.splitext(os.path.basename(file_path))[0] preview_path = find_preview_file(base_name, dir_path) if preview_path: diff --git a/py/services/other_scanner.py b/py/services/other_scanner.py index 0db2101a..b2a58ef4 100644 --- a/py/services/other_scanner.py +++ b/py/services/other_scanner.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional from ..utils.models import OtherModelMetadata from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3 from ..utils.metadata_manager import MetadataManager +from ..utils.sidecar_paths import get_preview_dir, is_centralized from ..config import config from .model_scanner import ModelScanner, _is_excluded_dir from .model_hash_index import ModelHashIndex @@ -72,10 +73,9 @@ class OtherScanner(ModelScanner): return None base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # AutoV3 reads only the safetensors header, so it is cheap even for # large files; record the checked state at creation time ("" = @@ -333,6 +333,11 @@ class OtherScanner(ModelScanner): async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]: """Scan filesystem for other-model metadata files with pending hash status.""" + # Centralized mode stores sidecars in the mirror tree, not next to the + # models; walk the mirror instead of the model folders. + if is_centralized(): + return self._find_pending_models_in_sidecar_mirror() + pending_models = [] for root_path in self.get_model_roots(): diff --git a/py/services/pending_delete_service.py b/py/services/pending_delete_service.py index 70bc0746..c57cf6e4 100644 --- a/py/services/pending_delete_service.py +++ b/py/services/pending_delete_service.py @@ -38,7 +38,7 @@ from typing import ( ) from ..utils.constants import PREVIEW_EXTENSIONS -from ..utils.sidecar_paths import get_metadata_path +from ..utils.sidecar_paths import get_metadata_path, get_sidecar_dir from ..utils import settings_paths logger = logging.getLogger(__name__) @@ -670,16 +670,22 @@ class PendingDeleteService: """Enumerate existing artifacts exactly like delete_model_artifacts.""" main_extension = ".safetensors" if main_extension is None else main_extension main_file = f"{file_name}{main_extension}" if main_extension else file_name - patterns = [ - main_file, - os.path.basename(get_metadata_path(os.path.join(target_dir, main_file))), - ] + model_path = os.path.join(target_dir, main_file) + + artifacts: List[str] = [] + main_path = os.path.abspath(model_path) + if os.path.exists(main_path): + artifacts.append(main_path) + + # Sidecars/previews live in the sidecar dir (the model's own dir in + # alongside mode, the centralized mirror otherwise). + sidecar_dir = get_sidecar_dir(model_path) + patterns = [os.path.basename(get_metadata_path(model_path))] for ext in PREVIEW_EXTENSIONS: patterns.append(f"{file_name}{ext}") - artifacts: List[str] = [] for pattern in patterns: - path = os.path.abspath(os.path.join(target_dir, pattern)) + path = os.path.abspath(os.path.join(sidecar_dir, pattern)) if os.path.exists(path): artifacts.append(path) return artifacts @@ -698,7 +704,9 @@ class PendingDeleteService: """ for original_path in artifacts: staged_path = os.path.join(batch_dir, os.path.basename(original_path)) - os.rename(original_path, staged_path) + # EXDEV-tolerant: centralized sidecars may live on a different + # filesystem than the staging batch dir under the model root. + self._restore_file(original_path, staged_path) staged_pairs.append( { "staged": os.path.abspath(staged_path), @@ -740,13 +748,14 @@ class PendingDeleteService: return staged_pairs def _restore_file(self, staged_path: str, original_path: str) -> None: - """Restore a staged file to its original path, tolerating EXDEV. + """Move a file between staging and library paths, tolerating EXDEV. ``os.rename`` is atomic and preferred (model staging and most recipe restores are same-volume). Recipe staging copies into the settings-dir - staging parent, which may live on a DIFFERENT filesystem than the - recipes dir; rename then raises EXDEV. Fall back to ``shutil.copy2`` + - ``os.remove`` so the bytes are restored and the staged copy removed. + staging parent, and centralized sidecars live under the configured + sidecar root; both may live on a DIFFERENT filesystem than the target + dir, so rename can raise EXDEV. Fall back to ``shutil.copy2`` + + ``os.remove`` so the bytes are moved and the source copy removed. """ try: os.rename(staged_path, original_path) diff --git a/py/services/preview_asset_service.py b/py/services/preview_asset_service.py index 1e215347..a6ada6a3 100644 --- a/py/services/preview_asset_service.py +++ b/py/services/preview_asset_service.py @@ -64,6 +64,9 @@ class PreviewAssetService: base_name = os.path.splitext(os.path.splitext(os.path.basename(metadata_path))[0])[0] preview_dir = os.path.dirname(metadata_path) + # Centralized mirrors may not exist yet (unlike the model's own + # directory in alongside mode). + os.makedirs(preview_dir, exist_ok=True) is_video = first_preview.get("type") == "video" preview_url = first_preview.get("url") @@ -161,6 +164,9 @@ class PreviewAssetService: base_name = os.path.splitext(os.path.basename(model_path))[0] folder = get_preview_dir(model_path) + # Centralized mirrors may not exist yet (unlike the model's own + # directory in alongside mode). + os.makedirs(folder, exist_ok=True) extension, optimized_data = await self._convert_preview( preview_data, content_type, original_filename diff --git a/py/services/settings_manager.py b/py/services/settings_manager.py index 4f25ef0e..0e874776 100644 --- a/py/services/settings_manager.py +++ b/py/services/settings_manager.py @@ -99,6 +99,8 @@ DEFAULT_SETTINGS: Dict[str, Any] = { "enable_other_models": False, "enabled_other_sub_types": list(DEFAULT_ENABLED_OTHER_SUB_TYPES), "recipes_path": "", + "sidecar_storage_mode": "alongside", + "sidecar_storage_path": "", "base_model_path_mappings": {}, "download_path_templates": {}, "download_filename_templates": {}, @@ -1616,9 +1618,30 @@ class SettingsManager: return os.path.abspath(os.path.normpath(os.path.expanduser(stripped))) + @staticmethod + def _normalize_sidecar_storage_mode(value: Any) -> str: + """Return a valid sidecar storage mode, falling back to ``alongside``.""" + + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in ("alongside", "centralized"): + return normalized + return "alongside" + + def _refresh_sidecar_storage_config(self) -> None: + """Rebuild dependent config state after sidecar storage settings change.""" + + try: + from ..config import config # Local import to avoid circular dependency + + config.refresh_preview_roots() + except Exception as exc: # pragma: no cover - defensive logging + logger.debug( + "Failed to refresh config after sidecar storage change: %s", exc + ) + def _get_effective_recipes_dir(self, recipes_path: Optional[str] = None) -> str: """Resolve the effective recipes directory for the active library.""" - normalized_custom = self._normalize_recipes_path_value( self.settings.get("recipes_path", "") if recipes_path is None @@ -1815,6 +1838,10 @@ class SettingsManager: target_recipes_dir = self._get_effective_recipes_dir(value) self._validate_recipes_storage_path(target_recipes_dir) self._migrate_recipes_directory(current_recipes_dir, target_recipes_dir) + elif key == "sidecar_storage_mode": + value = self._normalize_sidecar_storage_mode(value) + elif key == "sidecar_storage_path": + value = self._normalize_recipes_path_value(value) self.settings[key] = value portable_switch_pending = False if key == "use_portable_settings" and isinstance(value, bool): @@ -1845,6 +1872,8 @@ class SettingsManager: self._save_settings() if key == "recipes_path": self._notify_library_change(self.get_active_library_name()) + if key in ("sidecar_storage_mode", "sidecar_storage_path"): + self._refresh_sidecar_storage_config() if key in ("enable_other_models", "enabled_other_sub_types"): self._apply_other_model_settings_change() if portable_switch_pending: diff --git a/py/services/use_cases/__init__.py b/py/services/use_cases/__init__.py index ae6a6900..02756173 100644 --- a/py/services/use_cases/__init__.py +++ b/py/services/use_cases/__init__.py @@ -21,6 +21,10 @@ from .example_images import ( ImportExampleImagesValidationError, ) from .filename_template_use_case import FilenameTemplateUseCase +from .sidecar_migration_use_case import ( + SidecarMigrationProgressReporter, + SidecarMigrationUseCase, +) __all__ = [ "AutoOrganizeInProgressError", @@ -36,4 +40,6 @@ __all__ = [ "ImportExampleImagesUseCase", "ImportExampleImagesValidationError", "FilenameTemplateUseCase", + "SidecarMigrationProgressReporter", + "SidecarMigrationUseCase", ] diff --git a/py/services/use_cases/sidecar_migration_use_case.py b/py/services/use_cases/sidecar_migration_use_case.py new file mode 100644 index 00000000..9f50b6ff --- /dev/null +++ b/py/services/use_cases/sidecar_migration_use_case.py @@ -0,0 +1,418 @@ +"""Use case migrating sidecar metadata and previews between storage layouts. + +Two storage layouts exist (see :mod:`py.utils.sidecar_paths`): + +- ``alongside``: ``/.metadata.json`` and preview files live + next to the model file. +- ``centralized``: the same files live under the configured sidecar root, + mirroring the library-relative directory structure. + +This use case moves the ``.metadata.json`` sidecar and preview files for every +known model from one layout to the other. Model files themselves NEVER move. +Paths inside the moved sidecar (``file_path``, ``file_name``, ``preview_url``) +are rewritten the same way :meth:`ModelScanner._update_metadata_paths` does. + +Intended flow (settings-first): + +1. The user switches ``sidecar_storage_mode`` (and optionally + ``sidecar_storage_path``) in settings. +2. The migration runs in the direction of the NEW mode with ``force=True``. + After the switch, files in the OLD layout are the source of truth; the + guard below would otherwise refuse to run because the active mode already + matches the migration target. + +Both orderings work because all path computations are mode-independent: the +alongside location is derived from the model path directly, and the mirror +location is resolved via ``get_configured_sidecar_root()``, which ignores the +active mode. + +Guards (pass ``force=True`` to bypass): + +- ``migrate_to_centralized`` refuses when centralized storage is already the + active, resolvable mode. +- ``migrate_to_alongside`` refuses when the active mode is ``alongside``. +""" + +from __future__ import annotations + +import errno +import json +import logging +import os +import shutil +from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Sequence, Tuple + +from ..service_registry import ServiceRegistry +from ..settings_manager import get_settings_manager +from ...utils.constants import PREVIEW_EXTENSIONS +from ...utils.file_utils import get_preview_extension +from ...utils.metadata_manager import MetadataManager +from ...utils.sidecar_paths import ( + METADATA_SUFFIX, + STORAGE_MODE_CENTRALIZED, + get_configured_sidecar_root, + get_sidecar_root, + get_storage_mode, + resolve_centralized_dir_for_dir, +) + + +class SidecarMigrationProgressReporter(Protocol): + """Protocol for progress reporters used during sidecar migration.""" + + async def on_progress(self, payload: Dict[str, Any]) -> None: + """Handle a sidecar migration progress update.""" + + +ScannerFactory = Callable[[], Awaitable[Any]] + +DIRECTION_TO_CENTRALIZED = "to_centralized" +DIRECTION_TO_ALONGSIDE = "to_alongside" + + +class SidecarMigrationUseCase: + """Move sidecars and previews between alongside and centralized layouts.""" + + def __init__( + self, + *, + scanner_factories: Sequence[Tuple[str, ScannerFactory]] | None = None, + settings_service=None, + logger: Optional[logging.Logger] = None, + ) -> None: + self._settings = settings_service or get_settings_manager() + self._scanner_factories: Tuple[Tuple[str, ScannerFactory], ...] = tuple( + scanner_factories + or ( + ("lora", ServiceRegistry.get_lora_scanner), + ("checkpoint", ServiceRegistry.get_checkpoint_scanner), + ("embedding", ServiceRegistry.get_embedding_scanner), + ("other", ServiceRegistry.get_other_scanner), + ) + ) + self._logger = logger or logging.getLogger(__name__) + + async def migrate_to_centralized( + self, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Move sidecars/previews from alongside the models into the mirror root.""" + + if ( + not force + and get_storage_mode() == STORAGE_MODE_CENTRALIZED + and get_sidecar_root() + ): + return self._refusal( + DIRECTION_TO_CENTRALIZED, + "sidecar storage is already centralized; pass force=true to migrate anyway", + ) + return await self._migrate( + direction=DIRECTION_TO_CENTRALIZED, + to_centralized=True, + progress_cb=progress_cb, + ) + + async def migrate_to_alongside( + self, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + *, + force: bool = False, + ) -> Dict[str, Any]: + """Move sidecars/previews from the mirror root back next to the models.""" + + if not force and get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return self._refusal( + DIRECTION_TO_ALONGSIDE, + "sidecar storage is already alongside; pass force=true to migrate anyway", + ) + return await self._migrate( + direction=DIRECTION_TO_ALONGSIDE, + to_centralized=False, + progress_cb=progress_cb, + ) + + @staticmethod + def _refusal(direction: str, message: str) -> Dict[str, Any]: + return { + "success": False, + "error": message, + "direction": direction, + "models_total": 0, + "models_processed": 0, + "models_moved": 0, + "moved": 0, + "skipped": 0, + "conflicts": 0, + "errors": [], + "error_count": 0, + } + + def _active_scanner_factories(self) -> Tuple[Tuple[str, ScannerFactory], ...]: + """Drop the opt-in other scanner while Other Models is disabled.""" + + if self._settings.is_other_models_enabled(): + return self._scanner_factories + return tuple(entry for entry in self._scanner_factories if entry[0] != "other") + + async def _collect_model_paths(self, errors: List[Dict[str, str]]) -> List[str]: + """Enumerate model file paths across every active scanner's cache.""" + + paths: List[str] = [] + for model_type, factory in self._active_scanner_factories(): + try: + scanner = await factory() + cache = await scanner.get_cached_data() + except Exception as exc: + self._logger.error( + "Sidecar migration: failed to enumerate %s models: %s", + model_type, + exc, + ) + errors.append({"model": model_type, "error": f"enumeration failed: {exc}"}) + continue + for entry in cache.raw_data: + file_path = entry.get("file_path") + if file_path: + paths.append(file_path) + return paths + + @staticmethod + def _move_file(src: str, dst: str) -> None: + """Move a file, tolerating EXDEV when the layouts span filesystems.""" + + os.makedirs(os.path.dirname(dst), exist_ok=True) + try: + os.rename(src, dst) + except OSError as exc: + if exc.errno != errno.EXDEV: + raise + shutil.copy2(src, dst) + os.remove(src) + + async def _migrate( + self, + *, + direction: str, + to_centralized: bool, + progress_cb: Optional[SidecarMigrationProgressReporter], + ) -> Dict[str, Any]: + root = get_configured_sidecar_root() + if not root: + return self._refusal( + direction, + "cannot resolve the centralized sidecar root", + ) + + errors: List[Dict[str, str]] = [] + model_paths = await self._collect_model_paths(errors) + + total = len(model_paths) + processed = 0 + models_moved = 0 + moved = 0 + skipped = 0 + conflicts = 0 + + async def emit(status: str, **extra: Any) -> None: + if progress_cb is None: + return + payload: Dict[str, Any] = { + "type": "sidecar_migration_progress", + "status": status, + "direction": direction, + "total": total, + "processed": processed, + "moved": moved, + "skipped": skipped, + "conflicts": conflicts, + "errors": len(errors), + } + payload.update(extra) + await progress_cb.on_progress(payload) + + await emit("started") + + for model_path in model_paths: + processed += 1 + current = os.path.basename(model_path) + try: + result = await self._migrate_model( + model_path, + root=root, + to_centralized=to_centralized, + ) + moved += result["moved"] + conflicts += result["conflicts"] + if result["skipped"]: + skipped += 1 + if result["moved"]: + models_moved += 1 + except Exception as exc: + self._logger.error( + "Sidecar migration failed for %s: %s", model_path, exc, exc_info=True + ) + errors.append({"model": current, "error": str(exc)}) + await emit("processing", current=current) + + await emit("completed") + + return { + "success": not errors, + "direction": direction, + "models_total": total, + "models_processed": processed, + "models_moved": models_moved, + "moved": moved, + "skipped": skipped, + "conflicts": conflicts, + "errors": errors, + "error_count": len(errors), + } + + async def _migrate_model( + self, + model_path: str, + *, + root: str, + to_centralized: bool, + ) -> Dict[str, int]: + """Migrate one model's sidecar + previews; return per-model counters.""" + + result = {"moved": 0, "conflicts": 0, "skipped": 0} + + model_path = os.path.abspath(model_path) + if not os.path.exists(model_path): + self._logger.warning( + "Sidecar migration: model file missing, skipping: %s", model_path + ) + result["skipped"] = 1 + return result + + model_dir = os.path.dirname(model_path) + mirror_dir = resolve_centralized_dir_for_dir(model_dir, sidecar_root=root) + if mirror_dir is None: + self._logger.warning( + "Sidecar migration: %s is outside configured model roots, skipping", + model_path, + ) + result["skipped"] = 1 + return result + + if to_centralized: + src_dir, dst_dir = model_dir, mirror_dir + else: + src_dir, dst_dir = mirror_dir, model_dir + + if os.path.normpath(src_dir) == os.path.normpath(dst_dir): + result["skipped"] = 1 + return result + + stem = os.path.splitext(os.path.basename(model_path))[0] + sidecar_name = stem + METADATA_SUFFIX + + moved_previews: List[str] = [] + for ext in PREVIEW_EXTENSIONS: + src = os.path.join(src_dir, stem + ext) + if not os.path.exists(src): + continue + dst = os.path.join(dst_dir, stem + ext) + if self._transfer(src, dst, result): + moved_previews.append(dst) + + sidecar_src = os.path.join(src_dir, sidecar_name) + sidecar_moved = False + sidecar_dst = os.path.join(dst_dir, sidecar_name) + if os.path.exists(sidecar_src): + sidecar_moved = self._transfer(sidecar_src, sidecar_dst, result) + + if sidecar_moved: + await self._rewrite_sidecar_paths(sidecar_dst, model_path, moved_previews) + + return result + + def _transfer(self, src: str, dst: str, result: Dict[str, int]) -> bool: + """Move ``src`` to ``dst`` with keep-newer conflict resolution. + + Returns True when the file was actually moved to the destination. On a + conflict the newer file wins: a newer source replaces the destination; + a newer (or equal) destination is kept and the source is deleted. + """ + + if os.path.exists(dst): + result["conflicts"] += 1 + if os.path.getmtime(src) > os.path.getmtime(dst): + self._logger.info( + "Sidecar migration: conflict at %s; source is newer, replacing", dst + ) + os.remove(dst) + else: + self._logger.info( + "Sidecar migration: conflict at %s; destination is newer, keeping it", + dst, + ) + os.remove(src) + return False + self._move_file(src, dst) + result["moved"] += 1 + return True + + async def _rewrite_sidecar_paths( + self, + sidecar_path: str, + model_path: str, + moved_previews: List[str], + ) -> None: + """Update path fields inside a moved sidecar, mirroring ModelScanner.""" + + with open(sidecar_path, "r", encoding="utf-8") as handle: + metadata = json.load(handle) + + stem = os.path.splitext(os.path.basename(model_path))[0] + metadata["file_path"] = model_path.replace(os.sep, "/") + metadata["file_name"] = stem + + if moved_previews and metadata.get("preview_url"): + recorded_ext = get_preview_extension(metadata["preview_url"]) + chosen = next( + ( + path + for path in moved_previews + if get_preview_extension(path) == recorded_ext + ), + moved_previews[0], + ) + metadata["preview_url"] = chosen.replace(os.sep, "/") + + await MetadataManager.save_metadata(sidecar_path, metadata) + + async def execute_with_error_handling( + self, + *, + direction: str, + progress_cb: Optional[SidecarMigrationProgressReporter] = None, + force: bool = False, + ) -> Dict[str, Any]: + """Wrapper providing progress notification on unexpected failures.""" + + try: + if direction == DIRECTION_TO_CENTRALIZED: + return await self.migrate_to_centralized(progress_cb, force=force) + if direction == DIRECTION_TO_ALONGSIDE: + return await self.migrate_to_alongside(progress_cb, force=force) + raise ValueError( + f"direction must be {DIRECTION_TO_CENTRALIZED!r} or {DIRECTION_TO_ALONGSIDE!r}" + ) + except Exception as exc: + if progress_cb is not None: + await progress_cb.on_progress( + { + "type": "sidecar_migration_progress", + "status": "error", + "direction": direction, + "error": str(exc), + } + ) + raise diff --git a/py/utils/metadata_manager.py b/py/utils/metadata_manager.py index 1b03a813..dbd38244 100644 --- a/py/utils/metadata_manager.py +++ b/py/utils/metadata_manager.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Optional, Type, Union, cast from .models import BaseModelMetadata, CheckpointMetadata, EmbeddingMetadata, LoraMetadata from .file_utils import normalize_path, find_preview_file, calculate_sha256, calculate_autov3 from .lora_metadata import extract_lora_metadata, extract_checkpoint_metadata -from .sidecar_paths import get_metadata_path, resolve_metadata_path +from .sidecar_paths import get_metadata_path, get_preview_dir, resolve_metadata_path logger = logging.getLogger(__name__) @@ -189,6 +189,10 @@ class MetadataManager: temp_path = f"{metadata_path}.tmp" try: + # Centralized sidecar mirrors may not exist yet (unlike the model's + # own directory in alongside mode, which always does). + os.makedirs(os.path.dirname(metadata_path), exist_ok=True) + # Convert to dict if needed if isinstance(metadata, BaseModelMetadata): metadata_dict = metadata.to_dict() @@ -251,10 +255,9 @@ class MetadataManager: try: base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) - + # Find preview image - preview_url = find_preview_file(base_name, dir_path) + preview_url = find_preview_file(base_name, get_preview_dir(file_path)) # Calculate file hash start_hash_time = time.perf_counter() @@ -378,15 +381,16 @@ class MetadataManager: # Check if preview exists at the current location preview_url = metadata.preview_url if preview_url: - # Get directory parts of both paths - file_dir = os.path.dirname(file_path) + # Get directory parts of both paths; the preview directory is the + # sidecar/preview dir (the model's own dir in alongside mode, the + # centralized mirror otherwise). + file_dir = get_preview_dir(file_path) preview_dir = os.path.dirname(preview_url) # Update preview if it doesn't exist OR if model and preview are in different directories if not os.path.exists(preview_url) or file_dir != preview_dir: base_name = os.path.splitext(os.path.basename(file_path))[0] - dir_path = os.path.dirname(file_path) - new_preview_url = find_preview_file(base_name, dir_path) + new_preview_url = find_preview_file(base_name, file_dir) if new_preview_url: metadata.preview_url = normalize_path(new_preview_url) need_update = True diff --git a/py/utils/sidecar_paths.py b/py/utils/sidecar_paths.py index 05d89e2e..d43f4b54 100644 --- a/py/utils/sidecar_paths.py +++ b/py/utils/sidecar_paths.py @@ -64,15 +64,8 @@ def is_centralized() -> bool: return get_storage_mode() == STORAGE_MODE_CENTRALIZED and bool(get_sidecar_root()) -def get_sidecar_root() -> str: - """Return the absolute root directory for centralized sidecar storage. - - Empty string when centralized storage is not usable (mode alongside or an - unresolvable configured path). - """ - - if get_storage_mode() != STORAGE_MODE_CENTRALIZED: - return "" +def _resolve_root_from_settings() -> str: + """Resolve the configured/default centralized root, ignoring the active mode.""" configured = _get_settings_value("sidecar_storage_path", "") if configured and isinstance(configured, str): @@ -90,6 +83,31 @@ def get_sidecar_root() -> str: return "" +def get_sidecar_root() -> str: + """Return the absolute root directory for centralized sidecar storage. + + Empty string when centralized storage is not usable (mode alongside or an + unresolvable configured path). + """ + + if get_storage_mode() != STORAGE_MODE_CENTRALIZED: + return "" + + return _resolve_root_from_settings() + + +def get_configured_sidecar_root() -> str: + """Return the centralized sidecar root regardless of the active mode. + + Unlike :func:`get_sidecar_root`, this resolves the configured + ``sidecar_storage_path`` (or the ``/sidecars`` default) even + when the storage mode is ``alongside``. Migration tooling needs both + layouts at once and must not depend on which mode is currently active. + """ + + return _resolve_root_from_settings() + + def sanitize_path_component(name: str) -> str: """Return a filesystem-safe single path component.""" @@ -137,19 +155,38 @@ def resolve_centralized_dir(model_path: str) -> Optional[str]: or the path is not under any configured model root. """ - root = get_sidecar_root() + return resolve_centralized_dir_for_dir( + os.path.dirname(_normalize_for_match(model_path)) + ) + + +def resolve_centralized_dir_for_dir( + model_dir: str, *, sidecar_root: Optional[str] = None +) -> Optional[str]: + """Return the centralized mirror directory for a model *directory*. + + Same layout as :func:`resolve_centralized_dir`, but accepts the directory + itself. Used by folder-level operations (folder rename, mirror-tree walks) + that have no model file path to derive from. Passing a configured model + root returns the mirror base for that root. + + ``sidecar_root`` overrides the root lookup; pass + :func:`get_configured_sidecar_root` to resolve mirror paths independently + of the active storage mode (migration tooling). + """ + + root = sidecar_root if sidecar_root is not None else get_sidecar_root() if not root: return None - target = _normalize_for_match(model_path) - model_dir = os.path.dirname(target) + normalized_dir = _normalize_for_match(model_dir) best_root: Optional[str] = None for candidate in _iter_model_roots(): if not candidate: continue normalized = _normalize_for_match(candidate) - if model_dir == normalized or model_dir.startswith(normalized + os.sep): + if normalized_dir == normalized or normalized_dir.startswith(normalized + os.sep): if best_root is None or len(normalized) > len(best_root): best_root = normalized @@ -163,7 +200,7 @@ def resolve_centralized_dir(model_path: str) -> Optional[str]: except Exception: # pragma: no cover - defensive fallback library = "default" - rel_dir = os.path.relpath(model_dir, best_root) + rel_dir = os.path.relpath(normalized_dir, best_root) parts = [root, sanitize_path_component(library), sanitize_path_component(os.path.basename(best_root))] if rel_dir and rel_dir != os.curdir: parts.extend(sanitize_path_component(part) for part in rel_dir.split(os.sep) if part not in ("", os.curdir)) diff --git a/static/js/managers/SettingsManager.js b/static/js/managers/SettingsManager.js index 7ca0d22e..eb60c202 100644 --- a/static/js/managers/SettingsManager.js +++ b/static/js/managers/SettingsManager.js @@ -1192,6 +1192,9 @@ export class SettingsManager { this.updateExampleImagesOpenSettingsVisibility(); + // Load sidecar storage settings + this.loadSidecarStorageSettings(); + // Load download path templates this.loadDownloadPathTemplates(); @@ -1280,6 +1283,9 @@ export class SettingsManager { this.attachPathField('exampleImagesLocalRoot', { onAfterSelect: () => this.saveInputSetting('exampleImagesLocalRoot', 'example_images_local_root'), }); + this.attachPathField('sidecarStoragePath', { + onAfterSelect: () => this.saveInputSetting('sidecarStoragePath', 'sidecar_storage_path'), + }); } loadDownloadBackendSettings() { @@ -3379,6 +3385,192 @@ export class SettingsManager { this.updateExampleImagesOpenSettingsVisibility(); } + loadSidecarStorageSettings() { + const currentMode = state.global.settings.sidecar_storage_mode === 'centralized' + ? 'centralized' + : 'alongside'; + + const modeSelect = document.getElementById('sidecarStorageMode'); + if (modeSelect) { + modeSelect.value = currentMode; + } + // Baseline used to detect a mode change in handleSidecarStorageModeChange + this._loadedSidecarStorageMode = currentMode; + + const pathInput = document.getElementById('sidecarStoragePath'); + if (pathInput) { + pathInput.value = state.global.settings.sidecar_storage_path || ''; + } + + this.updateSidecarStorageVisibility(); + } + + updateSidecarStorageVisibility() { + const modeSelect = document.getElementById('sidecarStorageMode'); + const pathSetting = document.getElementById('sidecarStoragePathSetting'); + if (!pathSetting) return; + + const mode = modeSelect ? modeSelect.value : state.global.settings.sidecar_storage_mode; + pathSetting.style.display = mode === 'centralized' ? 'block' : 'none'; + } + + async handleSidecarStorageModeChange() { + const modeSelect = document.getElementById('sidecarStorageMode'); + if (!modeSelect) return; + + const previousMode = this._loadedSidecarStorageMode || 'alongside'; + + await this.saveSelectSetting('sidecarStorageMode', 'sidecar_storage_mode'); + this.updateSidecarStorageVisibility(); + + const newMode = modeSelect.value; + this._loadedSidecarStorageMode = newMode; + + // Existing sidecars are not moved automatically; offer to migrate them. + if (newMode !== previousMode) { + const direction = newMode === 'centralized' ? 'to_centralized' : 'to_alongside'; + const confirmed = await this.confirmSidecarMigration(direction); + if (confirmed) { + await this.migrateSidecars(direction); + } else { + showToast('settings.sidecarStorage.migrationDeferred', {}, 'info'); + } + } + } + + // Entry point for the "Migrate Sidecars Now" button: the direction follows + // the currently saved storage mode. + async confirmAndMigrateSidecars() { + const direction = state.global.settings.sidecar_storage_mode === 'centralized' + ? 'to_centralized' + : 'to_alongside'; + const confirmed = await this.confirmSidecarMigration(direction); + if (confirmed) { + await this.migrateSidecars(direction); + } + } + + confirmSidecarMigration(direction) { + const modalElement = document.getElementById('sidecarMigrationConfirmModal'); + if (!modalElement) { + return Promise.resolve(false); + } + + const isToCentralized = direction === 'to_centralized'; + + const titleElement = modalElement.querySelector('[data-role="title"]'); + if (titleElement) { + titleElement.textContent = isToCentralized + ? translate('modals.sidecarMigrationConfirm.titleToCentralized', {}, 'Move sidecars to centralized storage?') + : translate('modals.sidecarMigrationConfirm.titleToAlongside', {}, 'Move sidecars back next to model files?'); + } + + const messageElement = modalElement.querySelector('[data-role="message"]'); + if (messageElement) { + messageElement.textContent = isToCentralized + ? translate('settings.sidecarStorage.confirmToCentralized', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them into the centralized storage directory now? You can also do this later with the "Migrate Sidecars Now" button.') + : translate('settings.sidecarStorage.confirmToAlongside', {}, 'The storage mode changed, but existing .metadata.json sidecars and preview images are not moved automatically. Move them back next to their model files now? You can also do this later with the "Migrate Sidecars Now" button.'); + } + + const confirmButton = modalElement.querySelector('[data-action="confirm-sidecar-migration"]'); + const cancelButton = modalElement.querySelector('[data-action="cancel-sidecar-migration"]'); + if (!confirmButton || !cancelButton) { + return Promise.resolve(false); + } + + confirmButton.textContent = translate('modals.sidecarMigrationConfirm.confirmButton', {}, 'Migrate Now'); + + return new Promise((resolve) => { + let resolved = false; + + const cleanup = () => { + confirmButton.removeEventListener('click', handleConfirm); + cancelButton.removeEventListener('click', handleCancel); + document.removeEventListener('keydown', handleEscape, true); + }; + + const finalize = (proceed) => { + if (resolved) { + return; + } + resolved = true; + cleanup(); + modalElement.classList.remove('show'); + // Keep body.modal-open: the settings modal underneath is still open. + resolve(proceed); + }; + + const handleConfirm = (event) => { + event.preventDefault(); + finalize(true); + }; + + const handleCancel = (event) => { + event.preventDefault(); + finalize(false); + }; + + // Capture phase + stopPropagation so ESC never reaches the + // settings modal's own ESC handler underneath. + const handleEscape = (event) => { + if (event.key === 'Escape') { + event.stopPropagation(); + finalize(false); + } + }; + + confirmButton.addEventListener('click', handleConfirm); + cancelButton.addEventListener('click', handleCancel); + document.addEventListener('keydown', handleEscape, true); + + modalElement.classList.add('show'); + cancelButton.focus(); + }); + } + + async migrateSidecars(direction) { + const migrateBtn = document.getElementById('migrateSidecarsBtn'); + try { + if (migrateBtn) { + migrateBtn.disabled = true; + migrateBtn.textContent = translate('settings.sidecarStorage.migratingButton', {}, 'Migrating...'); + } + + state.loadingManager?.showSimpleLoading( + translate('settings.sidecarStorage.migrating', {}, 'Migrating sidecars...') + ); + + const response = await fetch('/api/lm/sidecars/migrate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + // The new mode is already saved by the time migration runs, + // so the backend guard requires force=true to confirm the + // "switch first, then migrate" flow. + body: JSON.stringify({ direction, force: true }), + }); + + const data = await response.json(); + if (!response.ok || data.success === false) { + throw new Error(data.error || 'Migration failed'); + } + + state.loadingManager?.hide(); + showToast('settings.sidecarStorage.migrateSuccess', {}, 'success'); + + // Reload so cards pick up metadata/preview paths from the new location + resetAndReload(true); + } catch (error) { + console.error('Error migrating sidecars:', error); + state.loadingManager?.hide(); + showToast('settings.sidecarStorage.migrateFailed', { message: error.message }, 'error'); + } finally { + if (migrateBtn) { + migrateBtn.disabled = false; + migrateBtn.textContent = translate('settings.sidecarStorage.migrateButton', {}, 'Migrate Sidecars Now'); + } + } + } + async loadMetadataArchiveSettings() { try { // Load current settings from state diff --git a/static/js/state/index.js b/static/js/state/index.js index 20eabd7c..f50e5ac8 100644 --- a/static/js/state/index.js +++ b/static/js/state/index.js @@ -62,6 +62,8 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({ download_skip_base_models: [], backup_auto_enabled: true, backup_retention_count: 5, + sidecar_storage_mode: 'alongside', + sidecar_storage_path: '', strip_lora_on_copy: false, use_new_license_icons: true, group_by_model: false, diff --git a/templates/components/modals/confirm_modals.html b/templates/components/modals/confirm_modals.html index cb114da2..6ca24b9d 100644 --- a/templates/components/modals/confirm_modals.html +++ b/templates/components/modals/confirm_modals.html @@ -96,6 +96,20 @@ + + + diff --git a/templates/components/modals/settings/library.html b/templates/components/modals/settings/library.html index 362c9324..89012f89 100644 --- a/templates/components/modals/settings/library.html +++ b/templates/components/modals/settings/library.html @@ -321,4 +321,60 @@ ('civitai_sqlite_archive', 'settings.metadataArchive.providerOrderCivitaiSqliteArchive'), ], 'settings.metadataArchive.providerOrderHelp') }} + + +
+ {{ sm.subsection_header('settings.sections.sidecarStorage') }} +
+
+
+ +
+
+ +
+
+
+ + + +
+
+
+ +
+
+ +
+
+
+
diff --git a/tests/frontend/managers/settingsManager.sidecarStorage.test.js b/tests/frontend/managers/settingsManager.sidecarStorage.test.js new file mode 100644 index 00000000..ae376076 --- /dev/null +++ b/tests/frontend/managers/settingsManager.sidecarStorage.test.js @@ -0,0 +1,284 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +vi.mock('../../../static/js/managers/ModalManager.js', () => ({ + modalManager: { + closeModal: vi.fn(), + }, +})); + +vi.mock('../../../static/js/utils/uiHelpers.js', () => ({ + showToast: vi.fn(), +})); + +vi.mock('../../../static/js/state/index.js', () => { + const settings = {}; + return { + state: { + global: { + settings, + }, + loadingManager: { + showSimpleLoading: vi.fn(), + hide: vi.fn(), + }, + }, + createDefaultSettings: () => ({ + language: 'en', + sidecar_storage_mode: 'alongside', + sidecar_storage_path: '', + }), + }; +}); + +vi.mock('../../../static/js/api/modelApiFactory.js', () => ({ + resetAndReload: vi.fn(), + getModelApiClient: vi.fn(), +})); + +vi.mock('../../../static/js/utils/constants.js', () => ({ + DOWNLOAD_PATH_TEMPLATES: {}, + DEFAULT_PATH_TEMPLATES: {}, + MAPPABLE_BASE_MODELS: [], + PATH_TEMPLATE_PLACEHOLDERS: {}, + FILENAME_TEMPLATE_PLACEHOLDERS: [], + DEFAULT_FILENAME_TEMPLATES: { lora: '', checkpoint: '', embedding: '' }, + DEFAULT_PRIORITY_TAG_CONFIG: { + lora: 'character, style', + checkpoint: 'base, guide', + embedding: 'hint', + }, + getMappableBaseModelsDynamic: () => [], +})); + +vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({ + translate: (_key, _params, fallback) => fallback ?? '', +})); + +vi.mock('../../../static/js/i18n/index.js', () => ({ + i18n: { + getCurrentLocale: () => 'en', + setLanguage: vi.fn().mockResolvedValue(), + }, +})); + +vi.mock('../../../static/js/components/shared/ModelCard.js', () => ({ + configureModelCardVideo: vi.fn(), +})); + +import { SettingsManager } from '../../../static/js/managers/SettingsManager.js'; +import { showToast } from '../../../static/js/utils/uiHelpers.js'; +import { resetAndReload } from '../../../static/js/api/modelApiFactory.js'; +import { state } from '../../../static/js/state/index.js'; + +const createManager = () => { + state.global.settings = {}; + const initSettingsSpy = vi + .spyOn(SettingsManager.prototype, 'initializeSettings') + .mockResolvedValue(); + const initializeSpy = vi + .spyOn(SettingsManager.prototype, 'initialize') + .mockImplementation(() => {}); + + const manager = new SettingsManager(); + + initSettingsSpy.mockRestore(); + initializeSpy.mockRestore(); + + return manager; +}; + +const appendSidecarControls = () => { + const select = document.createElement('select'); + select.id = 'sidecarStorageMode'; + ['alongside', 'centralized'].forEach((value) => { + const option = document.createElement('option'); + option.value = value; + select.appendChild(option); + }); + + const pathSetting = document.createElement('div'); + pathSetting.id = 'sidecarStoragePathSetting'; + pathSetting.style.display = 'none'; + + const pathInput = document.createElement('input'); + pathInput.id = 'sidecarStoragePath'; + + const migrateBtn = document.createElement('button'); + migrateBtn.id = 'migrateSidecarsBtn'; + + document.body.append(select, pathSetting, pathInput, migrateBtn); + return { select, pathSetting, pathInput, migrateBtn }; +}; + +const appendMigrationModal = () => { + const modal = document.createElement('div'); + modal.id = 'sidecarMigrationConfirmModal'; + modal.innerHTML = ` +

+

+ + `; + document.body.appendChild(modal); + return modal; +}; + +const mockFetchOk = (payload = { success: true }) => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(payload), + }); +}; + +beforeEach(() => { + document.body.innerHTML = ''; + vi.clearAllMocks(); +}); + +afterEach(() => { + delete global.fetch; +}); + +describe('SettingsManager sidecar storage', () => { + describe('loadSidecarStorageSettings', () => { + it('loads alongside mode and hides the centralized path input', () => { + const manager = createManager(); + const { select, pathSetting, pathInput } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'alongside', sidecar_storage_path: '/data/sidecars' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('alongside'); + expect(pathInput.value).toBe('/data/sidecars'); + expect(pathSetting.style.display).toBe('none'); + expect(manager._loadedSidecarStorageMode).toBe('alongside'); + }); + + it('loads centralized mode and shows the path input', () => { + const manager = createManager(); + const { select, pathSetting } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('centralized'); + expect(pathSetting.style.display).toBe('block'); + expect(manager._loadedSidecarStorageMode).toBe('centralized'); + }); + + it('falls back to alongside for unknown stored modes', () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + state.global.settings = { sidecar_storage_mode: 'bogus' }; + + manager.loadSidecarStorageSettings(); + + expect(select.value).toBe('alongside'); + }); + }); + + describe('handleSidecarStorageModeChange', () => { + it('does not prompt for migration when the mode is unchanged', async () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside' }; + manager._loadedSidecarStorageMode = 'alongside'; + select.value = 'alongside'; + mockFetchOk(); + + await manager.handleSidecarStorageModeChange(); + + expect(global.fetch).not.toHaveBeenCalledWith( + '/api/lm/sidecars/migrate', + expect.anything() + ); + expect(showToast).toHaveBeenCalledWith( + 'toast.settings.settingsUpdated', + expect.anything(), + 'success' + ); + }); + + it('migrates to centralized after the user confirms the prompt', async () => { + const manager = createManager(); + const { select, pathSetting } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'alongside' }; + manager._loadedSidecarStorageMode = 'alongside'; + select.value = 'centralized'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStorageModeChange(); + + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await changePromise; + + expect(state.global.settings.sidecar_storage_mode).toBe('centralized'); + expect(pathSetting.style.display).toBe('block'); + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ direction: 'to_centralized', force: true }), + })); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrateSuccess', {}, 'success'); + expect(resetAndReload).toHaveBeenCalledWith(true); + expect(modal.classList.contains('show')).toBe(false); + }); + + it('shows a deferred notice and skips migration when the user cancels', async () => { + const manager = createManager(); + const { select } = appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + manager._loadedSidecarStorageMode = 'centralized'; + select.value = 'alongside'; + mockFetchOk(); + + const changePromise = manager.handleSidecarStorageModeChange(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="cancel-sidecar-migration"]').click(); + await changePromise; + + const migrateCalls = global.fetch.mock.calls.filter(([url]) => url === '/api/lm/sidecars/migrate'); + expect(migrateCalls).toHaveLength(0); + expect(showToast).toHaveBeenCalledWith('settings.sidecarStorage.migrationDeferred', {}, 'info'); + }); + }); + + describe('confirmAndMigrateSidecars', () => { + it('derives the migration direction from the saved mode', async () => { + const manager = createManager(); + appendSidecarControls(); + const modal = appendMigrationModal(); + state.global.settings = { sidecar_storage_mode: 'centralized' }; + mockFetchOk(); + + const confirmPromise = manager.confirmAndMigrateSidecars(); + await vi.waitFor(() => expect(modal.classList.contains('show')).toBe(true)); + modal.querySelector('[data-action="confirm-sidecar-migration"]').click(); + await confirmPromise; + + expect(global.fetch).toHaveBeenCalledWith('/api/lm/sidecars/migrate', expect.objectContaining({ + body: JSON.stringify({ direction: 'to_centralized', force: true }), + })); + }); + }); + + describe('migrateSidecars', () => { + it('surfaces backend failures as an error toast', async () => { + const manager = createManager(); + const { migrateBtn } = appendSidecarControls(); + mockFetchOk({ success: false, error: 'disk full' }); + + await manager.migrateSidecars('to_alongside'); + + expect(showToast).toHaveBeenCalledWith( + 'settings.sidecarStorage.migrateFailed', + { message: 'disk full' }, + 'error' + ); + expect(resetAndReload).not.toHaveBeenCalled(); + expect(migrateBtn.disabled).toBe(false); + }); + }); +}); diff --git a/tests/routes/test_misc_routes.py b/tests/routes/test_misc_routes.py index cb8406a8..8ea65248 100644 --- a/tests/routes/test_misc_routes.py +++ b/tests/routes/test_misc_routes.py @@ -23,6 +23,7 @@ from py.routes.handlers.misc_handlers import ( NodeRegistryHandler, ServiceRegistryAdapter, SettingsHandler, + SidecarMigrationHandler, _collect_comfyui_session_logs, _is_wsl, _wsl_to_windows_path, @@ -2557,3 +2558,86 @@ async def test_get_model_versions_status_supported_type_stays_interactive(): "hasBeenDownloaded": False, } ] + + +class DummySidecarMigrationUseCase: + def __init__(self, result): + self.result = result + self.calls = [] + + async def execute_with_error_handling(self, *, direction, progress_cb=None, force=False): + self.calls.append({"direction": direction, "force": force}) + return self.result + + +def _sidecar_migration_handler(result): + use_case = DummySidecarMigrationUseCase(result) + handler = SidecarMigrationHandler( + use_case_factory=lambda: use_case, + progress_callback_factory=lambda: None, + ) + return handler, use_case + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_runs_to_centralized(): + result = {"success": True, "direction": "to_centralized", "moved": 3} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "to_centralized", "force": True}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert payload["moved"] == 3 + assert use_case.calls == [{"direction": "to_centralized", "force": True}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_rejects_bad_direction(): + handler, use_case = _sidecar_migration_handler({"success": True}) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "sideways"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert payload["success"] is False + assert use_case.calls == [] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_accepts_get_query_params(): + result = {"success": True, "direction": "to_alongside", "moved": 0} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest( # pyright: ignore[reportArgumentType] + query={"direction": "to_alongside", "force": "true"}, + method="GET", + ) + ) + payload = _json_payload(response) + + assert response.status == 200 + assert payload["success"] is True + assert use_case.calls == [{"direction": "to_alongside", "force": True}] + + +@pytest.mark.asyncio +async def test_sidecar_migration_handler_guard_refusal_is_400(): + result = {"success": False, "error": "sidecar storage is already centralized"} + handler, use_case = _sidecar_migration_handler(result) + + response = await handler.migrate_sidecars( + FakeRequest(json_data={"direction": "to_centralized"}) # pyright: ignore[reportArgumentType] + ) + payload = _json_payload(response) + + assert response.status == 400 + assert payload["success"] is False + assert "already centralized" in payload["error"] + assert use_case.calls == [{"direction": "to_centralized", "force": False}] diff --git a/tests/services/test_centralized_sidecar_storage.py b/tests/services/test_centralized_sidecar_storage.py new file mode 100644 index 00000000..2d9164ab --- /dev/null +++ b/tests/services/test_centralized_sidecar_storage.py @@ -0,0 +1,361 @@ +"""Centralized sidecar storage: lifecycle flows (delete/move/rename/scans).""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import AsyncMock + +import pytest + +from py.config import config +from py.services.checkpoint_scanner import CheckpointScanner +from py.services.model_lifecycle_service import ( + ModelLifecycleService, + delete_model_artifacts, +) +from py.services.pending_delete_service import PendingDeleteService +from py.services.settings_manager import get_settings_manager +from py.utils.metadata_manager import MetadataManager + + +def _normalize(path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def library_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Single checkpoint root under tmp_path; every other root emptied.""" + + root = tmp_path / "checkpoints" + root.mkdir() + for attr, value in ( + ("loras_roots", []), + ("base_models_roots", [str(root)]), + ("checkpoints_roots", [str(root)]), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + return root + + +@pytest.fixture +def centralized(library_root: Path, tmp_path: Path) -> Path: + """Enable centralized mode rooted at tmp_path/sidecars.""" + + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", str(sidecar_root)) + return sidecar_root + + +def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: + """Expected mirror directory for a library-relative path.""" + + library = get_settings_manager().get_active_library_name() + return sidecar_root.joinpath(library, "checkpoints", *rel) + + +def _write_sidecar( + mirror_dir: Path, + stem: str, + *, + file_path: str, + preview_name: str | None = None, + extra: Dict[str, Any] | None = None, +) -> Path: + mirror_dir.mkdir(parents=True, exist_ok=True) + payload: Dict[str, Any] = { + "file_name": stem, + "file_path": _normalize(file_path), + } + if preview_name: + payload["preview_url"] = _normalize(mirror_dir / preview_name) + (mirror_dir / preview_name).write_bytes(b"preview") + if extra: + payload.update(extra) + metadata_path = mirror_dir / f"{stem}.metadata.json" + metadata_path.write_text(json.dumps(payload), encoding="utf-8") + return metadata_path + + +@pytest.mark.asyncio +async def test_delete_model_artifacts_centralized( + library_root: Path, centralized: Path +): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") + + deleted = await delete_model_artifacts(str(library_root), "model") + + assert not model.exists() + assert not (mirror / "model.metadata.json").exists() + assert not (mirror / "model.preview.webp").exists() + assert any(path.endswith("model.safetensors") for path in deleted) + assert "model.metadata.json" in deleted + assert "model.preview.webp" in deleted + + +@pytest.mark.asyncio +async def test_delete_model_artifacts_alongside_still_siblings( + library_root: Path, +): + """Alongside mode (default) keeps deleting sidecars next to the model.""" + + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + sidecar = library_root / "model.metadata.json" + sidecar.write_text("{}", encoding="utf-8") + preview = library_root / "model.preview.webp" + preview.write_bytes(b"preview") + + await delete_model_artifacts(str(library_root), "model") + + assert not model.exists() + assert not sidecar.exists() + assert not preview.exists() + + +def test_enumerate_model_artifacts_centralized( + library_root: Path, centralized: Path +): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.png") + + service = PendingDeleteService.__new__(PendingDeleteService) + artifacts = service._enumerate_model_artifacts( + str(library_root), "model", ".safetensors" + ) + + assert artifacts == [ + os.path.abspath(str(model)), + os.path.abspath(str(mirror / "model.metadata.json")), + os.path.abspath(str(mirror / "model.preview.png")), + ] + + +class _RecordingScanner: + def __init__(self): + self.calls: List[tuple] = [] + self.model_type = "lora" + + async def update_single_model_cache(self, old_path, new_path, metadata): + self.calls.append((old_path, new_path, metadata)) + + +class _PassthroughMetadataManager: + async def save_metadata(self, path: str, metadata): + await MetadataManager.save_metadata(path, metadata) + return True + + +async def _json_metadata_loader(path: str) -> Dict[str, object]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +@pytest.mark.asyncio +async def test_rename_model_centralized(library_root: Path, centralized: Path): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar(mirror, "model", file_path=model, preview_name="model.preview.webp") + + service = ModelLifecycleService( + scanner=_RecordingScanner(), + metadata_manager=_PassthroughMetadataManager(), + metadata_loader=_json_metadata_loader, + ) + + result = await service.rename_model( + file_path=_normalize(model), new_file_name="renamed" + ) + + assert result["success"] is True + renamed = library_root / "renamed.safetensors" + assert renamed.exists() + assert not model.exists() + + # Mirror sidecar/preview renamed with the model; nothing left alongside. + assert (mirror / "renamed.metadata.json").exists() + assert (mirror / "renamed.preview.webp").exists() + assert not (mirror / "model.metadata.json").exists() + assert not (mirror / "model.preview.webp").exists() + assert not (library_root / "renamed.metadata.json").exists() + assert not (library_root / "renamed.preview.webp").exists() + + saved = json.loads((mirror / "renamed.metadata.json").read_text()) + assert saved["file_name"] == "renamed" + assert saved["file_path"].endswith("renamed.safetensors") + assert saved["preview_url"].endswith("renamed.preview.webp") + assert str(mirror).replace(os.sep, "/") in saved["preview_url"] + + +@pytest.mark.asyncio +async def test_move_model_centralized( + library_root: Path, centralized: Path, monkeypatch: pytest.MonkeyPatch +): + source_dir = library_root / "old" + source_dir.mkdir() + model = source_dir / "model.safetensors" + model.write_bytes(b"weights") + target_dir = library_root / "new" + + old_mirror = _mirror_dir(centralized, "old") + _write_sidecar( + old_mirror, "model", file_path=model, preview_name="model.preview.webp" + ) + + scanner = CheckpointScanner() + monkeypatch.setattr( + scanner, "update_single_model_cache", AsyncMock(return_value=True) + ) + + result = await scanner.move_model(_normalize(model), _normalize(target_dir)) + + assert result is not None + moved_model = target_dir / "model.safetensors" + assert moved_model.exists() + assert not model.exists() + + new_mirror = _mirror_dir(centralized, "new") + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.webp").exists() + assert not (old_mirror / "model.metadata.json").exists() + assert not (old_mirror / "model.preview.webp").exists() + + saved = json.loads((new_mirror / "model.metadata.json").read_text()) + assert saved["file_path"].endswith("new/model.safetensors") + assert saved["preview_url"].endswith("model.preview.webp") + assert str(new_mirror).replace(os.sep, "/") in saved["preview_url"] + + +class _FakeCache: + def __init__(self, raw_data: List[Dict[str, Any]]): + self.raw_data = raw_data + self.all_folders: List[str] = [] + self.folders: List[str] = [] + + def remove_from_version_index(self, _item) -> None: + pass + + def rebuild_version_index(self) -> None: + pass + + async def resort(self) -> None: + pass + + +@pytest.mark.asyncio +async def test_rename_known_folder_centralized( + library_root: Path, centralized: Path, monkeypatch: pytest.MonkeyPatch +): + # The caller (ModelFileService.rename_folder) has already renamed the + # model directory on disk when the scanner is asked to re-key records. + old_dir = library_root / "oldfolder" + new_dir = library_root / "newfolder" + new_dir.mkdir() + model = new_dir / "model.safetensors" + model.write_bytes(b"weights") + + old_model_path = old_dir / "model.safetensors" + old_mirror = _mirror_dir(centralized, "oldfolder") + _write_sidecar( + old_mirror, "model", file_path=old_model_path, preview_name="model.preview.webp" + ) + + cache_entry: Dict[str, Any] = { + "file_path": _normalize(old_model_path), + "folder": "oldfolder", + "preview_url": _normalize(old_mirror / "model.preview.webp"), + "sha256": "", + } + scanner = CheckpointScanner() + scanner._cache = _FakeCache([cache_entry]) + monkeypatch.setattr(scanner, "_persist_current_cache", AsyncMock()) + + changed = await scanner.rename_known_folder( + "oldfolder", + "newfolder", + previous_path=str(old_dir), + new_path=str(new_dir), + ) + + assert changed is True + + new_mirror = _mirror_dir(centralized, "newfolder") + assert (new_mirror / "model.metadata.json").exists() + assert (new_mirror / "model.preview.webp").exists() + assert not old_mirror.exists() + + saved = json.loads((new_mirror / "model.metadata.json").read_text()) + assert saved["file_path"] == _normalize(model) + assert saved["preview_url"] == _normalize(new_mirror / "model.preview.webp") + + assert cache_entry["file_path"] == _normalize(model) + assert cache_entry["folder"] == "newfolder" + assert cache_entry["preview_url"] == _normalize( + new_mirror / "model.preview.webp" + ) + + +@pytest.mark.asyncio +async def test_pending_models_mirror_walk(library_root: Path, centralized: Path): + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar( + mirror, + "model", + file_path=model, + extra={"hash_status": "pending", "sha256": ""}, + ) + # Orphan sidecar: recorded model path is gone and no stem match exists. + _write_sidecar( + mirror, + "ghost", + file_path=library_root / "ghost.safetensors", + extra={"hash_status": "pending", "sha256": ""}, + ) + + scanner = CheckpointScanner() + pending = await scanner._find_pending_models_from_filesystem() + + assert len(pending) == 1 + assert pending[0]["file_path"] == _normalize(model) + assert pending[0]["hash_status"] == "pending" + + +@pytest.mark.asyncio +async def test_pending_models_mirror_walk_uses_stem_fallback( + library_root: Path, centralized: Path +): + """A stale recorded file_path falls back to probing by stem + extension.""" + + model = library_root / "model.safetensors" + model.write_bytes(b"weights") + mirror = _mirror_dir(centralized) + _write_sidecar( + mirror, + "model", + file_path=library_root / "renamed-away.safetensors", + extra={"hash_status": "pending", "sha256": ""}, + ) + + scanner = CheckpointScanner() + pending = await scanner._find_pending_models_from_filesystem() + + assert len(pending) == 1 + assert pending[0]["file_path"] == _normalize(model) diff --git a/tests/services/use_cases/test_sidecar_migration_use_case.py b/tests/services/use_cases/test_sidecar_migration_use_case.py new file mode 100644 index 00000000..fca8c43f --- /dev/null +++ b/tests/services/use_cases/test_sidecar_migration_use_case.py @@ -0,0 +1,294 @@ +"""Sidecar migration use case: layout moves, conflicts, guards.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from py.config import config +from py.services.settings_manager import get_settings_manager +from py.services.use_cases.sidecar_migration_use_case import SidecarMigrationUseCase + + +def _normalize(path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def library_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Single lora root under tmp_path; every other root emptied.""" + + root = tmp_path / "loras" + root.mkdir() + for attr, value in ( + ("loras_roots", [str(root)]), + ("base_models_roots", []), + ("checkpoints_roots", []), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + return root + + +@pytest.fixture +def sidecar_root(tmp_path: Path) -> Path: + """Configured centralized root (mode-independent for migration).""" + + root = tmp_path / "sidecars" + get_settings_manager().set("sidecar_storage_path", str(root)) + return root + + +def _set_mode(mode: str) -> None: + get_settings_manager().set("sidecar_storage_mode", mode) + + +def _mirror_dir(sidecar_root: Path, *rel: str) -> Path: + """Expected mirror directory for a library-relative path.""" + + library = get_settings_manager().get_active_library_name() + return sidecar_root.joinpath(library, "loras", *rel) + + +def _write_model(directory: Path, stem: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + model = directory / f"{stem}.safetensors" + model.write_bytes(b"weights") + return model + + +def _write_sidecar(directory: Path, stem: str, model: Path, *, preview_ext: str | None = ".preview.webp") -> Path: + directory.mkdir(parents=True, exist_ok=True) + payload: Dict[str, Any] = { + "file_name": stem, + "file_path": _normalize(model), + } + if preview_ext: + payload["preview_url"] = _normalize(directory / f"{stem}{preview_ext}") + sidecar = directory / f"{stem}.metadata.json" + sidecar.write_text(json.dumps(payload), encoding="utf-8") + return sidecar + + +class _FakeCache: + def __init__(self, raw_data: List[Dict[str, Any]]) -> None: + self.raw_data = raw_data + + +class _FakeScanner: + def __init__(self, raw_data: List[Dict[str, Any]]) -> None: + self._cache = _FakeCache(raw_data) + + async def get_cached_data(self) -> _FakeCache: + return self._cache + + +def _make_use_case(model_paths: List[str]) -> SidecarMigrationUseCase: + scanner = _FakeScanner([{"file_path": path} for path in model_paths]) + + async def scanner_factory() -> _FakeScanner: + return scanner + + return SidecarMigrationUseCase( + scanner_factories=(("lora", scanner_factory),), + settings_service=get_settings_manager(), + ) + + +class _ProgressRecorder: + def __init__(self) -> None: + self.payloads: List[Dict[str, Any]] = [] + + async def on_progress(self, payload: Dict[str, Any]) -> None: + self.payloads.append(payload) + + +@pytest.mark.asyncio +async def test_migrate_to_centralized_moves_sidecar_and_previews( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + model = _write_model(library_root / "sub", "model") + sidecar = _write_sidecar(library_root / "sub", "model", model) + preview = library_root / "sub" / "model.preview.webp" + preview.write_bytes(b"preview") + extra_preview = library_root / "sub" / "model.png" + extra_preview.write_bytes(b"extra") + + recorder = _ProgressRecorder() + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_centralized(recorder, force=True) + + assert summary["success"] is True + assert summary["direction"] == "to_centralized" + assert summary["moved"] == 3 + assert summary["models_moved"] == 1 + assert summary["skipped"] == 0 + assert summary["conflicts"] == 0 + assert summary["errors"] == [] + + mirror = _mirror_dir(sidecar_root, "sub") + assert not sidecar.exists() + assert not preview.exists() + assert not extra_preview.exists() + moved_sidecar = mirror / "model.metadata.json" + assert moved_sidecar.exists() + assert (mirror / "model.preview.webp").exists() + assert (mirror / "model.png").exists() + # Model files never move. + assert model.exists() + + metadata = json.loads(moved_sidecar.read_text(encoding="utf-8")) + assert metadata["file_path"] == _normalize(model) + assert metadata["file_name"] == "model" + # Recorded extension wins when rewriting preview_url. + assert metadata["preview_url"] == _normalize(mirror / "model.preview.webp") + + statuses = [payload["status"] for payload in recorder.payloads] + assert statuses[0] == "started" + assert statuses[-1] == "completed" + assert all(p["type"] == "sidecar_migration_progress" for p in recorder.payloads) + + +@pytest.mark.asyncio +async def test_migrate_to_alongside_reverses_layout( + library_root: Path, sidecar_root: Path +): + _set_mode("alongside") + model = _write_model(library_root / "sub", "model") + mirror = _mirror_dir(sidecar_root, "sub") + sidecar = _write_sidecar(mirror, "model", model) + preview = mirror / "model.preview.webp" + preview.write_bytes(b"preview") + + use_case = _make_use_case([str(model)]) + summary = await use_case.migrate_to_alongside(force=True) + + assert summary["success"] is True + assert summary["moved"] == 2 + + assert not sidecar.exists() + assert not preview.exists() + moved_sidecar = library_root / "sub" / "model.metadata.json" + assert moved_sidecar.exists() + assert (library_root / "sub" / "model.preview.webp").exists() + + metadata = json.loads(moved_sidecar.read_text(encoding="utf-8")) + assert metadata["file_path"] == _normalize(model) + assert metadata["preview_url"] == _normalize( + library_root / "sub" / "model.preview.webp" + ) + + +@pytest.mark.asyncio +async def test_migrate_conflict_keeps_newer_file( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + mirror = _mirror_dir(sidecar_root) + + # Model A: destination (mirror) sidecar is newer -> destination wins. + model_a = _write_model(library_root, "model_a") + src_a = _write_sidecar(library_root, "model_a", model_a, preview_ext=None) + dst_a = _write_sidecar(mirror, "model_a", model_a, preview_ext=None) + os.utime(src_a, (1000, 1000)) + os.utime(dst_a, (2000, 2000)) + + # Model B: source (alongside) sidecar is newer -> source replaces. + model_b = _write_model(library_root, "model_b") + src_b = _write_sidecar(library_root, "model_b", model_b, preview_ext=None) + dst_b = _write_sidecar(mirror, "model_b", model_b, preview_ext=None) + (mirror / "model_b.metadata.json").write_text( + json.dumps({"stale": True}), encoding="utf-8" + ) + os.utime(src_b, (3000, 3000)) + os.utime(dst_b, (2000, 2000)) + + use_case = _make_use_case([str(model_a), str(model_b)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["conflicts"] == 2 + assert summary["moved"] == 1 + + # A: destination kept, source deleted, content untouched. + assert not src_a.exists() + metadata_a = json.loads(dst_a.read_text(encoding="utf-8")) + assert metadata_a["file_name"] == "model_a" + + # B: newer source replaced the stale destination. + assert not src_b.exists() + metadata_b = json.loads(dst_b.read_text(encoding="utf-8")) + assert metadata_b.get("stale") is None + assert metadata_b["file_name"] == "model_b" + + +@pytest.mark.asyncio +async def test_migrate_missing_model_file_is_skipped( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + missing_model = library_root / "ghost.safetensors" + sidecar = _write_sidecar(library_root, "ghost", missing_model, preview_ext=None) + + use_case = _make_use_case([str(missing_model)]) + summary = await use_case.migrate_to_centralized(force=True) + + assert summary["success"] is True + assert summary["skipped"] == 1 + assert summary["moved"] == 0 + # Sidecar stays put when the model file is gone. + assert sidecar.exists() + + +@pytest.mark.asyncio +async def test_migrate_empty_library_is_noop( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + recorder = _ProgressRecorder() + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_centralized(recorder, force=True) + + assert summary["success"] is True + assert summary["models_total"] == 0 + assert summary["moved"] == 0 + statuses = [payload["status"] for payload in recorder.payloads] + assert statuses == ["started", "completed"] + + +@pytest.mark.asyncio +async def test_migrate_to_centralized_refuses_when_already_centralized( + library_root: Path, sidecar_root: Path +): + _set_mode("centralized") + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_centralized() + + assert summary["success"] is False + assert "already centralized" in summary["error"] + assert summary["moved"] == 0 + + +@pytest.mark.asyncio +async def test_migrate_to_alongside_refuses_when_already_alongside( + library_root: Path, sidecar_root: Path +): + _set_mode("alongside") + use_case = _make_use_case([]) + + summary = await use_case.migrate_to_alongside() + + assert summary["success"] is False + assert "already alongside" in summary["error"] + assert summary["moved"] == 0 diff --git a/tests/utils/test_sidecar_paths.py b/tests/utils/test_sidecar_paths.py new file mode 100644 index 00000000..f09966d9 --- /dev/null +++ b/tests/utils/test_sidecar_paths.py @@ -0,0 +1,250 @@ +"""Tests for py.utils.sidecar_paths (alongside + centralized storage modes).""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from py.services.settings_manager import get_settings_manager +from py.utils import sidecar_paths +from py.utils.sidecar_paths import ( + METADATA_SUFFIX, + get_configured_sidecar_root, + get_metadata_path, + get_preview_dir, + get_sidecar_dir, + get_sidecar_root, + get_storage_mode, + is_centralized, + is_metadata_path, + resolve_centralized_dir, + resolve_centralized_dir_for_dir, + resolve_metadata_path, + sanitize_path_component, +) + + +def _normalize(path: Path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture +def model_roots(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: + """Point every config model root the sidecar module reads at tmp_path.""" + + from py.config import config + + loras = tmp_path / "loras" + checkpoints = tmp_path / "checkpoints" + loras.mkdir() + checkpoints.mkdir() + + for attr, value in ( + ("loras_roots", [str(loras)]), + ("base_models_roots", [str(checkpoints)]), + ("embeddings_roots", []), + ("other_roots", []), + ("extra_loras_roots", []), + ("extra_checkpoints_roots", []), + ("extra_unet_roots", []), + ("extra_embeddings_roots", []), + ): + monkeypatch.setattr(config, attr, value, raising=False) + + return {"loras": loras, "checkpoints": checkpoints} + + +@pytest.fixture +def centralized(model_roots: dict, tmp_path: Path) -> Path: + """Enable centralized mode rooted at tmp_path/sidecars; returns the root.""" + + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", str(sidecar_root)) + return sidecar_root + + +class TestAlongsideMode: + def test_default_mode_is_alongside(self): + assert get_storage_mode() == "alongside" + assert not is_centralized() + assert get_sidecar_root() == "" + + def test_metadata_path_next_to_model(self, tmp_path: Path): + model = tmp_path / "sub" / "model.safetensors" + assert get_metadata_path(str(model)) == os.path.join( + str(tmp_path), "sub", "model" + METADATA_SUFFIX + ) + + def test_preview_and_sidecar_dir_are_model_dir(self, tmp_path: Path): + model = tmp_path / "sub" / "model.safetensors" + expected = os.path.dirname(os.path.abspath(str(model))) + assert get_sidecar_dir(str(model)) == expected + assert get_preview_dir(str(model)) == expected + + +class TestPathPredicates: + def test_is_metadata_path(self): + assert is_metadata_path("/x/model.metadata.json") + assert not is_metadata_path("/x/model.safetensors") + assert not is_metadata_path("/x/model.metadata.json.bak") + + def test_resolve_metadata_path_passthrough_for_sidecar(self): + sidecar = "/x/model.metadata.json" + assert resolve_metadata_path(sidecar) == sidecar + + def test_resolve_metadata_path_derives_for_model(self, tmp_path: Path): + model = tmp_path / "model.safetensors" + assert resolve_metadata_path(str(model)) == get_metadata_path(str(model)) + + +class TestSanitizePathComponent: + def test_special_characters_replaced(self): + assert sanitize_path_component("foo bar/baz:qux") == "foo_bar_baz_qux" + + def test_safe_characters_kept(self): + assert sanitize_path_component("Flux-1.dev_v2") == "Flux-1.dev_v2" + + def test_empty_falls_back_to_underscore(self): + assert sanitize_path_component("") == "_" + assert sanitize_path_component(None) == "_" + + +class TestCentralizedMode: + def test_mirror_layout(self, model_roots: dict, centralized: Path): + model = model_roots["loras"] / "styles" / "anime" / "model.safetensors" + library = get_settings_manager().get_active_library_name() + + metadata_path = get_metadata_path(str(model)) + + expected = os.path.join( + str(centralized), library, "loras", "styles", "anime", "model" + METADATA_SUFFIX + ) + assert metadata_path == expected + assert get_preview_dir(str(model)) == os.path.dirname(expected) + assert is_centralized() + + def test_longest_root_wins(self, model_roots: dict, centralized: Path, monkeypatch: pytest.MonkeyPatch): + from py.config import config + + nested = model_roots["loras"] / "nested" + nested.mkdir() + monkeypatch.setattr( + config, + "loras_roots", + [str(model_roots["loras"]), str(nested)], + raising=False, + ) + library = get_settings_manager().get_active_library_name() + + model = nested / "model.safetensors" + assert get_metadata_path(str(model)) == os.path.join( + str(centralized), library, "nested", "model" + METADATA_SUFFIX + ) + + def test_outside_roots_falls_back_to_alongside( + self, model_roots: dict, centralized: Path, tmp_path: Path + ): + outside = tmp_path / "elsewhere" / "model.safetensors" + + assert resolve_centralized_dir(str(outside)) is None + assert get_sidecar_dir(str(outside)) == os.path.dirname( + os.path.abspath(str(outside)) + ) + assert get_metadata_path(str(outside)) == os.path.join( + str(tmp_path), "elsewhere", "model" + METADATA_SUFFIX + ) + + def test_resolve_centralized_dir_for_dir_matches_model_resolution( + self, model_roots: dict, centralized: Path + ): + model_dir = model_roots["checkpoints"] / "sub" + model = model_dir / "model.safetensors" + + assert resolve_centralized_dir_for_dir(str(model_dir)) == resolve_centralized_dir( + str(model) + ) + + def test_resolve_centralized_dir_for_dir_root_maps_to_mirror_base( + self, model_roots: dict, centralized: Path + ): + library = get_settings_manager().get_active_library_name() + + assert resolve_centralized_dir_for_dir(str(model_roots["loras"])) == os.path.join( + str(centralized), library, "loras" + ) + + def test_empty_path_uses_default_sidecar_root(self, model_roots: dict, tmp_path: Path): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "centralized") + settings.set("sidecar_storage_path", "") + + root = get_sidecar_root() + assert root + assert root.endswith(os.sep + "sidecars") + assert is_centralized() + + +class TestModeIndependentResolution: + """Migration tooling resolves the mirror layout regardless of active mode.""" + + def test_configured_root_resolves_in_alongside_mode(self, tmp_path: Path): + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + settings.set("sidecar_storage_path", str(sidecar_root)) + + assert get_sidecar_root() == "" + assert get_configured_sidecar_root() == os.path.abspath(str(sidecar_root)) + + def test_configured_root_defaults_to_settings_dir(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + settings.set("sidecar_storage_path", "") + + root = get_configured_sidecar_root() + assert root + assert root.endswith(os.sep + "sidecars") + + def test_resolve_centralized_dir_for_dir_with_explicit_root( + self, model_roots: dict, tmp_path: Path + ): + sidecar_root = tmp_path / "sidecars" + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "alongside") + library = settings.get_active_library_name() + + model_dir = model_roots["loras"] / "sub" + + # Alongside mode: no root resolves without the override. + assert resolve_centralized_dir_for_dir(str(model_dir)) is None + assert resolve_centralized_dir_for_dir( + str(model_dir), sidecar_root=str(sidecar_root) + ) == os.path.join(str(sidecar_root), library, "loras", "sub") + + +class TestSettingsValidation: + def test_invalid_mode_falls_back_to_alongside(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", "bogus") + assert settings.get("sidecar_storage_mode") == "alongside" + + def test_mode_is_normalized(self): + settings = get_settings_manager() + settings.set("sidecar_storage_mode", " Centralized ") + assert settings.get("sidecar_storage_mode") == "centralized" + + def test_path_is_normalized_to_absolute(self, tmp_path: Path): + settings = get_settings_manager() + settings.set("sidecar_storage_path", str(tmp_path / "sidecars")) + assert settings.get("sidecar_storage_path") == os.path.abspath( + str(tmp_path / "sidecars") + ) + + def test_non_string_path_becomes_empty(self): + settings = get_settings_manager() + settings.set("sidecar_storage_path", None) + assert settings.get("sidecar_storage_path") == ""