diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index 7c0aede7..00ba5585 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -63,6 +63,15 @@ def _is_hidden_relative_path(rel_path: str) -> bool: return any(part.startswith(".") for part in rel_path.replace(os.sep, "/").split("/")) +def _file_name_stem(file_path: str) -> str: + """Return the extension-free file name of a normalized model path. + + ``file_name`` cache/sidecar fields are defined as the on-disk stem, so this + is the authoritative value to compare stored names against (issue #1112). + """ + return os.path.splitext(os.path.basename(file_path))[0] + + # Maps a scanner model type to the manager page type used in progress # broadcasts (e.g. 'lora' -> 'loras'). PAGE_TYPE_MAP = { @@ -1076,6 +1085,26 @@ class ModelScanner: # Track found files and new files found_paths = set() new_files = [] + # Cached entries whose stored file_name no longer matches the file + # on disk (e.g. dotted stems truncated by the legacy .civitai.info + # migration, issue #1112). Repaired in place after the walk; the + # list stays empty on a clean library, so a no-change reconcile + # only pays one string compare per cached file. + stale_paths: List[str] = [] + stale_seen: Set[str] = set() + + def mark_stale_if_needed(cached_path: str) -> None: + """Queue a cached path for file_name repair when it drifted.""" + if cached_path in stale_seen: + return + item = path_to_item.get(cached_path) + if item is None: + return + if item.get("file_name") == _file_name_stem(cached_path): + return + stale_seen.add(cached_path) + stale_paths.append(cached_path) + visited_real_paths = set() discovered_real_files = set() discovered_folders: Set[str] = set() @@ -1110,6 +1139,7 @@ class ModelScanner: # Check if this file is already in cache if file_path in cached_paths: found_paths.add(file_path) + mark_stale_if_needed(file_path) continue # Only a cache miss needs the physical path, so the @@ -1120,6 +1150,7 @@ class ModelScanner: cached_real_match = lookup_cached_real_path(real_file_path) if cached_real_match: found_paths.add(cached_real_match) + mark_stale_if_needed(cached_real_match) continue if file_path in self._excluded_models: @@ -1132,6 +1163,7 @@ class ModelScanner: for cached_path in cached_paths: if cached_path.lower() == lower_path: found_paths.add(cached_path) + mark_stale_if_needed(cached_path) matched = True break if matched: @@ -1242,7 +1274,57 @@ class ModelScanner: elapsed_seconds=time.time() - start_time, ) return - + + # Repair rows whose file_name drifted from the file on disk. Only + # mismatching entries are re-read here, so a clean library never + # touches metadata during a refresh. Each repair goes through the + # single-row update path: load_metadata() normalizes the sidecar + # (MetadataManager._normalize_metadata_paths) and + # _sync_cache_from_metadata_impl() rewrites one targeted SQL delta + # instead of a full cache save, and the mismatch is gone + # afterwards, so the work never repeats (issue #1112). + total_repaired = 0 + if stale_paths: + logger.info( + "%s Scanner: Repairing %d cached entries whose file_name no longer matches the file on disk", + self.model_type.capitalize(), + len(stale_paths), + ) + for path in stale_paths: + if self.is_cancelled(): + logger.info(f"{self.model_type.capitalize()} Scanner: Reconcile repair cancelled") + break + try: + metadata, _should_skip = await MetadataManager.load_metadata( + path, self.model_class + ) + if metadata is None: + # Missing or corrupt sidecar: keep the existing row + # so a full rebuild can recreate the metadata from + # .civitai.info (or defaults) without losing cached + # fields such as tags or civitai data. + logger.debug( + "%s Scanner: Leaving %s unchanged (no usable metadata to repair from)", + self.model_type.capitalize(), + path, + ) + continue + + payload = metadata.to_dict() + unknown_fields = getattr(metadata, "_unknown_fields", None) + if isinstance(unknown_fields, dict): + payload.update(unknown_fields) + + if await self._sync_cache_from_metadata_impl(path, payload): + total_repaired += 1 + except Exception as exc: + logger.warning( + "%s Scanner: Failed to repair file_name for %s: %s", + self.model_type.capitalize(), + path, + exc, + ) + # Find missing files (in cache but not in filesystem) missing_files = cached_paths - found_paths total_removed = 0 @@ -1323,7 +1405,11 @@ class ModelScanner: elif folders_changed: await self._persist_current_cache() - logger.info(f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in {time.time() - start_time:.2f} seconds. Added {total_added}, removed {total_removed} models.") + logger.info( + f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in " + f"{time.time() - start_time:.2f} seconds. Added {total_added}, " + f"removed {total_removed}, repaired {total_repaired} models." + ) await self._broadcast_scan_progress( 'completed', 'process_new', 100, False, added=total_added, removed=total_removed, @@ -1552,11 +1638,16 @@ class ModelScanner: file_info = next((f for f in version_info.get('files', []) if f.get('primary')), None) if file_info: - file_name = os.path.splitext(os.path.basename(file_path))[0] - file_info['name'] = file_name - + local_stem = os.path.splitext(os.path.basename(file_path))[0] + # from_civitai_info expects an API-shaped file entry and + # strips one extension itself, so hand it the real + # basename: passing the already extension-free stem made + # it cut dotted names at their last dot ("lora-sd1.5-..." + # became "lora-sd1", issue #1112). + file_info['name'] = os.path.basename(file_path) + metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path) - metadata.preview_url = find_preview_file(file_name, os.path.dirname(file_path)) + metadata.preview_url = find_preview_file(local_stem, os.path.dirname(file_path)) await MetadataManager.save_metadata(file_path, metadata) logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)") except Exception as e: diff --git a/py/utils/models.py b/py/utils/models.py index ebbf71c7..a23a1d47 100644 --- a/py/utils/models.py +++ b/py/utils/models.py @@ -2,7 +2,11 @@ from dataclasses import dataclass, asdict, field from typing import Callable, Dict, Optional, List, Any from datetime import datetime import os -from .constants import CIVITAI_TYPE_TO_OTHER_SUB_TYPE, INVALID_AUTOV3_EMPTY_HASH +from .constants import ( + CIVITAI_TYPE_TO_OTHER_SUB_TYPE, + INVALID_AUTOV3_EMPTY_HASH, + MODEL_FILE_EXTENSIONS, +) from .model_utils import determine_base_model @@ -46,6 +50,24 @@ def autov3_from_civitai_files(civitai_data: Optional[Dict[str, Any]], sha256: st return None +def strip_model_extension(file_name: str) -> str: + """Strip a recognized model file extension, leaving dotted stems intact. + + ``os.path.splitext`` treats everything after the last dot as an extension, + so applying it to an already extension-free name truncates dotted stems: + ``lora-sd1.5-backlight_slider_v10`` becomes ``lora-sd1``. API filenames keep + their extension and need one strip, while migration paths (``.civitai.info``) + pass the local stem as-is, so only remove a suffix that is a known model + extension and both inputs resolve to the same stem (issue #1112). + """ + if not file_name: + return file_name + stem, extension = os.path.splitext(file_name) + if extension.lower() in MODEL_FILE_EXTENSIONS: + return stem + return file_name + + @dataclass class BaseModelMetadata: """Base class for all model metadata structures""" @@ -241,6 +263,7 @@ class LoraMetadata(BaseModelMetadata): ) -> "LoraMetadata": """Create LoraMetadata instance from Civitai version info""" file_name = file_info.get("name", "") + base_name = strip_model_extension(file_name) base_model = determine_base_model(version_info.get("baseModel", "")) # Extract tags and description if available @@ -255,8 +278,8 @@ class LoraMetadata(BaseModelMetadata): sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower() return cls( - file_name=os.path.splitext(file_name)[0], - model_name=model_data.get("name", os.path.splitext(file_name)[0]), + file_name=base_name, + model_name=model_data.get("name", base_name), file_path=save_path.replace(os.sep, "/"), size=file_info.get("sizeKB", 0) * 1024, modified=datetime.now().timestamp(), @@ -285,6 +308,7 @@ class CheckpointMetadata(BaseModelMetadata): ) -> "CheckpointMetadata": """Create CheckpointMetadata instance from Civitai version info""" file_name = file_info.get("name", "") + base_name = strip_model_extension(file_name) base_model = determine_base_model(version_info.get("baseModel", "")) sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower() sub_type = version_info.get("type", "checkpoint") @@ -299,8 +323,8 @@ class CheckpointMetadata(BaseModelMetadata): description = model_data["description"] return cls( - file_name=os.path.splitext(file_name)[0], - model_name=model_data.get("name", os.path.splitext(file_name)[0]), + file_name=base_name, + model_name=model_data.get("name", base_name), file_path=save_path.replace(os.sep, "/"), size=file_info.get("sizeKB", 0) * 1024, modified=datetime.now().timestamp(), @@ -336,6 +360,7 @@ class OtherModelMetadata(BaseModelMetadata): ) -> "OtherModelMetadata": """Create OtherModelMetadata instance from Civitai version info""" file_name = file_info.get("name", "") + base_name = strip_model_extension(file_name) base_model = determine_base_model(version_info.get("baseModel", "")) sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower() # Map the CivitAI model type onto our sub_types; unknown types keep the @@ -354,8 +379,8 @@ class OtherModelMetadata(BaseModelMetadata): description = model_data["description"] return cls( - file_name=os.path.splitext(file_name)[0], - model_name=model_data.get("name", os.path.splitext(file_name)[0]), + file_name=base_name, + model_name=model_data.get("name", base_name), file_path=save_path.replace(os.sep, "/"), size=file_info.get("sizeKB", 0) * 1024, modified=datetime.now().timestamp(), @@ -385,6 +410,7 @@ class EmbeddingMetadata(BaseModelMetadata): ) -> "EmbeddingMetadata": """Create EmbeddingMetadata instance from Civitai version info""" file_name = file_info.get("name", "") + base_name = strip_model_extension(file_name) base_model = determine_base_model(version_info.get("baseModel", "")) sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower() sub_type = version_info.get("type", "embedding") @@ -399,8 +425,8 @@ class EmbeddingMetadata(BaseModelMetadata): description = model_data["description"] return cls( - file_name=os.path.splitext(file_name)[0], - model_name=model_data.get("name", os.path.splitext(file_name)[0]), + file_name=base_name, + model_name=model_data.get("name", base_name), file_path=save_path.replace(os.sep, "/"), size=file_info.get("sizeKB", 0) * 1024, modified=datetime.now().timestamp(), diff --git a/tests/services/test_civitai_info_migration_filename.py b/tests/services/test_civitai_info_migration_filename.py new file mode 100644 index 00000000..28930b51 --- /dev/null +++ b/tests/services/test_civitai_info_migration_filename.py @@ -0,0 +1,89 @@ +"""End-to-end regression for issue #1112. + +Importing a third-party ``.civitai.info`` sidecar (a migration path) must keep +the local, dotted file name intact instead of cutting it at the model-version +dot (``lora-sd1.5-backlight_slider_v10`` -> ``lora-sd1``). +""" + +import json +import os +from pathlib import Path + +import pytest + +from py.services import model_scanner +from py.services.lora_scanner import LoraScanner +from py.services.model_scanner import ModelScanner + +DOTTED_STEM = "lora-sd1.5-backlight_slider_v10" + +CIVITAI_INFO = { + "id": 12345, + "baseModel": "SD 1.5", + "name": "v1.0", + "model": { + "id": 999, + "name": "Light Control", + "type": "LORA", + "description": "", + "tags": ["lighting"], + }, + "files": [ + { + "id": 1, + # Remote name from the CivitAI payload; the local file was renamed. + "name": "backlight_slider_v10.safetensors", + "primary": True, + "sizeKB": 1024, + "hashes": {"SHA256": "a" * 64}, + } + ], +} + + +def _normalize(path: Path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture(autouse=True) +def reset_model_scanner_singletons(): + ModelScanner._instances.clear() + ModelScanner._locks.clear() + yield + ModelScanner._instances.clear() + ModelScanner._locks.clear() + + +@pytest.mark.asyncio +async def test_civitai_info_migration_keeps_dotted_local_name(tmp_path, monkeypatch): + loras_root = tmp_path / "loras" + loras_root.mkdir() + + model_file = loras_root / f"{DOTTED_STEM}.safetensors" + model_file.write_text("fake lora weights", encoding="utf-8") + (loras_root / f"{DOTTED_STEM}.civitai.info").write_text( + json.dumps(CIVITAI_INFO), encoding="utf-8" + ) + + normalized_root = _normalize(loras_root) + monkeypatch.setattr( + model_scanner.config, "loras_roots", [normalized_root], raising=False + ) + monkeypatch.setattr( + model_scanner.config, "extra_loras_roots", [], raising=False + ) + + scanner = LoraScanner() + entry = await scanner._process_model_file(_normalize(model_file), normalized_root) + + assert entry is not None + assert entry["file_name"] == DOTTED_STEM + assert entry["model_name"] == "Light Control" + + sidecar = loras_root / f"{DOTTED_STEM}.metadata.json" + assert sidecar.exists() + saved = json.loads(sidecar.read_text(encoding="utf-8")) + assert saved["file_name"] == DOTTED_STEM + assert saved["model_name"] == "Light Control" + # The migration must not silently drop the CivitAI payload. + assert saved["civitai"]["name"] == "v1.0" diff --git a/tests/services/test_reconcile_filename_repair.py b/tests/services/test_reconcile_filename_repair.py new file mode 100644 index 00000000..c0e7a84e --- /dev/null +++ b/tests/services/test_reconcile_filename_repair.py @@ -0,0 +1,148 @@ +"""Regression tests for the reconcile-time ``file_name`` repair (issue #1112). + +Libraries already corrupted by the legacy ``.civitai.info`` migration keep the +truncated name in their sidecars and SQLite snapshot: the incremental refresh +skips cached paths and startup hydrates rows as-is, so the wrong name never +heals. A plain Refresh must repair those rows in place, exactly once. +""" + +import json +import os +from pathlib import Path + +import pytest + +from py.services import model_scanner +from py.services.lora_scanner import LoraScanner +from py.services.model_cache import ModelCache +from py.services.model_scanner import ModelScanner +from py.services.persistent_model_cache import PersistentModelCache +from py.utils.metadata_manager import MetadataManager + +DOTTED_STEM = "lora-sd1.5-backlight_slider_v10" +TRUNCATED_STEM = "lora-sd1" + + +def _normalize(path: Path) -> str: + return str(path).replace(os.sep, "/") + + +@pytest.fixture(autouse=True) +def reset_model_scanner_singletons(): + ModelScanner._instances.clear() + ModelScanner._locks.clear() + yield + ModelScanner._instances.clear() + ModelScanner._locks.clear() + + +async def _prepare_corrupted_library(tmp_path: Path, monkeypatch): + """Build a library whose sidecar + cache carry the truncated stem.""" + loras_root = tmp_path / "loras" + loras_root.mkdir() + model_file = loras_root / f"{DOTTED_STEM}.safetensors" + model_file.write_text("fake lora weights", encoding="utf-8") + + normalized_root = _normalize(loras_root) + monkeypatch.setattr( + model_scanner.config, "loras_roots", [normalized_root], raising=False + ) + monkeypatch.setattr( + model_scanner.config, "extra_loras_roots", [], raising=False + ) + + scanner = LoraScanner() + normalized_file = _normalize(model_file) + entry = await scanner._process_model_file(normalized_file, normalized_root) + assert entry is not None + + # Simulate the pre-fix migration output: sidecar and cache both hold the + # stem cut at the "1.5" dot. + metadata, _ = await MetadataManager.load_metadata(normalized_file, scanner.model_class) + metadata.file_name = TRUNCATED_STEM + await MetadataManager.save_metadata(normalized_file, metadata) + entry["file_name"] = TRUNCATED_STEM + + scanner._cache = ModelCache( + raw_data=[entry], + folders=[], + all_folders=[], + name_display_mode="file_name", + ) + scanner._persistent_cache = PersistentModelCache( + library_name="test", db_path=str(tmp_path / "models.sqlite") + ) + return scanner, model_file, entry + + +@pytest.mark.asyncio +async def test_reconcile_repairs_truncated_file_name(tmp_path, monkeypatch): + scanner, model_file, entry = await _prepare_corrupted_library(tmp_path, monkeypatch) + assert entry["file_name"] == TRUNCATED_STEM + + await scanner._reconcile_cache() + + # In-memory cache entry. + assert entry["file_name"] == DOTTED_STEM + + # On-disk sidecar (repaired by MetadataManager normalization). + sidecar = model_file.parent / f"{DOTTED_STEM}.metadata.json" + saved = json.loads(sidecar.read_text(encoding="utf-8")) + assert saved["file_name"] == DOTTED_STEM + + # Targeted SQL delta, so the fix survives a restart. + persisted = scanner._persistent_cache.load_cache(scanner.model_type) + assert persisted is not None + rows = [i for i in persisted.raw_data if i["file_path"] == _normalize(model_file)] + assert len(rows) == 1 + assert rows[0]["file_name"] == DOTTED_STEM + + +@pytest.mark.asyncio +async def test_reconcile_repair_runs_once(tmp_path, monkeypatch): + scanner, _model_file, _entry = await _prepare_corrupted_library( + tmp_path, monkeypatch + ) + + calls = {"sync": 0, "load": 0} + original_sync = scanner._sync_cache_from_metadata_impl + original_load = MetadataManager.load_metadata + + async def counting_sync(file_path, metadata_dict): + calls["sync"] += 1 + return await original_sync(file_path, metadata_dict) + + async def counting_load(*args, **kwargs): + calls["load"] += 1 + return await original_load(*args, **kwargs) + + monkeypatch.setattr(scanner, "_sync_cache_from_metadata_impl", counting_sync) + monkeypatch.setattr(MetadataManager, "load_metadata", counting_load) + + await scanner._reconcile_cache() + assert calls == {"sync": 1, "load": 1} + + # The mismatch is gone, so a second refresh must not touch metadata again. + await scanner._reconcile_cache() + assert calls == {"sync": 1, "load": 1} + + +@pytest.mark.asyncio +async def test_reconcile_clean_library_never_reads_metadata(tmp_path, monkeypatch): + """The repair probe must cost one string compare, not a metadata read.""" + scanner, _model_file, _entry = await _prepare_corrupted_library( + tmp_path, monkeypatch + ) + await scanner._reconcile_cache() + + calls = {"load": 0} + original_load = MetadataManager.load_metadata + + async def counting_load(*args, **kwargs): + calls["load"] += 1 + return await original_load(*args, **kwargs) + + monkeypatch.setattr(MetadataManager, "load_metadata", counting_load) + + await scanner._reconcile_cache() + assert calls["load"] == 0 diff --git a/tests/utils/test_model_filename_extension.py b/tests/utils/test_model_filename_extension.py new file mode 100644 index 00000000..7a8a2f13 --- /dev/null +++ b/tests/utils/test_model_filename_extension.py @@ -0,0 +1,106 @@ +"""Regression tests for issue #1112. + +Dotted model file names (``lora-sd1.5-backlight_slider_v10.safetensors``) must +not be truncated when metadata is built from a CivitAI payload. The builder has +to strip at most one *known* model extension: API filenames keep their +extension, while migration paths (``.civitai.info``) pass the local stem. +""" + +import pytest + +from py.utils.models import ( + CheckpointMetadata, + EmbeddingMetadata, + LoraMetadata, + OtherModelMetadata, + strip_model_extension, +) + +DOTTED_STEM = "lora-sd1.5-backlight_slider_v10" + +MODEL_CLASSES = [ + LoraMetadata, + CheckpointMetadata, + EmbeddingMetadata, + OtherModelMetadata, +] + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + # Dotted stems are not extensions. + (DOTTED_STEM, DOTTED_STEM), + (f"{DOTTED_STEM}.safetensors", DOTTED_STEM), + ("model.v1.5.safetensors", "model.v1.5"), + ("a.b.c", "a.b.c"), + # Every scanner extension is recognized, case-insensitively. + ("weights.GGUF", "weights"), + ("weights.pt2", "weights"), + ("weights.ckpt", "weights"), + # Extension-free plain names are unchanged. + ("plain_name", "plain_name"), + ("", ""), + ], +) +def test_strip_model_extension_strips_only_known_extensions(raw, expected): + assert strip_model_extension(raw) == expected + + +@pytest.mark.parametrize("model_cls", MODEL_CLASSES) +@pytest.mark.parametrize( + "file_name", + [ + f"{DOTTED_STEM}.safetensors", # CivitAI API / download shape + DOTTED_STEM, # .civitai.info migration shape (already extension-free) + ], +) +def test_from_civitai_info_keeps_dotted_stem(model_cls, file_name): + version_info = { + "baseModel": "SD 1.5", + "name": "v1.0", + "model": {"name": "Light Control", "description": "", "tags": []}, + } + file_info = {"name": file_name, "sizeKB": 1024, "hashes": {"SHA256": "a" * 64}} + + metadata = model_cls.from_civitai_info( + version_info, file_info, f"/models/{DOTTED_STEM}.safetensors" + ) + + assert metadata.file_name == DOTTED_STEM + assert metadata.model_name == "Light Control" + + +@pytest.mark.parametrize("model_cls", MODEL_CLASSES) +def test_from_civitai_info_model_name_fallback_uses_full_stem(model_cls): + """A sidecar without ``model.name`` must fall back to the full local stem.""" + version_info = {"baseModel": "SD 1.5", "model": {"description": "", "tags": []}} + file_info = {"name": DOTTED_STEM, "sizeKB": 1024, "hashes": {}} + + metadata = model_cls.from_civitai_info( + version_info, file_info, f"/models/{DOTTED_STEM}.safetensors" + ) + + assert metadata.file_name == DOTTED_STEM + assert metadata.model_name == DOTTED_STEM + + +@pytest.mark.parametrize("model_cls", MODEL_CLASSES) +def test_from_civitai_info_model_name_still_wins_over_stem(model_cls): + """The CivitAI model name stays authoritative when present.""" + version_info = { + "baseModel": "SDXL", + "model": {"name": "Chiaroscuro Light", "description": "", "tags": ["light"]}, + } + file_info = { + "name": f"{DOTTED_STEM}.safetensors", + "sizeKB": 1024, + "hashes": {}, + } + + metadata = model_cls.from_civitai_info( + version_info, file_info, f"/models/{DOTTED_STEM}.safetensors" + ) + + assert metadata.file_name == DOTTED_STEM + assert metadata.model_name == "Chiaroscuro Light"