fix(scanner): stop truncating dotted model file names (#1112)

A LoRA named `lora-sd1.5-backlight_slider_v10.safetensors` showed up in the
manager as `lora-sd1`, hid itself from searches for the rest of its name, and
collapsed into the same lora syntax tag as every sibling sharing the prefix.

The name was cut twice.  `_process_model_file()` imports a third-party
`.civitai.info` sidecar by handing `from_civitai_info()` the local stem with
the extension already stripped, and the builder then stripped a second
"extension" from it -- `os.path.splitext` reads everything after the last dot
as one, so the version dot in `1.5` ended the name.  The download path never
hit this because API filenames keep their extension and only need one strip.

Pass the real basename from the migration site, and make the builder strip
only a recognized model extension (`strip_model_extension`), so both input
shapes resolve to the same stem.  The `model_name` fallback that reused the
same expression is fixed with it: on a sidecar without `model.name` the
display name was truncated too.

Libraries already corrupted do not heal on their own: the incremental Refresh
skips paths already in the cache (only a full rebuild reloads metadata) and
startup hydrates rows from SQLite as-is, so the wrong name survives restarts.
Reconcile now compares each cached row against the stem of its file path --
one string compare per file and no extra syscall, so a clean library pays
nothing -- and repairs mismatching rows through `load_metadata()` (which
normalizes the sidecar) and the existing in-place `_sync_cache_from_metadata_impl()`
path, which writes a targeted single-row SQL delta instead of a full save.
Repairs are one-shot, and a missing or corrupt sidecar keeps its row so a full
rebuild can recreate it without losing tags or civitai data.

Tests: the builder keeps dotted stems for all four model classes and still
strips real extensions; the migration writes the full local name to the
sidecar; and reconcile repairs memory, sidecar and SQLite row, runs exactly
once, and never reads metadata on a clean library.
This commit is contained in:
Will Miao
2026-09-15 09:07:56 +08:00
parent 942717f0b6
commit 2ceb1e2850
5 changed files with 475 additions and 15 deletions
+97 -6
View File
@@ -63,6 +63,15 @@ def _is_hidden_relative_path(rel_path: str) -> bool:
return any(part.startswith(".") for part in rel_path.replace(os.sep, "/").split("/"))
def _file_name_stem(file_path: str) -> str:
"""Return the extension-free file name of a normalized model path.
``file_name`` cache/sidecar fields are defined as the on-disk stem, so this
is the authoritative value to compare stored names against (issue #1112).
"""
return os.path.splitext(os.path.basename(file_path))[0]
# Maps a scanner model type to the manager page type used in progress
# broadcasts (e.g. 'lora' -> 'loras').
PAGE_TYPE_MAP = {
@@ -1076,6 +1085,26 @@ class ModelScanner:
# Track found files and new files
found_paths = set()
new_files = []
# Cached entries whose stored file_name no longer matches the file
# on disk (e.g. dotted stems truncated by the legacy .civitai.info
# migration, issue #1112). Repaired in place after the walk; the
# list stays empty on a clean library, so a no-change reconcile
# only pays one string compare per cached file.
stale_paths: List[str] = []
stale_seen: Set[str] = set()
def mark_stale_if_needed(cached_path: str) -> None:
"""Queue a cached path for file_name repair when it drifted."""
if cached_path in stale_seen:
return
item = path_to_item.get(cached_path)
if item is None:
return
if item.get("file_name") == _file_name_stem(cached_path):
return
stale_seen.add(cached_path)
stale_paths.append(cached_path)
visited_real_paths = set()
discovered_real_files = set()
discovered_folders: Set[str] = set()
@@ -1110,6 +1139,7 @@ class ModelScanner:
# Check if this file is already in cache
if file_path in cached_paths:
found_paths.add(file_path)
mark_stale_if_needed(file_path)
continue
# Only a cache miss needs the physical path, so the
@@ -1120,6 +1150,7 @@ class ModelScanner:
cached_real_match = lookup_cached_real_path(real_file_path)
if cached_real_match:
found_paths.add(cached_real_match)
mark_stale_if_needed(cached_real_match)
continue
if file_path in self._excluded_models:
@@ -1132,6 +1163,7 @@ class ModelScanner:
for cached_path in cached_paths:
if cached_path.lower() == lower_path:
found_paths.add(cached_path)
mark_stale_if_needed(cached_path)
matched = True
break
if matched:
@@ -1242,7 +1274,57 @@ class ModelScanner:
elapsed_seconds=time.time() - start_time,
)
return
# Repair rows whose file_name drifted from the file on disk. Only
# mismatching entries are re-read here, so a clean library never
# touches metadata during a refresh. Each repair goes through the
# single-row update path: load_metadata() normalizes the sidecar
# (MetadataManager._normalize_metadata_paths) and
# _sync_cache_from_metadata_impl() rewrites one targeted SQL delta
# instead of a full cache save, and the mismatch is gone
# afterwards, so the work never repeats (issue #1112).
total_repaired = 0
if stale_paths:
logger.info(
"%s Scanner: Repairing %d cached entries whose file_name no longer matches the file on disk",
self.model_type.capitalize(),
len(stale_paths),
)
for path in stale_paths:
if self.is_cancelled():
logger.info(f"{self.model_type.capitalize()} Scanner: Reconcile repair cancelled")
break
try:
metadata, _should_skip = await MetadataManager.load_metadata(
path, self.model_class
)
if metadata is None:
# Missing or corrupt sidecar: keep the existing row
# so a full rebuild can recreate the metadata from
# .civitai.info (or defaults) without losing cached
# fields such as tags or civitai data.
logger.debug(
"%s Scanner: Leaving %s unchanged (no usable metadata to repair from)",
self.model_type.capitalize(),
path,
)
continue
payload = metadata.to_dict()
unknown_fields = getattr(metadata, "_unknown_fields", None)
if isinstance(unknown_fields, dict):
payload.update(unknown_fields)
if await self._sync_cache_from_metadata_impl(path, payload):
total_repaired += 1
except Exception as exc:
logger.warning(
"%s Scanner: Failed to repair file_name for %s: %s",
self.model_type.capitalize(),
path,
exc,
)
# Find missing files (in cache but not in filesystem)
missing_files = cached_paths - found_paths
total_removed = 0
@@ -1323,7 +1405,11 @@ class ModelScanner:
elif folders_changed:
await self._persist_current_cache()
logger.info(f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in {time.time() - start_time:.2f} seconds. Added {total_added}, removed {total_removed} models.")
logger.info(
f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in "
f"{time.time() - start_time:.2f} seconds. Added {total_added}, "
f"removed {total_removed}, repaired {total_repaired} models."
)
await self._broadcast_scan_progress(
'completed', 'process_new', 100, False,
added=total_added, removed=total_removed,
@@ -1552,11 +1638,16 @@ class ModelScanner:
file_info = next((f for f in version_info.get('files', []) if f.get('primary')), None)
if file_info:
file_name = os.path.splitext(os.path.basename(file_path))[0]
file_info['name'] = file_name
local_stem = os.path.splitext(os.path.basename(file_path))[0]
# from_civitai_info expects an API-shaped file entry and
# strips one extension itself, so hand it the real
# basename: passing the already extension-free stem made
# it cut dotted names at their last dot ("lora-sd1.5-..."
# became "lora-sd1", issue #1112).
file_info['name'] = os.path.basename(file_path)
metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path)
metadata.preview_url = find_preview_file(file_name, os.path.dirname(file_path))
metadata.preview_url = find_preview_file(local_stem, os.path.dirname(file_path))
await MetadataManager.save_metadata(file_path, metadata)
logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)")
except Exception as e:
+35 -9
View File
@@ -2,7 +2,11 @@ from dataclasses import dataclass, asdict, field
from typing import Callable, Dict, Optional, List, Any
from datetime import datetime
import os
from .constants import CIVITAI_TYPE_TO_OTHER_SUB_TYPE, INVALID_AUTOV3_EMPTY_HASH
from .constants import (
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
INVALID_AUTOV3_EMPTY_HASH,
MODEL_FILE_EXTENSIONS,
)
from .model_utils import determine_base_model
@@ -46,6 +50,24 @@ def autov3_from_civitai_files(civitai_data: Optional[Dict[str, Any]], sha256: st
return None
def strip_model_extension(file_name: str) -> str:
"""Strip a recognized model file extension, leaving dotted stems intact.
``os.path.splitext`` treats everything after the last dot as an extension,
so applying it to an already extension-free name truncates dotted stems:
``lora-sd1.5-backlight_slider_v10`` becomes ``lora-sd1``. API filenames keep
their extension and need one strip, while migration paths (``.civitai.info``)
pass the local stem as-is, so only remove a suffix that is a known model
extension and both inputs resolve to the same stem (issue #1112).
"""
if not file_name:
return file_name
stem, extension = os.path.splitext(file_name)
if extension.lower() in MODEL_FILE_EXTENSIONS:
return stem
return file_name
@dataclass
class BaseModelMetadata:
"""Base class for all model metadata structures"""
@@ -241,6 +263,7 @@ class LoraMetadata(BaseModelMetadata):
) -> "LoraMetadata":
"""Create LoraMetadata instance from Civitai version info"""
file_name = file_info.get("name", "")
base_name = strip_model_extension(file_name)
base_model = determine_base_model(version_info.get("baseModel", ""))
# Extract tags and description if available
@@ -255,8 +278,8 @@ class LoraMetadata(BaseModelMetadata):
sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower()
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_name=base_name,
model_name=model_data.get("name", base_name),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
@@ -285,6 +308,7 @@ class CheckpointMetadata(BaseModelMetadata):
) -> "CheckpointMetadata":
"""Create CheckpointMetadata instance from Civitai version info"""
file_name = file_info.get("name", "")
base_name = strip_model_extension(file_name)
base_model = determine_base_model(version_info.get("baseModel", ""))
sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower()
sub_type = version_info.get("type", "checkpoint")
@@ -299,8 +323,8 @@ class CheckpointMetadata(BaseModelMetadata):
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_name=base_name,
model_name=model_data.get("name", base_name),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
@@ -336,6 +360,7 @@ class OtherModelMetadata(BaseModelMetadata):
) -> "OtherModelMetadata":
"""Create OtherModelMetadata instance from Civitai version info"""
file_name = file_info.get("name", "")
base_name = strip_model_extension(file_name)
base_model = determine_base_model(version_info.get("baseModel", ""))
sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower()
# Map the CivitAI model type onto our sub_types; unknown types keep the
@@ -354,8 +379,8 @@ class OtherModelMetadata(BaseModelMetadata):
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_name=base_name,
model_name=model_data.get("name", base_name),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
@@ -385,6 +410,7 @@ class EmbeddingMetadata(BaseModelMetadata):
) -> "EmbeddingMetadata":
"""Create EmbeddingMetadata instance from Civitai version info"""
file_name = file_info.get("name", "")
base_name = strip_model_extension(file_name)
base_model = determine_base_model(version_info.get("baseModel", ""))
sha256_value = (file_info.get("hashes") or {}).get("SHA256", "").lower()
sub_type = version_info.get("type", "embedding")
@@ -399,8 +425,8 @@ class EmbeddingMetadata(BaseModelMetadata):
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_name=base_name,
model_name=model_data.get("name", base_name),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
@@ -0,0 +1,89 @@
"""End-to-end regression for issue #1112.
Importing a third-party ``.civitai.info`` sidecar (a migration path) must keep
the local, dotted file name intact instead of cutting it at the model-version
dot (``lora-sd1.5-backlight_slider_v10`` -> ``lora-sd1``).
"""
import json
import os
from pathlib import Path
import pytest
from py.services import model_scanner
from py.services.lora_scanner import LoraScanner
from py.services.model_scanner import ModelScanner
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
CIVITAI_INFO = {
"id": 12345,
"baseModel": "SD 1.5",
"name": "v1.0",
"model": {
"id": 999,
"name": "Light Control",
"type": "LORA",
"description": "",
"tags": ["lighting"],
},
"files": [
{
"id": 1,
# Remote name from the CivitAI payload; the local file was renamed.
"name": "backlight_slider_v10.safetensors",
"primary": True,
"sizeKB": 1024,
"hashes": {"SHA256": "a" * 64},
}
],
}
def _normalize(path: Path) -> str:
return str(path).replace(os.sep, "/")
@pytest.fixture(autouse=True)
def reset_model_scanner_singletons():
ModelScanner._instances.clear()
ModelScanner._locks.clear()
yield
ModelScanner._instances.clear()
ModelScanner._locks.clear()
@pytest.mark.asyncio
async def test_civitai_info_migration_keeps_dotted_local_name(tmp_path, monkeypatch):
loras_root = tmp_path / "loras"
loras_root.mkdir()
model_file = loras_root / f"{DOTTED_STEM}.safetensors"
model_file.write_text("fake lora weights", encoding="utf-8")
(loras_root / f"{DOTTED_STEM}.civitai.info").write_text(
json.dumps(CIVITAI_INFO), encoding="utf-8"
)
normalized_root = _normalize(loras_root)
monkeypatch.setattr(
model_scanner.config, "loras_roots", [normalized_root], raising=False
)
monkeypatch.setattr(
model_scanner.config, "extra_loras_roots", [], raising=False
)
scanner = LoraScanner()
entry = await scanner._process_model_file(_normalize(model_file), normalized_root)
assert entry is not None
assert entry["file_name"] == DOTTED_STEM
assert entry["model_name"] == "Light Control"
sidecar = loras_root / f"{DOTTED_STEM}.metadata.json"
assert sidecar.exists()
saved = json.loads(sidecar.read_text(encoding="utf-8"))
assert saved["file_name"] == DOTTED_STEM
assert saved["model_name"] == "Light Control"
# The migration must not silently drop the CivitAI payload.
assert saved["civitai"]["name"] == "v1.0"
@@ -0,0 +1,148 @@
"""Regression tests for the reconcile-time ``file_name`` repair (issue #1112).
Libraries already corrupted by the legacy ``.civitai.info`` migration keep the
truncated name in their sidecars and SQLite snapshot: the incremental refresh
skips cached paths and startup hydrates rows as-is, so the wrong name never
heals. A plain Refresh must repair those rows in place, exactly once.
"""
import json
import os
from pathlib import Path
import pytest
from py.services import model_scanner
from py.services.lora_scanner import LoraScanner
from py.services.model_cache import ModelCache
from py.services.model_scanner import ModelScanner
from py.services.persistent_model_cache import PersistentModelCache
from py.utils.metadata_manager import MetadataManager
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
TRUNCATED_STEM = "lora-sd1"
def _normalize(path: Path) -> str:
return str(path).replace(os.sep, "/")
@pytest.fixture(autouse=True)
def reset_model_scanner_singletons():
ModelScanner._instances.clear()
ModelScanner._locks.clear()
yield
ModelScanner._instances.clear()
ModelScanner._locks.clear()
async def _prepare_corrupted_library(tmp_path: Path, monkeypatch):
"""Build a library whose sidecar + cache carry the truncated stem."""
loras_root = tmp_path / "loras"
loras_root.mkdir()
model_file = loras_root / f"{DOTTED_STEM}.safetensors"
model_file.write_text("fake lora weights", encoding="utf-8")
normalized_root = _normalize(loras_root)
monkeypatch.setattr(
model_scanner.config, "loras_roots", [normalized_root], raising=False
)
monkeypatch.setattr(
model_scanner.config, "extra_loras_roots", [], raising=False
)
scanner = LoraScanner()
normalized_file = _normalize(model_file)
entry = await scanner._process_model_file(normalized_file, normalized_root)
assert entry is not None
# Simulate the pre-fix migration output: sidecar and cache both hold the
# stem cut at the "1.5" dot.
metadata, _ = await MetadataManager.load_metadata(normalized_file, scanner.model_class)
metadata.file_name = TRUNCATED_STEM
await MetadataManager.save_metadata(normalized_file, metadata)
entry["file_name"] = TRUNCATED_STEM
scanner._cache = ModelCache(
raw_data=[entry],
folders=[],
all_folders=[],
name_display_mode="file_name",
)
scanner._persistent_cache = PersistentModelCache(
library_name="test", db_path=str(tmp_path / "models.sqlite")
)
return scanner, model_file, entry
@pytest.mark.asyncio
async def test_reconcile_repairs_truncated_file_name(tmp_path, monkeypatch):
scanner, model_file, entry = await _prepare_corrupted_library(tmp_path, monkeypatch)
assert entry["file_name"] == TRUNCATED_STEM
await scanner._reconcile_cache()
# In-memory cache entry.
assert entry["file_name"] == DOTTED_STEM
# On-disk sidecar (repaired by MetadataManager normalization).
sidecar = model_file.parent / f"{DOTTED_STEM}.metadata.json"
saved = json.loads(sidecar.read_text(encoding="utf-8"))
assert saved["file_name"] == DOTTED_STEM
# Targeted SQL delta, so the fix survives a restart.
persisted = scanner._persistent_cache.load_cache(scanner.model_type)
assert persisted is not None
rows = [i for i in persisted.raw_data if i["file_path"] == _normalize(model_file)]
assert len(rows) == 1
assert rows[0]["file_name"] == DOTTED_STEM
@pytest.mark.asyncio
async def test_reconcile_repair_runs_once(tmp_path, monkeypatch):
scanner, _model_file, _entry = await _prepare_corrupted_library(
tmp_path, monkeypatch
)
calls = {"sync": 0, "load": 0}
original_sync = scanner._sync_cache_from_metadata_impl
original_load = MetadataManager.load_metadata
async def counting_sync(file_path, metadata_dict):
calls["sync"] += 1
return await original_sync(file_path, metadata_dict)
async def counting_load(*args, **kwargs):
calls["load"] += 1
return await original_load(*args, **kwargs)
monkeypatch.setattr(scanner, "_sync_cache_from_metadata_impl", counting_sync)
monkeypatch.setattr(MetadataManager, "load_metadata", counting_load)
await scanner._reconcile_cache()
assert calls == {"sync": 1, "load": 1}
# The mismatch is gone, so a second refresh must not touch metadata again.
await scanner._reconcile_cache()
assert calls == {"sync": 1, "load": 1}
@pytest.mark.asyncio
async def test_reconcile_clean_library_never_reads_metadata(tmp_path, monkeypatch):
"""The repair probe must cost one string compare, not a metadata read."""
scanner, _model_file, _entry = await _prepare_corrupted_library(
tmp_path, monkeypatch
)
await scanner._reconcile_cache()
calls = {"load": 0}
original_load = MetadataManager.load_metadata
async def counting_load(*args, **kwargs):
calls["load"] += 1
return await original_load(*args, **kwargs)
monkeypatch.setattr(MetadataManager, "load_metadata", counting_load)
await scanner._reconcile_cache()
assert calls["load"] == 0
@@ -0,0 +1,106 @@
"""Regression tests for issue #1112.
Dotted model file names (``lora-sd1.5-backlight_slider_v10.safetensors``) must
not be truncated when metadata is built from a CivitAI payload. The builder has
to strip at most one *known* model extension: API filenames keep their
extension, while migration paths (``.civitai.info``) pass the local stem.
"""
import pytest
from py.utils.models import (
CheckpointMetadata,
EmbeddingMetadata,
LoraMetadata,
OtherModelMetadata,
strip_model_extension,
)
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
MODEL_CLASSES = [
LoraMetadata,
CheckpointMetadata,
EmbeddingMetadata,
OtherModelMetadata,
]
@pytest.mark.parametrize(
("raw", "expected"),
[
# Dotted stems are not extensions.
(DOTTED_STEM, DOTTED_STEM),
(f"{DOTTED_STEM}.safetensors", DOTTED_STEM),
("model.v1.5.safetensors", "model.v1.5"),
("a.b.c", "a.b.c"),
# Every scanner extension is recognized, case-insensitively.
("weights.GGUF", "weights"),
("weights.pt2", "weights"),
("weights.ckpt", "weights"),
# Extension-free plain names are unchanged.
("plain_name", "plain_name"),
("", ""),
],
)
def test_strip_model_extension_strips_only_known_extensions(raw, expected):
assert strip_model_extension(raw) == expected
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
@pytest.mark.parametrize(
"file_name",
[
f"{DOTTED_STEM}.safetensors", # CivitAI API / download shape
DOTTED_STEM, # .civitai.info migration shape (already extension-free)
],
)
def test_from_civitai_info_keeps_dotted_stem(model_cls, file_name):
version_info = {
"baseModel": "SD 1.5",
"name": "v1.0",
"model": {"name": "Light Control", "description": "", "tags": []},
}
file_info = {"name": file_name, "sizeKB": 1024, "hashes": {"SHA256": "a" * 64}}
metadata = model_cls.from_civitai_info(
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
)
assert metadata.file_name == DOTTED_STEM
assert metadata.model_name == "Light Control"
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
def test_from_civitai_info_model_name_fallback_uses_full_stem(model_cls):
"""A sidecar without ``model.name`` must fall back to the full local stem."""
version_info = {"baseModel": "SD 1.5", "model": {"description": "", "tags": []}}
file_info = {"name": DOTTED_STEM, "sizeKB": 1024, "hashes": {}}
metadata = model_cls.from_civitai_info(
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
)
assert metadata.file_name == DOTTED_STEM
assert metadata.model_name == DOTTED_STEM
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
def test_from_civitai_info_model_name_still_wins_over_stem(model_cls):
"""The CivitAI model name stays authoritative when present."""
version_info = {
"baseModel": "SDXL",
"model": {"name": "Chiaroscuro Light", "description": "", "tags": ["light"]},
}
file_info = {
"name": f"{DOTTED_STEM}.safetensors",
"sizeKB": 1024,
"hashes": {},
}
metadata = model_cls.from_civitai_info(
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
)
assert metadata.file_name == DOTTED_STEM
assert metadata.model_name == "Chiaroscuro Light"