mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-20 18:51:26 -03:00
fix(scanner): stop truncating dotted model file names (#1112)
A LoRA named `lora-sd1.5-backlight_slider_v10.safetensors` showed up in the manager as `lora-sd1`, hid itself from searches for the rest of its name, and collapsed into the same lora syntax tag as every sibling sharing the prefix. The name was cut twice. `_process_model_file()` imports a third-party `.civitai.info` sidecar by handing `from_civitai_info()` the local stem with the extension already stripped, and the builder then stripped a second "extension" from it -- `os.path.splitext` reads everything after the last dot as one, so the version dot in `1.5` ended the name. The download path never hit this because API filenames keep their extension and only need one strip. Pass the real basename from the migration site, and make the builder strip only a recognized model extension (`strip_model_extension`), so both input shapes resolve to the same stem. The `model_name` fallback that reused the same expression is fixed with it: on a sidecar without `model.name` the display name was truncated too. Libraries already corrupted do not heal on their own: the incremental Refresh skips paths already in the cache (only a full rebuild reloads metadata) and startup hydrates rows from SQLite as-is, so the wrong name survives restarts. Reconcile now compares each cached row against the stem of its file path -- one string compare per file and no extra syscall, so a clean library pays nothing -- and repairs mismatching rows through `load_metadata()` (which normalizes the sidecar) and the existing in-place `_sync_cache_from_metadata_impl()` path, which writes a targeted single-row SQL delta instead of a full save. Repairs are one-shot, and a missing or corrupt sidecar keeps its row so a full rebuild can recreate it without losing tags or civitai data. Tests: the builder keeps dotted stems for all four model classes and still strips real extensions; the migration writes the full local name to the sidecar; and reconcile repairs memory, sidecar and SQLite row, runs exactly once, and never reads metadata on a clean library.
This commit is contained in:
Will Miao
@@ -0,0 +1,89 @@
|
||||
"""End-to-end regression for issue #1112.
|
||||
|
||||
Importing a third-party ``.civitai.info`` sidecar (a migration path) must keep
|
||||
the local, dotted file name intact instead of cutting it at the model-version
|
||||
dot (``lora-sd1.5-backlight_slider_v10`` -> ``lora-sd1``).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from py.services import model_scanner
|
||||
from py.services.lora_scanner import LoraScanner
|
||||
from py.services.model_scanner import ModelScanner
|
||||
|
||||
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
|
||||
|
||||
CIVITAI_INFO = {
|
||||
"id": 12345,
|
||||
"baseModel": "SD 1.5",
|
||||
"name": "v1.0",
|
||||
"model": {
|
||||
"id": 999,
|
||||
"name": "Light Control",
|
||||
"type": "LORA",
|
||||
"description": "",
|
||||
"tags": ["lighting"],
|
||||
},
|
||||
"files": [
|
||||
{
|
||||
"id": 1,
|
||||
# Remote name from the CivitAI payload; the local file was renamed.
|
||||
"name": "backlight_slider_v10.safetensors",
|
||||
"primary": True,
|
||||
"sizeKB": 1024,
|
||||
"hashes": {"SHA256": "a" * 64},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _normalize(path: Path) -> str:
|
||||
return str(path).replace(os.sep, "/")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_model_scanner_singletons():
|
||||
ModelScanner._instances.clear()
|
||||
ModelScanner._locks.clear()
|
||||
yield
|
||||
ModelScanner._instances.clear()
|
||||
ModelScanner._locks.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_civitai_info_migration_keeps_dotted_local_name(tmp_path, monkeypatch):
|
||||
loras_root = tmp_path / "loras"
|
||||
loras_root.mkdir()
|
||||
|
||||
model_file = loras_root / f"{DOTTED_STEM}.safetensors"
|
||||
model_file.write_text("fake lora weights", encoding="utf-8")
|
||||
(loras_root / f"{DOTTED_STEM}.civitai.info").write_text(
|
||||
json.dumps(CIVITAI_INFO), encoding="utf-8"
|
||||
)
|
||||
|
||||
normalized_root = _normalize(loras_root)
|
||||
monkeypatch.setattr(
|
||||
model_scanner.config, "loras_roots", [normalized_root], raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_scanner.config, "extra_loras_roots", [], raising=False
|
||||
)
|
||||
|
||||
scanner = LoraScanner()
|
||||
entry = await scanner._process_model_file(_normalize(model_file), normalized_root)
|
||||
|
||||
assert entry is not None
|
||||
assert entry["file_name"] == DOTTED_STEM
|
||||
assert entry["model_name"] == "Light Control"
|
||||
|
||||
sidecar = loras_root / f"{DOTTED_STEM}.metadata.json"
|
||||
assert sidecar.exists()
|
||||
saved = json.loads(sidecar.read_text(encoding="utf-8"))
|
||||
assert saved["file_name"] == DOTTED_STEM
|
||||
assert saved["model_name"] == "Light Control"
|
||||
# The migration must not silently drop the CivitAI payload.
|
||||
assert saved["civitai"]["name"] == "v1.0"
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Regression tests for the reconcile-time ``file_name`` repair (issue #1112).
|
||||
|
||||
Libraries already corrupted by the legacy ``.civitai.info`` migration keep the
|
||||
truncated name in their sidecars and SQLite snapshot: the incremental refresh
|
||||
skips cached paths and startup hydrates rows as-is, so the wrong name never
|
||||
heals. A plain Refresh must repair those rows in place, exactly once.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from py.services import model_scanner
|
||||
from py.services.lora_scanner import LoraScanner
|
||||
from py.services.model_cache import ModelCache
|
||||
from py.services.model_scanner import ModelScanner
|
||||
from py.services.persistent_model_cache import PersistentModelCache
|
||||
from py.utils.metadata_manager import MetadataManager
|
||||
|
||||
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
|
||||
TRUNCATED_STEM = "lora-sd1"
|
||||
|
||||
|
||||
def _normalize(path: Path) -> str:
|
||||
return str(path).replace(os.sep, "/")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_model_scanner_singletons():
|
||||
ModelScanner._instances.clear()
|
||||
ModelScanner._locks.clear()
|
||||
yield
|
||||
ModelScanner._instances.clear()
|
||||
ModelScanner._locks.clear()
|
||||
|
||||
|
||||
async def _prepare_corrupted_library(tmp_path: Path, monkeypatch):
|
||||
"""Build a library whose sidecar + cache carry the truncated stem."""
|
||||
loras_root = tmp_path / "loras"
|
||||
loras_root.mkdir()
|
||||
model_file = loras_root / f"{DOTTED_STEM}.safetensors"
|
||||
model_file.write_text("fake lora weights", encoding="utf-8")
|
||||
|
||||
normalized_root = _normalize(loras_root)
|
||||
monkeypatch.setattr(
|
||||
model_scanner.config, "loras_roots", [normalized_root], raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_scanner.config, "extra_loras_roots", [], raising=False
|
||||
)
|
||||
|
||||
scanner = LoraScanner()
|
||||
normalized_file = _normalize(model_file)
|
||||
entry = await scanner._process_model_file(normalized_file, normalized_root)
|
||||
assert entry is not None
|
||||
|
||||
# Simulate the pre-fix migration output: sidecar and cache both hold the
|
||||
# stem cut at the "1.5" dot.
|
||||
metadata, _ = await MetadataManager.load_metadata(normalized_file, scanner.model_class)
|
||||
metadata.file_name = TRUNCATED_STEM
|
||||
await MetadataManager.save_metadata(normalized_file, metadata)
|
||||
entry["file_name"] = TRUNCATED_STEM
|
||||
|
||||
scanner._cache = ModelCache(
|
||||
raw_data=[entry],
|
||||
folders=[],
|
||||
all_folders=[],
|
||||
name_display_mode="file_name",
|
||||
)
|
||||
scanner._persistent_cache = PersistentModelCache(
|
||||
library_name="test", db_path=str(tmp_path / "models.sqlite")
|
||||
)
|
||||
return scanner, model_file, entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_repairs_truncated_file_name(tmp_path, monkeypatch):
|
||||
scanner, model_file, entry = await _prepare_corrupted_library(tmp_path, monkeypatch)
|
||||
assert entry["file_name"] == TRUNCATED_STEM
|
||||
|
||||
await scanner._reconcile_cache()
|
||||
|
||||
# In-memory cache entry.
|
||||
assert entry["file_name"] == DOTTED_STEM
|
||||
|
||||
# On-disk sidecar (repaired by MetadataManager normalization).
|
||||
sidecar = model_file.parent / f"{DOTTED_STEM}.metadata.json"
|
||||
saved = json.loads(sidecar.read_text(encoding="utf-8"))
|
||||
assert saved["file_name"] == DOTTED_STEM
|
||||
|
||||
# Targeted SQL delta, so the fix survives a restart.
|
||||
persisted = scanner._persistent_cache.load_cache(scanner.model_type)
|
||||
assert persisted is not None
|
||||
rows = [i for i in persisted.raw_data if i["file_path"] == _normalize(model_file)]
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["file_name"] == DOTTED_STEM
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_repair_runs_once(tmp_path, monkeypatch):
|
||||
scanner, _model_file, _entry = await _prepare_corrupted_library(
|
||||
tmp_path, monkeypatch
|
||||
)
|
||||
|
||||
calls = {"sync": 0, "load": 0}
|
||||
original_sync = scanner._sync_cache_from_metadata_impl
|
||||
original_load = MetadataManager.load_metadata
|
||||
|
||||
async def counting_sync(file_path, metadata_dict):
|
||||
calls["sync"] += 1
|
||||
return await original_sync(file_path, metadata_dict)
|
||||
|
||||
async def counting_load(*args, **kwargs):
|
||||
calls["load"] += 1
|
||||
return await original_load(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(scanner, "_sync_cache_from_metadata_impl", counting_sync)
|
||||
monkeypatch.setattr(MetadataManager, "load_metadata", counting_load)
|
||||
|
||||
await scanner._reconcile_cache()
|
||||
assert calls == {"sync": 1, "load": 1}
|
||||
|
||||
# The mismatch is gone, so a second refresh must not touch metadata again.
|
||||
await scanner._reconcile_cache()
|
||||
assert calls == {"sync": 1, "load": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_clean_library_never_reads_metadata(tmp_path, monkeypatch):
|
||||
"""The repair probe must cost one string compare, not a metadata read."""
|
||||
scanner, _model_file, _entry = await _prepare_corrupted_library(
|
||||
tmp_path, monkeypatch
|
||||
)
|
||||
await scanner._reconcile_cache()
|
||||
|
||||
calls = {"load": 0}
|
||||
original_load = MetadataManager.load_metadata
|
||||
|
||||
async def counting_load(*args, **kwargs):
|
||||
calls["load"] += 1
|
||||
return await original_load(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(MetadataManager, "load_metadata", counting_load)
|
||||
|
||||
await scanner._reconcile_cache()
|
||||
assert calls["load"] == 0
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Regression tests for issue #1112.
|
||||
|
||||
Dotted model file names (``lora-sd1.5-backlight_slider_v10.safetensors``) must
|
||||
not be truncated when metadata is built from a CivitAI payload. The builder has
|
||||
to strip at most one *known* model extension: API filenames keep their
|
||||
extension, while migration paths (``.civitai.info``) pass the local stem.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from py.utils.models import (
|
||||
CheckpointMetadata,
|
||||
EmbeddingMetadata,
|
||||
LoraMetadata,
|
||||
OtherModelMetadata,
|
||||
strip_model_extension,
|
||||
)
|
||||
|
||||
DOTTED_STEM = "lora-sd1.5-backlight_slider_v10"
|
||||
|
||||
MODEL_CLASSES = [
|
||||
LoraMetadata,
|
||||
CheckpointMetadata,
|
||||
EmbeddingMetadata,
|
||||
OtherModelMetadata,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
# Dotted stems are not extensions.
|
||||
(DOTTED_STEM, DOTTED_STEM),
|
||||
(f"{DOTTED_STEM}.safetensors", DOTTED_STEM),
|
||||
("model.v1.5.safetensors", "model.v1.5"),
|
||||
("a.b.c", "a.b.c"),
|
||||
# Every scanner extension is recognized, case-insensitively.
|
||||
("weights.GGUF", "weights"),
|
||||
("weights.pt2", "weights"),
|
||||
("weights.ckpt", "weights"),
|
||||
# Extension-free plain names are unchanged.
|
||||
("plain_name", "plain_name"),
|
||||
("", ""),
|
||||
],
|
||||
)
|
||||
def test_strip_model_extension_strips_only_known_extensions(raw, expected):
|
||||
assert strip_model_extension(raw) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"file_name",
|
||||
[
|
||||
f"{DOTTED_STEM}.safetensors", # CivitAI API / download shape
|
||||
DOTTED_STEM, # .civitai.info migration shape (already extension-free)
|
||||
],
|
||||
)
|
||||
def test_from_civitai_info_keeps_dotted_stem(model_cls, file_name):
|
||||
version_info = {
|
||||
"baseModel": "SD 1.5",
|
||||
"name": "v1.0",
|
||||
"model": {"name": "Light Control", "description": "", "tags": []},
|
||||
}
|
||||
file_info = {"name": file_name, "sizeKB": 1024, "hashes": {"SHA256": "a" * 64}}
|
||||
|
||||
metadata = model_cls.from_civitai_info(
|
||||
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
|
||||
)
|
||||
|
||||
assert metadata.file_name == DOTTED_STEM
|
||||
assert metadata.model_name == "Light Control"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
|
||||
def test_from_civitai_info_model_name_fallback_uses_full_stem(model_cls):
|
||||
"""A sidecar without ``model.name`` must fall back to the full local stem."""
|
||||
version_info = {"baseModel": "SD 1.5", "model": {"description": "", "tags": []}}
|
||||
file_info = {"name": DOTTED_STEM, "sizeKB": 1024, "hashes": {}}
|
||||
|
||||
metadata = model_cls.from_civitai_info(
|
||||
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
|
||||
)
|
||||
|
||||
assert metadata.file_name == DOTTED_STEM
|
||||
assert metadata.model_name == DOTTED_STEM
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_cls", MODEL_CLASSES)
|
||||
def test_from_civitai_info_model_name_still_wins_over_stem(model_cls):
|
||||
"""The CivitAI model name stays authoritative when present."""
|
||||
version_info = {
|
||||
"baseModel": "SDXL",
|
||||
"model": {"name": "Chiaroscuro Light", "description": "", "tags": ["light"]},
|
||||
}
|
||||
file_info = {
|
||||
"name": f"{DOTTED_STEM}.safetensors",
|
||||
"sizeKB": 1024,
|
||||
"hashes": {},
|
||||
}
|
||||
|
||||
metadata = model_cls.from_civitai_info(
|
||||
version_info, file_info, f"/models/{DOTTED_STEM}.safetensors"
|
||||
)
|
||||
|
||||
assert metadata.file_name == DOTTED_STEM
|
||||
assert metadata.model_name == "Chiaroscuro Light"
|
||||
Reference in New Issue
Block a user