mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-10-12 20:32:12 -03:00
fix(scanner): compare model roots by identity, not by spelling (#1146)
ComfyUI hands its model folders over in the host's own spelling -- `utils/extra_config.py` runs every configured path through `os.path.normpath()`, which is all backslashes on Windows, and `main.py` registers `<output>/loras` unconditionally for its save nodes. `Config` kept that spelling verbatim while `_prepare_*_paths()` normalized the live root lists to forward slashes, and the scanner compared the two as raw strings. On Windows every configured root therefore looked like a root that was not in the live list: the Refresh menu listed each directory twice (the twin showing "0 models"), the completion toast claimed reachable roots were "currently inaccessible" and kept their whole cache count as `kept_unreachable`, and the settings sync persisted host-registered directories that were never created. The duplication was not only cosmetic. `_UnreachablePaths.add()` normalizes the prefix it is given, so the configured twin of a live root registered exactly the live root's prefix as unreadable: in the prune phase every cached-but-missing entry under it matched and was "kept", so models deleted from disk were never removed from the cache again. * `py/utils/root_paths.py` holds the single `normalize_root_identity()` (separator style, redundant/trailing separators, and case where the filesystem folds it: whole paths on Windows, drive letters everywhere). `config.py` and `settings_manager.py` each carried a private copy and `model_scanner.py` compared raw strings. * `ModelScanner` compares root identities in `get_configured_model_roots()`, `describe_model_roots()` (dedupe + `available`), the reconcile's `configured_only` list and the `ReconcileScope.roots` membership test, so one directory is one root whatever the host called it. `_normalized_root_prefix()` is built on the same identity, and an empty prefix no longer matches every path. * A configured root that never made it into the live list is reported only while it still holds cached entries -- in the menu and in `skipped_roots` / `unavailable_paths` alike. A directory that neither exists nor was ever scanned has nothing to keep, so `<output>/loras` stops being an unavailable root. * The startup sync drops saved roots that neither exist nor hold cached entries (`PersistentModelCache.roots_with_cached_entries()`, one scan per model type, separator-agnostic and wildcard-escaped). A root named by the library's `default_*_root` is kept even when empty: dropping it would let the next boot "repair" the default onto another root. When the cache cannot be consulted, every root is kept. * `GET /scan?roots=` validates requested roots by identity too, so the browser extension or a script passing the spelling settings.json holds selects the root instead of answering 400 "Unknown model root(s)". Verified: 8 of the new tests fail against the pre-fix source (checked with the fix stashed); 3798 passed, 7 skipped, 11 deselected. The probe costs 71ms for a 100k-model library with seven missing roots (357ms when probing root by root).
This commit is contained in:
Will Miao
@@ -2949,3 +2949,133 @@ def test_reconcile_walk_skips_the_recipes_dir(tmp_path: Path, monkeypatch):
|
||||
assert "keep" in result.discovered_folders
|
||||
assert "recipes" not in result.discovered_folders
|
||||
assert "recipes/sub" not in result.discovered_folders
|
||||
|
||||
|
||||
# --- the host's path spelling (issue #1146) ---------------------------------
|
||||
|
||||
|
||||
def _windows_spelling(path: Path) -> str:
|
||||
"""Return a path the way ComfyUI hands it over on Windows.
|
||||
|
||||
``utils/extra_config.py`` normalizes every configured model folder with
|
||||
``os.path.normpath``, so on Windows the configured list is all backslashes
|
||||
while the live root list is forward-slashed.
|
||||
"""
|
||||
return str(path).replace("/", "\\")
|
||||
|
||||
|
||||
async def _windows_spelling_scanner(tmp_path: Path, monkeypatch):
|
||||
"""Scanner configured with backslash roots plus a never-created output dir.
|
||||
|
||||
``<output>/loras`` is what ComfyUI registers for its save nodes whether or not
|
||||
the directory exists (``main.py`` calls ``add_model_folder_path`` with it
|
||||
unconditionally).
|
||||
"""
|
||||
primary = tmp_path / "primary"
|
||||
(primary / "pack").mkdir(parents=True)
|
||||
(primary / "pack" / "a.txt").write_text("a", encoding="utf-8")
|
||||
never_created = tmp_path / "output" / "loras"
|
||||
|
||||
scanner = MultiRootDummyScanner([primary])
|
||||
await scanner._initialize_cache()
|
||||
monkeypatch.setattr(
|
||||
scanner,
|
||||
"get_configured_model_roots",
|
||||
lambda: [_windows_spelling(primary), _windows_spelling(never_created)],
|
||||
)
|
||||
return scanner, primary, never_created
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_describe_roots_folds_the_host_path_spelling(tmp_path: Path, monkeypatch):
|
||||
"""One directory is one root, whatever separator the host used."""
|
||||
scanner, primary, _never_created = await _windows_spelling_scanner(
|
||||
tmp_path, monkeypatch
|
||||
)
|
||||
|
||||
described = scanner.describe_model_roots()
|
||||
|
||||
assert [entry["path"] for entry in described] == [_normalize_path(primary)]
|
||||
assert described[0]["available"] is True
|
||||
assert described[0]["reachable"] is True
|
||||
assert described[0]["models"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_does_not_shadow_a_backslash_configured_root(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
"""A configured twin must not register its live root as unreachable.
|
||||
|
||||
``_UnreachablePaths`` normalizes the prefixes it is given, so the backslash
|
||||
spelling of a root the walk just read normalized to the same prefix: every
|
||||
cached entry under it counted as "kept", which is what made the refresh toast
|
||||
claim thousands of reachable models were inaccessible -- and what stopped
|
||||
deleted models from ever being pruned.
|
||||
"""
|
||||
scanner, primary, _never_created = await _windows_spelling_scanner(
|
||||
tmp_path, monkeypatch
|
||||
)
|
||||
(primary / "pack" / "a.txt").unlink()
|
||||
|
||||
summary = await scanner._reconcile_cache()
|
||||
|
||||
assert summary is not None
|
||||
assert summary["skipped_roots"] == []
|
||||
assert summary["unavailable_paths"] == []
|
||||
assert summary["kept_unreachable"] == 0
|
||||
assert summary["removed"] == 1
|
||||
assert scanner._cache.raw_data == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_root_that_never_existed_is_not_reported(tmp_path: Path, monkeypatch):
|
||||
"""A registered directory that neither exists nor holds cache is not a root.
|
||||
|
||||
Only roots the user really configured and whose models are cached are worth an
|
||||
offline row or a "models kept" notice.
|
||||
"""
|
||||
primary = tmp_path / "primary"
|
||||
(primary / "pack").mkdir(parents=True)
|
||||
(primary / "pack" / "a.txt").write_text("a", encoding="utf-8")
|
||||
never_created = tmp_path / "output" / "loras"
|
||||
|
||||
scanner = MultiRootDummyScanner([primary])
|
||||
await scanner._initialize_cache()
|
||||
monkeypatch.setattr(
|
||||
scanner,
|
||||
"get_configured_model_roots",
|
||||
lambda: [str(primary), str(never_created)],
|
||||
)
|
||||
|
||||
assert [entry["path"] for entry in scanner.describe_model_roots()] == [
|
||||
_normalize_path(primary)
|
||||
]
|
||||
|
||||
summary = await scanner._reconcile_cache()
|
||||
|
||||
assert summary is not None
|
||||
assert summary["skipped_roots"] == []
|
||||
assert summary["kept_unreachable"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scoped_scan_accepts_the_host_path_spelling(tmp_path: Path, monkeypatch):
|
||||
"""A caller may name a root the way the host spells it (issue #1146).
|
||||
|
||||
The Refresh menu sends back the path `/roots` rendered, but the browser
|
||||
extension and scripts can pass what settings.json holds -- backslashes on
|
||||
Windows -- and that must select the root rather than scan nothing.
|
||||
"""
|
||||
primary = tmp_path / "primary"
|
||||
(primary / "pack").mkdir(parents=True)
|
||||
(primary / "pack" / "a.txt").write_text("a", encoding="utf-8")
|
||||
|
||||
scanner = MultiRootDummyScanner([primary])
|
||||
await scanner._initialize_cache()
|
||||
|
||||
scope = ReconcileScope(roots=(_windows_spelling(primary),))
|
||||
summary = await scanner._reconcile_cache(scope=scope)
|
||||
|
||||
assert summary is not None
|
||||
assert summary["scanned_roots"] == [primary.name]
|
||||
|
||||
@@ -451,3 +451,100 @@ def test_get_models_missing_autov3_self_terminates_after_marking(tmp_path: Path,
|
||||
store.update_single_model('dummy', new_item, old_item=old_item)
|
||||
|
||||
assert store.get_models_missing_autov3('dummy') == []
|
||||
|
||||
|
||||
# --- which roots still hold cached entries (issue #1146) --------------------
|
||||
|
||||
|
||||
def _cache_with_paths(tmp_path: Path, monkeypatch, entries) -> PersistentModelCache:
|
||||
"""Build a cache holding ``entries`` (``(model_type, file_path)`` pairs)."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
db_path = tmp_path / 'roots.sqlite'
|
||||
store = PersistentModelCache(db_path=str(db_path))
|
||||
with sqlite3.connect(str(db_path)) as conn:
|
||||
conn.executemany(
|
||||
"INSERT INTO models (model_type, file_path) VALUES (?, ?)", entries
|
||||
)
|
||||
conn.commit()
|
||||
return store
|
||||
|
||||
|
||||
def test_roots_with_cached_entries_matches_the_hosts_slash_style(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""The configured root may be spelled with backslashes (Windows)."""
|
||||
|
||||
store = _cache_with_paths(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
[("lora", "E:/StableDiffusion/cc_models/loras/pack/a.safetensors")],
|
||||
)
|
||||
|
||||
matched = store.roots_with_cached_entries(
|
||||
"lora", ["E:\\StableDiffusion\\cc_models\\loras", "E:\\empty\\loras"]
|
||||
)
|
||||
|
||||
assert matched == {"E:\\StableDiffusion\\cc_models\\loras"}
|
||||
|
||||
|
||||
def test_roots_with_cached_entries_compares_whole_path_segments(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""``loras-extra`` and ``loras2`` are not inside ``loras``."""
|
||||
|
||||
store = _cache_with_paths(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
[
|
||||
("lora", "E:/loras-extra/a.safetensors"),
|
||||
("lora", "E:/loras2/b.safetensors"),
|
||||
("checkpoint", "E:/loras/c.safetensors"),
|
||||
],
|
||||
)
|
||||
|
||||
assert store.roots_with_cached_entries("lora", ["E:/loras"]) == set()
|
||||
assert store.roots_with_cached_entries("checkpoint", ["E:/loras"]) == {"E:/loras"}
|
||||
|
||||
|
||||
def test_roots_with_cached_entries_escapes_like_wildcards(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""A directory name may contain ``_``/``%`` without becoming a wildcard."""
|
||||
|
||||
store = _cache_with_paths(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
[("lora", "E:/myXloras/a.safetensors"), ("lora", "E:/100%/a.safetensors")],
|
||||
)
|
||||
|
||||
assert store.roots_with_cached_entries("lora", ["E:/my_loras"]) == set()
|
||||
assert store.roots_with_cached_entries("lora", ["E:/100x"]) == set()
|
||||
assert store.roots_with_cached_entries("lora", ["E:/100%"]) == {"E:/100%"}
|
||||
|
||||
|
||||
def test_roots_with_cached_entries_raises_when_unknown(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
"""No cache, no answer: the caller must not drop roots on a guess."""
|
||||
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '1')
|
||||
disabled = PersistentModelCache(db_path=str(tmp_path / 'disabled.sqlite'))
|
||||
with pytest.raises(RuntimeError):
|
||||
disabled.roots_with_cached_entries("lora", ["E:/loras"])
|
||||
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
missing = PersistentModelCache(db_path=str(tmp_path / 'missing.sqlite'))
|
||||
(tmp_path / 'missing.sqlite').unlink()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
missing.roots_with_cached_entries("lora", ["E:/loras"])
|
||||
|
||||
|
||||
def test_roots_with_cached_entries_without_candidates(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
store = PersistentModelCache(db_path=str(tmp_path / 'empty.sqlite'))
|
||||
|
||||
assert store.roots_with_cached_entries("lora", []) == set()
|
||||
assert store.roots_with_cached_entries("lora", ["", " "]) == set()
|
||||
|
||||
Reference in New Issue
Block a user