fix(scanner): compare model roots by identity, not by spelling (#1146)

ComfyUI hands its model folders over in the host's own spelling --
`utils/extra_config.py` runs every configured path through `os.path.normpath()`,
which is all backslashes on Windows, and `main.py` registers `<output>/loras`
unconditionally for its save nodes. `Config` kept that spelling verbatim while
`_prepare_*_paths()` normalized the live root lists to forward slashes, and the
scanner compared the two as raw strings. On Windows every configured root
therefore looked like a root that was not in the live list: the Refresh menu
listed each directory twice (the twin showing "0 models"), the completion toast
claimed reachable roots were "currently inaccessible" and kept their whole cache
count as `kept_unreachable`, and the settings sync persisted host-registered
directories that were never created.

The duplication was not only cosmetic. `_UnreachablePaths.add()` normalizes the
prefix it is given, so the configured twin of a live root registered exactly the
live root's prefix as unreadable: in the prune phase every cached-but-missing
entry under it matched and was "kept", so models deleted from disk were never
removed from the cache again.

* `py/utils/root_paths.py` holds the single `normalize_root_identity()`
  (separator style, redundant/trailing separators, and case where the filesystem
  folds it: whole paths on Windows, drive letters everywhere). `config.py` and
  `settings_manager.py` each carried a private copy and `model_scanner.py`
  compared raw strings.
* `ModelScanner` compares root identities in `get_configured_model_roots()`,
  `describe_model_roots()` (dedupe + `available`), the reconcile's
  `configured_only` list and the `ReconcileScope.roots` membership test, so one
  directory is one root whatever the host called it. `_normalized_root_prefix()`
  is built on the same identity, and an empty prefix no longer matches every
  path.
* A configured root that never made it into the live list is reported only while
  it still holds cached entries -- in the menu and in `skipped_roots` /
  `unavailable_paths` alike. A directory that neither exists nor was ever scanned
  has nothing to keep, so `<output>/loras` stops being an unavailable root.
* The startup sync drops saved roots that neither exist nor hold cached entries
  (`PersistentModelCache.roots_with_cached_entries()`, one scan per model type,
  separator-agnostic and wildcard-escaped). A root named by the library's
  `default_*_root` is kept even when empty: dropping it would let the next boot
  "repair" the default onto another root. When the cache cannot be consulted,
  every root is kept.
* `GET /scan?roots=` validates requested roots by identity too, so the browser
  extension or a script passing the spelling settings.json holds selects the root
  instead of answering 400 "Unknown model root(s)".

Verified: 8 of the new tests fail against the pre-fix source (checked with the
fix stashed); 3798 passed, 7 skipped, 11 deselected. The probe costs 71ms for a
100k-model library with seven missing roots (357ms when probing root by root).
This commit is contained in:
Will Miao
2026-10-12 08:43:06 +08:00
parent e056142669
commit 6d0d7cb15b
11 changed files with 822 additions and 57 deletions
+104 -15
View File
@@ -4,7 +4,6 @@
# import cycles. Breaking them would require an architectural refactor.
import os
import platform
import posixpath
import threading
from pathlib import Path
import folder_paths # pyright: ignore[reportMissingImports]
@@ -20,6 +19,7 @@ from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_
from .utils.constants import (
OTHER_MODEL_FOLDER_SUBTYPES,
)
from .utils.root_paths import normalize_root_identity
from .utils.settings_paths import (
ensure_settings_file,
get_settings_dir,
@@ -34,14 +34,15 @@ standalone_mode = (
logger = logging.getLogger(__name__)
def _normalize_root_identity(path: str) -> str:
"""Normalize a root path for comparisons across slash styles."""
normalized = posixpath.normpath(path.strip().replace("\\", "/"))
if len(normalized) >= 2 and normalized[1] == ":":
return normalized.lower()
return normalized
# Saved folder-path keys mapped to the model type whose cache can tell whether a
# root that is missing right now is still worth keeping (issue #1146). The
# checkpoint page holds both checkpoints and unet roots.
_CACHE_MODEL_TYPE_BY_FOLDER_KEY = {
"loras": "lora",
"checkpoints": "checkpoint",
"unet": "checkpoint",
"embeddings": "embedding",
}
def _append_new_paths(current: List[str], candidates: Iterable[str]) -> List[str]:
@@ -52,11 +53,11 @@ def _append_new_paths(current: List[str], candidates: Iterable[str]) -> List[str
to the root *order* would move user data.
"""
merged = list(current)
seen = {_normalize_root_identity(path) for path in merged}
seen = {normalize_root_identity(path) for path in merged}
for path in candidates:
if not isinstance(path, str) or not path.strip():
continue
identity = _normalize_root_identity(path)
identity = normalize_root_identity(path)
if identity in seen:
continue
seen.add(identity)
@@ -78,15 +79,15 @@ def _resolve_valid_default_root(
stripped = path.strip()
if not stripped:
continue
identity = _normalize_root_identity(stripped)
identity = normalize_root_identity(stripped)
if identity in seen:
continue
seen.add(identity)
fallback_paths.append(stripped)
allowed = {_normalize_root_identity(path) for path in fallback_paths}
allowed = {normalize_root_identity(path) for path in fallback_paths}
if current and _normalize_root_identity(current) in allowed:
if current and normalize_root_identity(current) in allowed:
return current
if not valid_paths:
@@ -320,6 +321,77 @@ class Config:
"Could not load extra paths from library settings: %s", exc
)
def _drop_missing_roots_without_cache(
self,
folder_paths_map: Dict[str, List[str]],
default_roots: Mapping[str, str],
) -> Dict[str, List[str]]:
"""Drop saved roots that neither exist nor hold cached entries (issue #1146).
The startup sync persists what is *configured* rather than what is readable,
so a switched-off drive keeps its path (see ``save_folder_paths_to_settings``).
A directory the host registered without ever creating it -- ComfyUI adds
``<output>/loras`` and friends for its save nodes whether or not they exist --
came along for the ride and stayed there as a permanently "unavailable" root.
A root that still holds cached entries is exactly the switched-off drive the
persistence exists for, so it is kept. A root named by the library's
``default_*_root`` is kept even when it is empty: dropping it would let the
next boot repair the default onto another root.
Best effort: when the cache cannot be consulted, every root is kept.
"""
try:
from .services.persistent_model_cache import get_persistent_cache
cache = get_persistent_cache()
except Exception as exc: # pragma: no cover - defensive logging
logger.debug("Keeping every configured model root: %s", exc)
return folder_paths_map
protected = {
identity
for identity in (
normalize_root_identity(path) for path in default_roots.values()
)
if identity
}
filtered: Dict[str, List[str]] = {}
for key, paths in folder_paths_map.items():
missing = [path for path in paths if not os.path.exists(path)]
if not missing:
filtered[key] = list(paths)
continue
probe = [
path
for path in missing
if normalize_root_identity(path) not in protected
]
try:
cached = cache.roots_with_cached_entries(
_CACHE_MODEL_TYPE_BY_FOLDER_KEY.get(key, "other"), probe
)
except Exception as exc:
logger.debug(
"Keeping every configured %s root (%s): %s", key, missing, exc
)
filtered[key] = list(paths)
continue
cached_ids = {normalize_root_identity(path) for path in cached}
filtered[key] = [
path
for path in paths
if os.path.exists(path)
or normalize_root_identity(path) in protected
or normalize_root_identity(path) in cached_ids
]
return filtered
def save_folder_paths_to_settings(self):
"""Persist ComfyUI-derived folder paths to the multi-library settings."""
try:
@@ -533,9 +605,26 @@ class Config:
or self._should_activate_comfy_library(libraries, libraries_changed)
)
# A root that the host registered but never created (and that holds no
# cached models) is not a library path: persisting it would bring it
# back as an unavailable root on every boot, and in standalone mode the
# saved paths are the only source of truth (issue #1146).
default_roots = {
"loras": default_lora_root
or comfy_library.get("default_lora_root", ""),
"checkpoints": default_checkpoint_root
or comfy_library.get("default_checkpoint_root", ""),
"unet": comfy_library.get("default_unet_root", ""),
"embeddings": default_embedding_root
or comfy_library.get("default_embedding_root", ""),
}
persisted_folder_paths = self._drop_missing_roots_without_cache(
target_folder_paths, default_roots
)
settings_service.upsert_library(
"comfyui",
folder_paths=target_folder_paths,
folder_paths=persisted_folder_paths,
extra_folder_paths=extra_folder_paths,
default_lora_root=default_lora_root,
default_checkpoint_root=default_checkpoint_root,
+13 -1
View File
@@ -55,6 +55,7 @@ from ...utils.civitai_utils import resolve_license_payload
from ...utils.file_utils import calculate_sha256
from ...utils.metadata_manager import MetadataManager
from ...utils.paid_access import is_early_access_deadline_active
from ...utils.root_paths import normalize_root_identity
from ...utils.sidecar_paths import get_metadata_path
from ...utils.url_utils import relative_root_prefix
@@ -1170,7 +1171,18 @@ class ModelQueryHandler:
if requested_roots:
configured = self._service.get_model_roots()
unknown = [root for root in requested_roots if root not in configured]
# Compare by identity, not by string: callers may pass the host's
# own spelling of a root (backslashes or a differently cased drive
# letter on Windows, as settings.json and ComfyUI show it), which
# names the same directory the live list holds forward-slashed.
known_roots = {
normalize_root_identity(root) for root in configured
}
unknown = [
root
for root in requested_roots
if normalize_root_identity(root) not in known_roots
]
if unknown:
return web.json_response(
{"error": "Unknown model root(s)", "roots": unknown}, status=400
+74 -25
View File
@@ -25,6 +25,7 @@ from ..utils.recipes_paths import (
get_effective_recipes_dir,
normalized_recipes_dir_key,
)
from ..utils.root_paths import normalize_root_identity
from .model_cache import ModelCache
from .model_hash_index import ModelHashIndex
from .model_lifecycle_service import delete_model_artifacts, _require_path_in_library_roots
@@ -203,16 +204,33 @@ def _root_display_labels(roots: Sequence[str]) -> Dict[str, str]:
return {root: _truncate_label(label) for root, label in labels.items()}
def _root_identity(root: str) -> str:
"""Comparable identity of a model root, folding the host's spelling (issue #1146).
The live root list is forward-slashed while ``Config`` keeps the configured
paths exactly as the host handed them over -- ComfyUI's ``os.path.normpath()``
output is all backslashes on Windows. Comparing the two as raw strings turns
one directory into two roots: a duplicated menu row, a bogus "unavailable"
report, and -- worse -- an unreachable prefix registered over a root the walk
just read, which stops deleted models from ever being pruned.
"""
return normalize_root_identity(root)
def _normalized_root_prefix(root: str) -> str:
"""Forward-slash root prefix used to attribute cached paths to a root."""
prefix = os.path.normpath(root).replace(os.sep, "/")
if not prefix.endswith("/"):
prefix = _root_identity(root)
if prefix and not prefix.endswith("/"):
prefix += "/"
return prefix.lower() if _CASE_INSENSITIVE_PATHS else prefix
return prefix
def _path_matches_prefix(path: str, prefix: str) -> bool:
"""Match a cached business path against a normalized scope prefix."""
if not prefix:
# An empty prefix would match every path; callers filter empty roots, so
# keep it inert instead of turning it into a catch-all.
return False
candidate = path.lower() if _CASE_INSENSITIVE_PATHS else path
return candidate.startswith(prefix)
@@ -332,7 +350,8 @@ def _count_cached_entries_for_prefixes(
for path in cached_paths:
candidate = path.lower() if case_insensitive else path
for key, prefix in ordered:
if candidate.startswith(prefix):
# An empty prefix would claim every path; see _path_matches_prefix.
if prefix and candidate.startswith(prefix):
counts[key] += 1
break
return counts
@@ -1621,8 +1640,9 @@ class ModelScanner:
seen: Set[str] = set()
for key in keys:
for root in config.configured_roots_for(key):
if root and root not in seen:
seen.add(root)
identity = _root_identity(root)
if identity and identity not in seen:
seen.add(identity)
roots.append(root)
return roots
@@ -1651,32 +1671,44 @@ class ModelScanner:
roots: List[str] = []
seen: Set[str] = set()
for root in self.get_model_roots():
if not root or root in seen:
identity = _root_identity(root)
if not identity or identity in seen:
continue
seen.add(root)
seen.add(identity)
roots.append(root)
# Configured roots that are missing from the live list: their directory
# did not exist when it was built (a drive switched off at startup).
# They are reported so the menu offers a row in both states instead of
# hiding the root until the process is restarted.
unavailable = [
root for root in self.get_configured_model_roots() if root not in seen
]
# Compared by identity, never by string -- the live list is forward-slashed
# while Config keeps the host's own spelling, which is all backslashes on
# Windows (issue #1146). Only roots that still hold cached entries are
# reported: a path that neither exists nor was ever scanned has nothing to
# keep, and ComfyUI registers `<output>/loras` for its save nodes whether or
# not that directory is there.
candidates: List[str] = []
reported: Set[str] = set()
for root in self.get_configured_model_roots():
identity = _root_identity(root)
if not identity or identity in seen or identity in reported:
continue
reported.add(identity)
candidates.append(root)
labels = _root_display_labels(roots + unavailable)
cached_paths = (
{item.get('file_path', '') for item in self._cache.raw_data}
if self._cache is not None
else set()
)
counts = _count_cached_entries_per_root(cached_paths, roots + unavailable)
counts = _count_cached_entries_per_root(cached_paths, roots + candidates)
unavailable = [root for root in candidates if counts.get(root, 0)]
labels = _root_display_labels(roots + unavailable)
return [
{
'path': root.replace(os.sep, '/'),
'label': labels[root],
'reachable': os.path.exists(root),
'available': root in seen,
'available': _root_identity(root) in seen,
'models': counts.get(root, 0),
}
for root in roots + unavailable
@@ -1801,25 +1833,37 @@ class ModelScanner:
configured_roots: List[str] = []
seen_roots: Set[str] = set()
for root_path in self.get_model_roots():
if not root_path or root_path in seen_roots:
identity = _root_identity(root_path)
if not identity or identity in seen_roots:
continue
seen_roots.add(root_path)
seen_roots.add(identity)
configured_roots.append(root_path)
# Configured roots that never made it into the live list (their
# directory was missing when it was built). They are outside the scan
# scope entirely — which is exactly why their cached entries survive
# — so they only need reporting, and their labels must not collide
# with the live ones.
configured_only = [
root for root in self.get_configured_model_roots()
if root and root not in seen_roots
]
# with the live ones. Identity, not string: the live list is
# forward-slashed while Config keeps the host's own spelling -- all
# backslashes on Windows -- and a raw compare would report every root
# twice and shadow it as unreachable (issue #1146).
configured_only: List[str] = []
for root in self.get_configured_model_roots():
identity = _root_identity(root)
if not identity or identity in seen_roots:
continue
seen_roots.add(identity)
configured_only.append(root)
all_labels = _root_display_labels(configured_roots + configured_only)
scope_root_ids = (
{_root_identity(root) for root in scope.roots}
if scope is not None and scope.roots is not None
else None
)
scope_roots = [
root for root in configured_roots
if scope is None or scope.roots is None or root in scope.roots
if scope_root_ids is None or _root_identity(root) in scope_root_ids
]
scope_folder = scope.folder if scope is not None else None
@@ -1853,7 +1897,12 @@ class ModelScanner:
)
for root_path, _prefix in folder_candidates:
kept = folder_counts.get(root_path, 0)
if scope_folder is not None and not kept:
# Only a root that still holds cached entries is reported --
# and protected from pruning. A configured path that neither
# exists nor was ever scanned has nothing to keep, and ComfyUI
# registers `<output>/loras` for its save nodes even when that
# directory is absent (issue #1146).
if not kept:
continue
unreachable.add(root_path, 'root_unavailable', kept=kept)
skipped_roots.append({
+73 -1
View File
@@ -4,16 +4,33 @@ import os
import sqlite3
import threading
from dataclasses import dataclass, field
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple
from ..utils.cache_db import connect_cache_db
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
from ..utils.file_lock import exclusive_lock
from ..utils.root_paths import normalize_root_identity
from .model_sources import normalize_metadata_source
logger = logging.getLogger(__name__)
def _like_prefix_pattern(identity: str) -> str:
"""SQLite LIKE pattern matching every path inside *identity*.
``%``/``_`` are legal characters in a path and ``\\`` is the escape character,
so all three are escaped. The separator is added only when the identity does not
already end in one, so a filesystem root (``/``) still matches everything below
it instead of only itself.
"""
escaped = (
identity.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
)
prefix = escaped if escaped.endswith("/") else f"{escaped}/"
return f"{prefix}%"
@dataclass
class PersistedCacheData:
"""Lightweight structure returned by the persistent cache."""
@@ -254,6 +271,61 @@ class PersistentModelCache:
all_folders=all_folders,
)
def roots_with_cached_entries(
self, model_type: str, roots: Sequence[str]
) -> Set[str]:
"""Return the subset of *roots* that hold at least one cached entry.
Used by the settings sync to tell a root that was registered but never
scanned -- ComfyUI registers ``<output>/loras`` for its save nodes whether
or not that directory exists -- from a real root whose drive happens to be
switched off: only the latter must keep its place in the saved library
paths (issue #1146). Paths are matched on their forward-slash identity, so
the host's own slash style does not matter.
Raises when the answer is unknown (a disabled cache, a missing database, an
unreadable file): the caller keeps every root in that case.
"""
if not self.is_enabled():
raise RuntimeError("persistent model cache is disabled")
candidates: Dict[str, str] = {}
for root in roots:
identity = normalize_root_identity(root)
if identity:
candidates.setdefault(identity, root)
if not candidates:
return set()
matched: Set[str] = set()
# One scan per model type rather than one per root: the probe runs at
# startup for every configured root that is missing right now, which on
# Windows includes all of ComfyUI's `<output>/<type>` folders.
clause = "replace(file_path, '\\', '/') LIKE ? ESCAPE '\\'"
sql = (
"SELECT DISTINCT replace(file_path, '\\', '/') FROM models "
f"WHERE model_type = ? AND ({' OR '.join([clause] * len(candidates))})"
)
params: List[Any] = [model_type]
params.extend(_like_prefix_pattern(identity) for identity in candidates)
with self._db_lock:
conn = self._connect(readonly=True)
try:
rows = conn.execute(sql, params).fetchall()
finally:
conn.close()
for (cached_path,) in rows:
folded = normalize_root_identity(cached_path)
for identity, root in candidates.items():
prefix = identity if identity.endswith("/") else f"{identity}/"
if folded.startswith(prefix):
matched.add(root)
break
return matched
def save_cache(self, model_type: str, raw_data: Sequence[Dict[str, Any]], hash_index: Dict[str, List[str]], excluded_models: Sequence[str], autov3_hash_index: Optional[Dict[str, List[str]]] = None, all_folders: Optional[Sequence[str]] = None) -> None:
if not self.is_enabled():
return
+6 -15
View File
@@ -2,7 +2,6 @@ import asyncio
import copy
import json
import os
import posixpath
import shutil
import tempfile
import logging
@@ -36,6 +35,7 @@ from ..utils.constants import (
normalize_other_sub_types,
)
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
from ..utils.root_paths import normalize_root_identity
from ..utils.settings_paths import (
APP_NAME,
_portable_env_override,
@@ -156,15 +156,6 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
}
def _normalize_root_identity(path: str) -> str:
"""Normalize a root path for equality checks across slash styles."""
normalized = posixpath.normpath(path.strip().replace("\\", "/"))
if len(normalized) >= 2 and normalized[1] == ":":
return normalized.lower()
return normalized
class SettingsManager:
def __init__(self):
self.settings_file = ensure_settings_file(logger)
@@ -1058,7 +1049,7 @@ class SettingsManager:
return False
allowed_roots = self._get_allowed_roots(key)
if current and _normalize_root_identity(current) in allowed_roots:
if current and normalize_root_identity(current) in allowed_roots:
return False
self.settings[setting_key] = primary_candidates[0]
@@ -1094,7 +1085,7 @@ class SettingsManager:
candidate_identities: set[str] = set()
for folder_key in OTHER_SUB_TYPE_FOLDER_KEYS.get(sub_type, []):
for candidate in self._get_valid_root_candidates(folder_key):
identity = _normalize_root_identity(candidate)
identity = normalize_root_identity(candidate)
if identity in candidate_identities:
continue
candidate_identities.add(identity)
@@ -1102,7 +1093,7 @@ class SettingsManager:
if not candidates:
continue
current = other_roots.get(sub_type, "")
if current and _normalize_root_identity(current) in candidate_identities:
if current and normalize_root_identity(current) in candidate_identities:
continue
other_roots[sub_type] = candidates[0]
if current:
@@ -1152,7 +1143,7 @@ class SettingsManager:
normalized = value.strip()
if not normalized:
continue
identity = _normalize_root_identity(normalized)
identity = normalize_root_identity(normalized)
if identity in seen:
continue
seen.add(identity)
@@ -1162,7 +1153,7 @@ class SettingsManager:
def _get_allowed_roots(self, key: str) -> set[str]:
"""Return all valid roots for a model type, including extra roots."""
return {_normalize_root_identity(path) for path in self._get_valid_root_candidates(key)}
return {normalize_root_identity(path) for path in self._get_valid_root_candidates(key)}
def _check_environment_variables(self) -> None:
"""Check for environment variables and update settings if needed"""
+36
View File
@@ -0,0 +1,36 @@
"""Path identity helpers for model roots.
ComfyUI hands its model folders over in whatever spelling the host uses --
``os.path.normpath()`` output, which is all backslashes on Windows -- while LoRA
Manager's own root lists and cached model paths are forward-slashed. Comparing
the two spellings as raw strings makes one directory look like two roots, so every
root identity check goes through :func:`normalize_root_identity` (issue #1146).
"""
from __future__ import annotations
import os
import posixpath
def normalize_root_identity(path: str) -> str:
"""Return the comparable identity of a model root path.
Folds the separator style (``\\`` and ``/`` are both accepted as separators,
because the host may spell a Windows path either way), redundant and trailing
separators, and the case where the filesystem does not distinguish it: Windows
compares whole paths case-insensitively, and every platform folds a drive
letter (``E:`` == ``e:``). Symlinks are never resolved -- business paths stay
business paths, matching the rest of the codebase.
"""
if not isinstance(path, str):
return ""
candidate = path.strip()
if not candidate:
return ""
normalized = posixpath.normpath(candidate.replace("\\", "/"))
if os.name == "nt" or (len(normalized) >= 2 and normalized[1] == ":"):
return normalized.lower()
return normalized