alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-09-21 03:01:27 -03:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): serve folder tree from scan-recorded, persisted directory list (#1110)

Browse Source 
The include_empty folder tree (download/move modals) walked every model
root synchronously on the event loop via get_all_folders(). On network
(NAS) roots this froze the whole server for the duration of the walk —
blocking WebSocket progress, aria2 RPC and the download queue — and the
5s TTL re-triggered the walk on nearly every modal interaction.

The scanners already visit every directory during cache scans, so record
the full directory list (including empty folders) there instead:

- _gather_model_data/_reconcile_cache collect directories during the
  existing walks; reconcile refreshes and persists the list even when no
  model files changed.
- ModelCache gains an all_folders field (None = never recorded).
- PersistentModelCache stores the list in a new folders table, with a
  cache_meta flag distinguishing 'recorded empty' from legacy snapshots.
- get_all_folders() is now a pure in-memory read. A legacy snapshot
  triggers a one-shot backfill walk in a worker thread (never on the
  event loop) that records and persists the list.
- Moves add the destination folder (and parents) incrementally instead
  of invalidating a TTL cache.
This commit is contained in:
Will Miao
2026-09-11 23:03:24 +08:00
parent 91b2735dad
commit 6d3f82976f
5 changed files with 278 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/services/model_scanner.py
+123 -38
View File
@@ -62,10 +62,6 @@ def _is_hidden_relative_path(rel_path: str) -> bool:
return any(part.startswith(".") for part in rel_path.replace(os.sep, "/").split("/"))
# TTL (seconds) for the get_all_folders() live-walk cache, so rapid repeated
# requests (modal open + autocomplete) do not re-walk the model roots.
ALL_FOLDERS_CACHE_TTL_SECONDS = 5.0
# Maps a scanner model type to the manager page type used in progress
# broadcasts (e.g. 'lora' -> 'loras').
PAGE_TYPE_MAP = {
@@ -89,6 +85,10 @@ class CacheBuildResult:
hash_index: ModelHashIndex
tags_count: Dict[str, int]
excluded_models: List[str]
# Every directory under the model roots (including empty ones) discovered
# during the scan, or None when the source has no folder information
# (e.g. a persisted snapshot predating folder recording).
all_folders: Optional[List[str]] = None
class ModelScanner:
"""Base service for scanning and managing model files"""
@@ -144,8 +144,9 @@ class ModelScanner:
self._name_display_mode = self._resolve_name_display_mode()
self._cancel_requested = False # Flag for cancellation
self._autov3_backfill_scheduled = False # One-time AutoV3 backfill trigger per process
# Short-lived cache for get_all_folders(): (timestamp, folders) or None
self._all_folders_ttl_cache: Optional[Tuple[float, List[str]]] = None
# Guard against concurrent all-folders backfill walks (cold fallback
# for persisted snapshots that predate folder recording).
self._all_folders_backfill_running = False
try:
loop = asyncio.get_running_loop()
except RuntimeError:
@@ -217,7 +218,6 @@ class ModelScanner:
self._excluded_models = []
self._is_initializing = False
self._name_display_mode = self._resolve_name_display_mode()
self.invalidate_all_folders_cache()
self.bump_cache_version()
try:
@@ -702,7 +702,8 @@ class ModelScanner:
raw_data=valid_entries,
hash_index=hash_index,
tags_count=tags_count,
excluded_models=list(persisted.excluded_models)
excluded_models=list(persisted.excluded_models),
all_folders=list(persisted.all_folders) if persisted.all_folders is not None else None,
)
return scan_result, invalid_entries
@@ -737,6 +738,7 @@ class ModelScanner:
hash_snapshot,
list(scan_result.excluded_models),
autov3_snapshot,
scan_result.all_folders,
)
except Exception as exc:
logger.warning("%s Scanner: Failed to persist cache: %s", self.model_type.capitalize(), exc)
@@ -784,7 +786,12 @@ class ModelScanner:
raw_data=list(self._cache.raw_data),
hash_index=self._hash_index,
tags_count=dict(self._tags_count),
excluded_models=list(self._excluded_models)
excluded_models=list(self._excluded_models),
all_folders=(
list(self._cache.all_folders)
if self._cache.all_folders is not None
else None
),
)
await self._save_persistent_cache(snapshot)
await self._sync_download_history(snapshot.raw_data, source='scan')
@@ -1034,6 +1041,7 @@ class ModelScanner:
new_files = []
visited_real_paths = set()
discovered_real_files = set()
discovered_folders: Set[str] = set()
# Scan all model roots
for root_path in self.get_model_roots():
@@ -1048,6 +1056,14 @@ class ModelScanner:
continue
visited_real_paths.add(real_root)
# Record every visited directory (including empty ones) so
# the folder tree stays accurate without a live walk.
rel_dir = os.path.relpath(
os.path.abspath(root), os.path.abspath(root_path)
).replace(os.path.sep, "/")
if rel_dir != "." and not _is_hidden_relative_path(rel_dir):
discovered_folders.add(rel_dir)
for file in files:
ext = os.path.splitext(file)[1].lower()
if ext in self.file_extensions:
@@ -1247,6 +1263,14 @@ class ModelScanner:
self._cache.raw_data = list(reversed(deduped))
total_removed += dedup_removed
# The walk above visited every directory, so refresh the recorded
# folder list (including empty folders) even when no model files
# changed — e.g. an empty folder was created or removed externally.
sorted_discovered = sorted(discovered_folders, key=lambda x: x.lower())
folders_changed = self._cache.all_folders != sorted_discovered
if folders_changed:
self._cache.all_folders = sorted_discovered
# Resort cache if changes were made
if total_added > 0 or total_removed > 0:
# Update folders list
@@ -1259,6 +1283,8 @@ class ModelScanner:
await self._cache.resort()
await self._persist_current_cache()
elif folders_changed:
await self._persist_current_cache()
logger.info(f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in {time.time() - start_time:.2f} seconds. Added {total_added}, removed {total_removed} models.")
await self._broadcast_scan_progress(
@@ -1298,22 +1324,73 @@ class ModelScanner:
raise NotImplementedError("Subclasses must implement get_model_roots")
async def get_all_folders(self) -> List[str]:
"""Return every known directory under the model roots.
The directory list (including empty ones) is recorded during cache
scans and hydrated from the persisted snapshot, so this is a pure
in-memory read — no filesystem walk ever runs on the event loop
(walking network roots synchronously used to freeze the whole
server, see issue #1110). The result is unioned with the
model-derived folders so it is always a superset of
``cache.folders``.
Cold fallback: when the cache was hydrated from a persisted snapshot
that predates folder recording (``all_folders is None``), a one-shot
background walk is scheduled off the event loop to backfill and
persist the list; until it lands, the models-only folders are
returned.
"""
folders: Set[str] = set()
cache = self._cache
if cache is not None:
folders |= {item.get('folder', '') for item in cache.raw_data}
recorded = getattr(cache, 'all_folders', None)
if recorded is None:
self._schedule_all_folders_backfill()
else:
folders |= set(recorded)
else:
self._schedule_all_folders_backfill()
return sorted(folders, key=lambda x: x.lower())
def _schedule_all_folders_backfill(self) -> None:
"""Kick off a one-shot background folder walk if none is running."""
if self._all_folders_backfill_running:
return
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return
self._all_folders_backfill_running = True
loop.create_task(self._run_all_folders_backfill())
async def _run_all_folders_backfill(self) -> None:
"""Walk the roots in a worker thread, then record and persist the result."""
try:
loop = asyncio.get_running_loop()
folders = await loop.run_in_executor(None, self._walk_all_folders_sync)
cache = self._cache
# A scan may have recorded the list while the walk was in flight;
# prefer the fresher scan data in that case.
if cache is not None and cache.all_folders is None:
cache.all_folders = folders
await self._persist_current_cache()
except Exception as exc:
logger.warning(
"%s Scanner: all-folders backfill failed: %s",
self.model_type.capitalize(),
exc,
)
finally:
self._all_folders_backfill_running = False
def _walk_all_folders_sync(self) -> List[str]:
"""Enumerate every directory under the model roots, live from disk.
Unlike the models-only ``cache.folders``, this includes empty
directories, so it stays accurate even when the in-memory cache was
hydrated from a persisted snapshot without a filesystem walk. Hidden
directories (any segment starting with '.') and the pending-delete
staging dir are excluded. The result is unioned with the model-derived
folders so it is always a superset of ``cache.folders``, and cached
for ``ALL_FOLDERS_CACHE_TTL_SECONDS`` to avoid repeated walks.
Runs in a worker thread. Hidden directories (any segment starting
with '.') and the pending-delete staging dir are excluded.
"""
now = time.monotonic()
if self._all_folders_ttl_cache is not None:
cached_at, cached_folders = self._all_folders_ttl_cache
if now - cached_at < ALL_FOLDERS_CACHE_TTL_SECONDS:
return cached_folders
discovered: Set[str] = set()
visited_real_paths: Set[str] = set()
@@ -1335,17 +1412,7 @@ class ModelScanner:
if rel_dir != "." and not _is_hidden_relative_path(rel_dir):
discovered.add(rel_dir)
folders = set(discovered)
if self._cache is not None:
folders |= {item.get('folder', '') for item in self._cache.raw_data}
result = sorted(folders, key=lambda x: x.lower())
self._all_folders_ttl_cache = (now, result)
return result
def invalidate_all_folders_cache(self) -> None:
"""Drop the cached get_all_folders() result (e.g. after a move)."""
self._all_folders_ttl_cache = None
return sorted(discovered, key=lambda x: x.lower())
async def _create_default_metadata(self, file_path: str) -> Optional[BaseModelMetadata]:
"""Get model file info and metadata (extensible for different model types)"""
@@ -1569,6 +1636,9 @@ class ModelScanner:
else:
self._cache.raw_data = list(scan_result.raw_data)
if scan_result.all_folders is not None:
self._cache.all_folders = list(scan_result.all_folders)
# resort() rebuilds folders and the version index on every path, so a
# separate rebuild_version_index() call here would be redundant.
await self._cache.resort()
@@ -1666,6 +1736,7 @@ class ModelScanner:
processed_files = 0
processed_real_files: Set[str] = set()
visited_real_dirs: Set[str] = set()
discovered_folders: Set[str] = set()
async def handle_progress(current_name: str = '') -> None:
if progress_callback is None:
@@ -1744,6 +1815,13 @@ class ModelScanner:
elif entry.is_dir(follow_symlinks=True):
if _is_excluded_dir(entry.name):
continue
# Record every directory (including empty ones) so
# the folder tree can be served without a live walk.
rel_dir = os.path.relpath(
os.path.abspath(entry.path), os.path.abspath(root_path)
).replace(os.path.sep, "/")
if not _is_hidden_relative_path(rel_dir):
discovered_folders.add(rel_dir)
await scan_recursive(entry.path, root_path, visited_paths)
except Exception as entry_error:
logger.error(f"Error processing entry {entry.path}: {entry_error}")
@@ -1763,7 +1841,8 @@ class ModelScanner:
raw_data=raw_data,
hash_index=hash_index,
tags_count=tags_count,
excluded_models=excluded_models
excluded_models=excluded_models,
all_folders=sorted(discovered_folders, key=lambda x: x.lower()),
)
async def add_model_to_cache(self, metadata_dict: Dict[str, Any], folder: str = '') -> bool:
@@ -2020,6 +2099,16 @@ class ModelScanner:
all_folders = set(item['folder'] for item in cache.raw_data)
cache.folders = sorted(list(all_folders), key=lambda x: x.lower())
# The move target may live in directories the last scan never saw;
# record the destination folder (and its parents) in the known
# folder list so the folder tree reflects it without a rescan.
if cache.all_folders is not None and folder_value:
parts = folder_value.split("/")
known = set(cache.all_folders)
for i in range(1, len(parts) + 1):
known.add("/".join(parts[:i]))
cache.all_folders = sorted(known, key=lambda x: x.lower())
for tag in cache_entry.get('tags', []):
self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
@@ -2027,10 +2116,6 @@ class ModelScanner:
await cache.resort()
# A move may have created new directories; drop the cached live-walk
# result so the next include_empty request sees them.
self.invalidate_all_folders_cache()
if cache_modified:
await self._persist_current_cache()
self.bump_cache_version()