mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-06 22:10:14 -03:00
fix(example-images): move multi→single-library consolidation to startup, eliminate per-request os.listdir()
Move reverse-migration logic from get_model_folder() (hot path, called on every metadata/example-images request) to ExampleImagesMigration, where it runs once at startup. On network storage this was causing 22-38s delays per LoRA card click. Additionally optimize prune_stale_example_images() to read the directory listing once instead of per image entry (O(N*M) → O(M)). Also reorder consolidation checks so regex filters run before filesystem stat calls.
This commit is contained in:
@@ -475,13 +475,19 @@ class MetadataUpdater:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
model_folder = get_model_folder(model_hash)
|
model_folder = get_model_folder(model_hash)
|
||||||
if not model_folder:
|
if not model_folder or not os.path.isdir(model_folder):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
civitai = getattr(metadata, "civitai", None)
|
civitai = getattr(metadata, "civitai", None)
|
||||||
if not isinstance(civitai, dict):
|
if not isinstance(civitai, dict):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
# Read the directory listing once so every image entry reuses it.
|
||||||
|
try:
|
||||||
|
dir_entries = os.listdir(model_folder)
|
||||||
|
except OSError:
|
||||||
|
dir_entries = []
|
||||||
|
|
||||||
has_changes = False
|
has_changes = False
|
||||||
|
|
||||||
custom_images = civitai.get("customImages")
|
custom_images = civitai.get("customImages")
|
||||||
@@ -493,24 +499,15 @@ class MetadataUpdater:
|
|||||||
if not img_id:
|
if not img_id:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not os.path.isdir(model_folder):
|
prefix = f"custom_{img_id}"
|
||||||
|
found = any(
|
||||||
|
f.startswith(prefix) and os.path.isfile(
|
||||||
|
os.path.join(model_folder, f)
|
||||||
|
)
|
||||||
|
for f in dir_entries
|
||||||
|
)
|
||||||
|
if not found:
|
||||||
stale.append(idx)
|
stale.append(idx)
|
||||||
else:
|
|
||||||
found = False
|
|
||||||
try:
|
|
||||||
prefix = f"custom_{img_id}"
|
|
||||||
for fname in os.listdir(model_folder):
|
|
||||||
if fname.startswith(prefix) and os.path.isfile(
|
|
||||||
os.path.join(model_folder, fname)
|
|
||||||
):
|
|
||||||
found = True
|
|
||||||
break
|
|
||||||
except OSError:
|
|
||||||
stale.append(idx)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not found:
|
|
||||||
stale.append(idx)
|
|
||||||
|
|
||||||
if stale:
|
if stale:
|
||||||
for idx in reversed(stale):
|
for idx in reversed(stale):
|
||||||
@@ -532,22 +529,9 @@ class MetadataUpdater:
|
|||||||
# is gone.
|
# is gone.
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not os.path.isdir(model_folder):
|
prefix = f"image_{idx}."
|
||||||
|
if not any(f.startswith(prefix) for f in dir_entries):
|
||||||
stale.append(idx)
|
stale.append(idx)
|
||||||
else:
|
|
||||||
found = False
|
|
||||||
try:
|
|
||||||
prefix = f"image_{idx}."
|
|
||||||
for fname in os.listdir(model_folder):
|
|
||||||
if fname.startswith(prefix):
|
|
||||||
found = True
|
|
||||||
break
|
|
||||||
except OSError:
|
|
||||||
stale.append(idx)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not found:
|
|
||||||
stale.append(idx)
|
|
||||||
|
|
||||||
if stale:
|
if stale:
|
||||||
for idx in reversed(stale):
|
for idx in reversed(stale):
|
||||||
|
|||||||
@@ -3,9 +3,16 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
|
import shutil
|
||||||
from ..services.settings_manager import get_settings_manager
|
from ..services.settings_manager import get_settings_manager
|
||||||
from ..services.service_registry import ServiceRegistry
|
from ..services.service_registry import ServiceRegistry
|
||||||
from ..utils.example_images_paths import iter_library_roots
|
from ..utils.example_images_paths import (
|
||||||
|
get_example_images_root,
|
||||||
|
is_hash_folder,
|
||||||
|
iter_library_roots,
|
||||||
|
uses_library_scoped_folders,
|
||||||
|
_library_folder_has_only_hash_dirs,
|
||||||
|
)
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from ..utils.example_images_processor import ExampleImagesProcessor
|
from ..utils.example_images_processor import ExampleImagesProcessor
|
||||||
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
||||||
@@ -36,6 +43,90 @@ settings = _SettingsProxy()
|
|||||||
class ExampleImagesMigration:
|
class ExampleImagesMigration:
|
||||||
"""Handles migrations for example images naming conventions"""
|
"""Handles migrations for example images naming conventions"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _consolidate_library_folders():
|
||||||
|
"""Move hash folders from library-named subdirectories back to root.
|
||||||
|
|
||||||
|
When a user switches from multi-library mode back to single-library
|
||||||
|
mode, example images previously stored under e.g.
|
||||||
|
``<root>/default/<hash>/`` need to be moved back to
|
||||||
|
``<root>/<hash>/``. Running this once at startup removes the need
|
||||||
|
for ``get_model_folder()`` to perform directory scans on every
|
||||||
|
request.
|
||||||
|
"""
|
||||||
|
if uses_library_scoped_folders():
|
||||||
|
return
|
||||||
|
|
||||||
|
root = get_example_images_root()
|
||||||
|
if not root or not os.path.isdir(root):
|
||||||
|
return
|
||||||
|
|
||||||
|
moved: list[str] = []
|
||||||
|
cleaned: list[str] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
for entry in os.listdir(root):
|
||||||
|
# Fast regex checks first — no filesystem I/O.
|
||||||
|
if is_hash_folder(entry) or entry == "_deleted":
|
||||||
|
continue
|
||||||
|
|
||||||
|
entry_path = os.path.join(root, entry)
|
||||||
|
if not os.path.isdir(entry_path):
|
||||||
|
continue
|
||||||
|
if not _library_folder_has_only_hash_dirs(entry_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
for hash_entry in os.listdir(entry_path):
|
||||||
|
hash_path = os.path.join(entry_path, hash_entry)
|
||||||
|
if not os.path.isdir(hash_path) or not is_hash_folder(hash_entry):
|
||||||
|
continue
|
||||||
|
target = os.path.join(root, hash_entry)
|
||||||
|
if not os.path.exists(target):
|
||||||
|
try:
|
||||||
|
shutil.move(hash_path, target)
|
||||||
|
moved.append(hash_entry)
|
||||||
|
except (OSError, shutil.Error) as exc:
|
||||||
|
logger.error(
|
||||||
|
"Failed to move '%s' → '%s': %s",
|
||||||
|
hash_path, target, exc,
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.error(
|
||||||
|
"Failed to list library subdirectory '%s': %s",
|
||||||
|
entry_path, exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
remaining = os.listdir(entry_path)
|
||||||
|
except OSError:
|
||||||
|
remaining = []
|
||||||
|
if not remaining:
|
||||||
|
try:
|
||||||
|
os.rmdir(entry_path)
|
||||||
|
cleaned.append(entry)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.debug(
|
||||||
|
"Could not remove empty library dir '%s': %s",
|
||||||
|
entry_path, exc,
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.error(
|
||||||
|
"Failed to list example images root during consolidation: %s",
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
if moved:
|
||||||
|
logger.info(
|
||||||
|
"Consolidated %d example image folder(s) to root",
|
||||||
|
len(moved),
|
||||||
|
)
|
||||||
|
if cleaned:
|
||||||
|
logger.info(
|
||||||
|
"Removed %d empty library directories",
|
||||||
|
len(cleaned),
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def check_and_run_migrations():
|
async def check_and_run_migrations():
|
||||||
"""Check if migrations are needed and run them in background"""
|
"""Check if migrations are needed and run them in background"""
|
||||||
@@ -44,6 +135,10 @@ class ExampleImagesMigration:
|
|||||||
logger.debug("No example images path configured or path doesn't exist, skipping migrations")
|
logger.debug("No example images path configured or path doesn't exist, skipping migrations")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Run library-to-root consolidation once at startup so the hot
|
||||||
|
# path (get_model_folder) stays a pure-path computation.
|
||||||
|
ExampleImagesMigration._consolidate_library_folders()
|
||||||
|
|
||||||
for library_name, library_path in iter_library_roots():
|
for library_name, library_path in iter_library_roots():
|
||||||
if not library_path or not os.path.exists(library_path):
|
if not library_path or not os.path.exists(library_path):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -83,7 +83,12 @@ def ensure_library_root_exists(library_name: Optional[str] = None) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def get_model_folder(model_hash: str, library_name: Optional[str] = None) -> str:
|
def get_model_folder(model_hash: str, library_name: Optional[str] = None) -> str:
|
||||||
"""Return the folder path for a model's example images."""
|
"""Return the folder path for a model's example images.
|
||||||
|
|
||||||
|
Multi-library ↔ single-library consolidation is handled once at startup by
|
||||||
|
``ExampleImagesMigration._consolidate_library_folders`` — this function is a
|
||||||
|
pure path computation on the hot path (no directory scans).
|
||||||
|
"""
|
||||||
|
|
||||||
if not model_hash:
|
if not model_hash:
|
||||||
return ""
|
return ""
|
||||||
@@ -113,35 +118,6 @@ def get_model_folder(model_hash: str, library_name: Optional[str] = None) -> str
|
|||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
return legacy_folder
|
return legacy_folder
|
||||||
elif not os.path.exists(resolved_folder):
|
|
||||||
# Reverse migration: when consolidating from multi-library to
|
|
||||||
# single-library mode (e.g. after "default" was cleaned up), look
|
|
||||||
# for existing example images inside library-named subdirectories
|
|
||||||
# and bring them back to the root level.
|
|
||||||
root = get_example_images_root()
|
|
||||||
if root:
|
|
||||||
try:
|
|
||||||
for entry in os.listdir(root):
|
|
||||||
entry_path = os.path.join(root, entry)
|
|
||||||
if not os.path.isdir(entry_path):
|
|
||||||
continue
|
|
||||||
if is_hash_folder(entry) or entry == "_deleted":
|
|
||||||
continue
|
|
||||||
if not _library_folder_has_only_hash_dirs(entry_path):
|
|
||||||
continue
|
|
||||||
legacy = os.path.join(entry_path, normalized_hash)
|
|
||||||
if os.path.exists(legacy):
|
|
||||||
shutil.move(legacy, resolved_folder)
|
|
||||||
logger.info(
|
|
||||||
"Consolidated example images from '%s' to '%s'",
|
|
||||||
legacy, resolved_folder,
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except OSError as exc:
|
|
||||||
logger.error(
|
|
||||||
"Failed to consolidate example images during "
|
|
||||||
"library merge: %s", exc,
|
|
||||||
)
|
|
||||||
|
|
||||||
return resolved_folder
|
return resolved_folder
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user