mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 11:11:26 -03:00
feat(settings): filename templates for download and bulk rename (#1071)
Add per-model-type filename templates ({model_name}, {version_name},
{base_model}, {author}, {first_tag}, {hash_short}, {original_name}) so
downloaded files get informative names instead of e.g. V1.safetensors.
Empty template keeps the current filename (opt-in, off by default).
- apply template automatically after downloads; rename conflicts keep
the original name and never fail the download
- record original_file_name in metadata on rename for traceability
- bulk apply via GET|POST /api/lm/{prefix}/apply-filename-template with
WebSocket progress, sharing the auto-organize lock
- settings UI lives in the new Organization tab with validation, live
preview, and per-type 'apply to library' actions
This commit is contained in:
@@ -33,7 +33,7 @@ from ..utils.constants import (
|
||||
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
|
||||
from ..utils.file_utils import calculate_sha256, calculate_autov3
|
||||
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
|
||||
from ..utils.utils import sanitize_folder_name
|
||||
from ..utils.utils import calculate_filename_for_model, sanitize_folder_name
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from .service_registry import ServiceRegistry
|
||||
@@ -45,6 +45,7 @@ from .errors import RateLimitError
|
||||
from .aria2_downloader import Aria2Error, get_aria2_downloader
|
||||
from .aria2_transfer_state import Aria2TransferStateStore
|
||||
from .download_queue_service import DownloadQueueService
|
||||
from .model_lifecycle_service import ModelLifecycleService, load_local_metadata
|
||||
|
||||
# Download to temporary file first
|
||||
import tempfile
|
||||
@@ -2746,6 +2747,7 @@ class DownloadManager:
|
||||
else None
|
||||
)
|
||||
|
||||
downloaded_metadata: List[Dict[str, Any]] = []
|
||||
for index, entry in enumerate(metadata_entries):
|
||||
file_path_for_adjust = getattr(
|
||||
entry, "file_path", actual_file_paths[index]
|
||||
@@ -2788,6 +2790,15 @@ class DownloadManager:
|
||||
if scanner is not None:
|
||||
await scanner.add_model_to_cache(metadata_dict, relative_path)
|
||||
|
||||
downloaded_metadata.append(metadata_dict)
|
||||
|
||||
await self._apply_download_filename_template(
|
||||
scanner=scanner,
|
||||
model_type=model_type,
|
||||
downloaded_metadata=downloaded_metadata,
|
||||
download_id=download_id,
|
||||
)
|
||||
|
||||
if transfer_backend == "aria2" and download_id:
|
||||
await self._aria2_state_store.remove(download_id)
|
||||
|
||||
@@ -2827,6 +2838,83 @@ class DownloadManager:
|
||||
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def _apply_download_filename_template(
|
||||
self,
|
||||
*,
|
||||
scanner,
|
||||
model_type: str,
|
||||
downloaded_metadata: List[Dict[str, Any]],
|
||||
download_id: Optional[str],
|
||||
) -> None:
|
||||
"""Rename freshly downloaded models according to the filename template.
|
||||
|
||||
Best-effort post-download step: any failure (including name conflicts)
|
||||
is logged and skipped so a successful download is never turned into a
|
||||
failure by a rename problem.
|
||||
"""
|
||||
try:
|
||||
if scanner is None or not downloaded_metadata:
|
||||
return
|
||||
|
||||
template = get_settings_manager().get_download_filename_template(
|
||||
model_type
|
||||
)
|
||||
if not template:
|
||||
return
|
||||
|
||||
lifecycle_service = ModelLifecycleService(
|
||||
scanner=scanner,
|
||||
metadata_manager=MetadataManager,
|
||||
metadata_loader=load_local_metadata,
|
||||
recipe_scanner_factory=ServiceRegistry.get_recipe_scanner,
|
||||
)
|
||||
|
||||
for metadata_dict in downloaded_metadata:
|
||||
file_path = metadata_dict.get("file_path")
|
||||
if not isinstance(file_path, str) or not file_path:
|
||||
continue
|
||||
|
||||
new_stem = calculate_filename_for_model(metadata_dict, model_type)
|
||||
if not new_stem:
|
||||
continue
|
||||
|
||||
current_stem = os.path.splitext(os.path.basename(file_path))[0]
|
||||
if new_stem == current_stem or os.path.normcase(
|
||||
new_stem
|
||||
) == os.path.normcase(current_stem):
|
||||
continue
|
||||
|
||||
try:
|
||||
result = await lifecycle_service.rename_model(
|
||||
file_path=file_path, new_file_name=new_stem
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.warning(
|
||||
"Keeping original filename for %s: %s", file_path, exc
|
||||
)
|
||||
continue
|
||||
|
||||
new_file_path = result.get("new_file_path")
|
||||
if download_id and isinstance(new_file_path, str):
|
||||
info = self._active_downloads.get(download_id)
|
||||
if info is None:
|
||||
continue
|
||||
if info.get("file_path") == file_path:
|
||||
info["file_path"] = new_file_path
|
||||
extracted = info.get("extracted_paths")
|
||||
if isinstance(extracted, list):
|
||||
info["extracted_paths"] = [
|
||||
new_file_path if path == file_path else path
|
||||
for path in extracted
|
||||
]
|
||||
except Exception as exc: # Rename phase must never fail the download
|
||||
logger.warning(
|
||||
"Filename template rename failed for %s download: %s",
|
||||
model_type,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _get_supported_extensions_for_type(self, model_type: str) -> Set[str]:
|
||||
if model_type in ("checkpoint", "other"):
|
||||
return {
|
||||
|
||||
@@ -43,10 +43,22 @@ class AutoOrganizeResult:
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary"""
|
||||
if self.operation_type == 'filename_template':
|
||||
message = (
|
||||
f'Filename template applied: {self.success_count} renamed, '
|
||||
f'{self.skipped_count} skipped, {self.failure_count} failed '
|
||||
f'out of {self.total} total'
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
f'Auto-organize {self.operation_type} completed: '
|
||||
f'{self.success_count} moved, {self.skipped_count} skipped, '
|
||||
f'{self.failure_count} failed out of {self.total} total'
|
||||
)
|
||||
result: Dict[str, Any] = {
|
||||
'success': self.status != 'error',
|
||||
'status': self.status,
|
||||
'message': f'Auto-organize {self.operation_type} completed: {self.success_count} moved, {self.skipped_count} skipped, {self.failure_count} failed out of {self.total} total',
|
||||
'message': message,
|
||||
'summary': {
|
||||
'total': self.total,
|
||||
'success': self.success_count,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, TYPE_CHECKING, cast
|
||||
@@ -17,6 +18,26 @@ if TYPE_CHECKING:
|
||||
from ..services.model_update_service import ModelUpdateService
|
||||
|
||||
|
||||
async def load_local_metadata(metadata_path: str) -> Dict[str, Any]:
|
||||
"""Load a metadata sidecar JSON, returning an empty dict when missing.
|
||||
|
||||
Thin equivalent of ``MetadataSyncService.load_local_metadata`` for callers
|
||||
(download manager, use cases) that do not hold a sync-service instance.
|
||||
"""
|
||||
|
||||
if not os.path.exists(metadata_path):
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load metadata from %s: %s", metadata_path, exc)
|
||||
return {}
|
||||
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
async def delete_model_artifacts(
|
||||
target_dir: str, file_name: str, main_extension: str | None = None
|
||||
) -> List[str]:
|
||||
@@ -404,6 +425,9 @@ class ModelLifecycleService:
|
||||
if metadata and new_metadata_path:
|
||||
metadata["file_name"] = new_file_name
|
||||
metadata["file_path"] = new_file_path
|
||||
# Preserve the pre-rename stem so the original download filename
|
||||
# stays recoverable after template-driven renames.
|
||||
metadata.setdefault("original_file_name", old_file_name)
|
||||
|
||||
if metadata.get("preview_url"):
|
||||
old_preview = str(metadata["preview_url"])
|
||||
|
||||
@@ -98,6 +98,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"recipes_path": "",
|
||||
"base_model_path_mappings": {},
|
||||
"download_path_templates": {},
|
||||
"download_filename_templates": {},
|
||||
"folder_paths": {},
|
||||
"extra_folder_paths": {},
|
||||
"example_images_path": "",
|
||||
@@ -1276,6 +1277,7 @@ class SettingsManager:
|
||||
defaults = copy.deepcopy(DEFAULT_SETTINGS)
|
||||
defaults["base_model_path_mappings"] = {}
|
||||
defaults["download_path_templates"] = {}
|
||||
defaults["download_filename_templates"] = {}
|
||||
defaults["priority_tags"] = DEFAULT_PRIORITY_TAG_CONFIG.copy()
|
||||
defaults.setdefault("folder_paths", {})
|
||||
defaults.setdefault("extra_folder_paths", {})
|
||||
@@ -2424,6 +2426,49 @@ class SettingsManager:
|
||||
model_type, DEFAULT_DOWNLOAD_PATH_TEMPLATES.get(model_type, "")
|
||||
)
|
||||
|
||||
def get_download_filename_template(self, model_type: str) -> str:
|
||||
"""Get the download filename template for a specific model type.
|
||||
|
||||
Args:
|
||||
model_type: The type of model ('lora', 'checkpoint', 'embedding',
|
||||
'other')
|
||||
|
||||
Returns:
|
||||
Template string for the model type. Empty string (the default for
|
||||
every model type) means downloaded files keep their original
|
||||
filename.
|
||||
"""
|
||||
templates = self.settings.get("download_filename_templates", {})
|
||||
|
||||
# Handle edge case where templates might be stored as JSON string
|
||||
if isinstance(templates, str):
|
||||
try:
|
||||
parsed_templates = json.loads(templates)
|
||||
if isinstance(parsed_templates, dict):
|
||||
self.settings["download_filename_templates"] = parsed_templates
|
||||
self._save_settings()
|
||||
templates = parsed_templates
|
||||
logger.info(
|
||||
"Successfully parsed download_filename_templates from JSON string"
|
||||
)
|
||||
else:
|
||||
raise ValueError("Parsed JSON is not a dictionary")
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning(
|
||||
f"Failed to parse download_filename_templates JSON string: {e}. Resetting to empty templates."
|
||||
)
|
||||
templates = {}
|
||||
self.settings["download_filename_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
if not isinstance(templates, dict):
|
||||
templates = {}
|
||||
self.settings["download_filename_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
template = templates.get(model_type, "")
|
||||
return template if isinstance(template, str) else ""
|
||||
|
||||
|
||||
_SETTINGS_MANAGER: Optional["SettingsManager"] = None
|
||||
_SETTINGS_MANAGER_LOCK = Lock()
|
||||
|
||||
@@ -20,6 +20,7 @@ from .example_images import (
|
||||
ImportExampleImagesUseCase,
|
||||
ImportExampleImagesValidationError,
|
||||
)
|
||||
from .filename_template_use_case import FilenameTemplateUseCase
|
||||
|
||||
__all__ = [
|
||||
"AutoOrganizeInProgressError",
|
||||
@@ -34,4 +35,5 @@ __all__ = [
|
||||
"DownloadExampleImagesUseCase",
|
||||
"ImportExampleImagesUseCase",
|
||||
"ImportExampleImagesValidationError",
|
||||
"FilenameTemplateUseCase",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Filename template use case: bulk-rename library models per the configured template."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from ...utils.constants import AUTO_ORGANIZE_BATCH_SIZE
|
||||
from ...utils.utils import calculate_filename_for_model
|
||||
from ..model_file_service import AutoOrganizeResult, ProgressCallback
|
||||
from ..model_lifecycle_service import ModelLifecycleService
|
||||
from ..settings_manager import get_settings_manager
|
||||
from .auto_organize_use_case import (
|
||||
AutoOrganizeInProgressError,
|
||||
AutoOrganizeLockProvider,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PROGRESS_TYPE = "filename_template_progress"
|
||||
|
||||
|
||||
class FilenameTemplateUseCase:
|
||||
"""Apply the download filename template to existing library models.
|
||||
|
||||
Shares the auto-organize lock (and its in-progress error) so a bulk
|
||||
rename never runs concurrently with an auto-organize operation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
scanner,
|
||||
lifecycle_service: ModelLifecycleService,
|
||||
lock_provider: AutoOrganizeLockProvider,
|
||||
model_type: str,
|
||||
) -> None:
|
||||
self._scanner = scanner
|
||||
self._lifecycle_service = lifecycle_service
|
||||
self._lock_provider = lock_provider
|
||||
self._model_type = model_type
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
*,
|
||||
file_paths: Optional[Sequence[str]] = None,
|
||||
progress_callback: Optional[ProgressCallback] = None,
|
||||
) -> AutoOrganizeResult:
|
||||
"""Run the bulk rename guarded by the shared library-operation lock."""
|
||||
|
||||
is_running = getattr(self._lock_provider, "is_filename_template_running", None)
|
||||
if callable(is_running) and is_running():
|
||||
raise AutoOrganizeInProgressError(
|
||||
"A filename template operation is already running"
|
||||
)
|
||||
if self._lock_provider.is_auto_organize_running():
|
||||
raise AutoOrganizeInProgressError("Auto-organize is already running")
|
||||
|
||||
lock = await self._lock_provider.get_auto_organize_lock()
|
||||
if lock.locked():
|
||||
raise AutoOrganizeInProgressError(
|
||||
"Another library operation is already running"
|
||||
)
|
||||
|
||||
async with lock:
|
||||
return await self._run(
|
||||
file_paths=file_paths, progress_callback=progress_callback
|
||||
)
|
||||
|
||||
async def _run(
|
||||
self,
|
||||
*,
|
||||
file_paths: Optional[Sequence[str]],
|
||||
progress_callback: Optional[ProgressCallback],
|
||||
) -> AutoOrganizeResult:
|
||||
result = AutoOrganizeResult()
|
||||
result.operation_type = "filename_template"
|
||||
|
||||
self._scanner.reset_cancellation()
|
||||
|
||||
try:
|
||||
template = get_settings_manager().get_download_filename_template(
|
||||
self._model_type
|
||||
)
|
||||
|
||||
cache = await self._scanner.get_cached_data()
|
||||
models = list(cache.raw_data)
|
||||
if file_paths:
|
||||
wanted = set(file_paths)
|
||||
models = [
|
||||
model for model in models if model.get("file_path") in wanted
|
||||
]
|
||||
|
||||
result.total = len(models)
|
||||
|
||||
await self._emit_progress(progress_callback, result, "started")
|
||||
|
||||
for index in range(0, result.total, AUTO_ORGANIZE_BATCH_SIZE):
|
||||
if self._scanner.is_cancelled():
|
||||
logger.info(
|
||||
"Filename template apply cancelled for %s", self._model_type
|
||||
)
|
||||
break
|
||||
|
||||
batch = models[index : index + AUTO_ORGANIZE_BATCH_SIZE]
|
||||
for model in batch:
|
||||
if self._scanner.is_cancelled():
|
||||
break
|
||||
await self._process_model(model, template, result)
|
||||
result.processed += 1
|
||||
|
||||
await self._emit_progress(progress_callback, result, "processing")
|
||||
# Yield between batches so the server stays responsive.
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
if self._scanner.is_cancelled():
|
||||
result.status = "cancelled"
|
||||
await self._emit_progress(progress_callback, result, "cancelled")
|
||||
return result
|
||||
|
||||
await self._emit_progress(progress_callback, result, "completed")
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Error in filename template apply: %s", exc, exc_info=True)
|
||||
if progress_callback:
|
||||
await progress_callback.on_progress(
|
||||
{
|
||||
"type": _PROGRESS_TYPE,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
"operation_type": result.operation_type,
|
||||
}
|
||||
)
|
||||
raise
|
||||
|
||||
async def _process_model(
|
||||
self,
|
||||
model: Dict[str, Any],
|
||||
template: str,
|
||||
result: AutoOrganizeResult,
|
||||
) -> None:
|
||||
model_name = model.get("model_name", "Unknown")
|
||||
try:
|
||||
file_path = model.get("file_path")
|
||||
if not file_path:
|
||||
self._add_result(result, model_name, False, "No file path found")
|
||||
result.failure_count += 1
|
||||
return
|
||||
|
||||
if not template:
|
||||
result.skipped_count += 1
|
||||
return
|
||||
|
||||
new_stem = calculate_filename_for_model(model, self._model_type)
|
||||
if not new_stem:
|
||||
result.skipped_count += 1
|
||||
return
|
||||
|
||||
current_stem = os.path.splitext(os.path.basename(file_path))[0]
|
||||
if new_stem == current_stem or os.path.normcase(
|
||||
new_stem
|
||||
) == os.path.normcase(current_stem):
|
||||
result.skipped_count += 1
|
||||
return
|
||||
|
||||
await self._lifecycle_service.rename_model(
|
||||
file_path=file_path, new_file_name=new_stem
|
||||
)
|
||||
result.success_count += 1
|
||||
|
||||
except ValueError as exc:
|
||||
# Conflicts (e.g. target name already exists) count as failures
|
||||
# without aborting the batch.
|
||||
self._add_result(result, model_name, False, str(exc))
|
||||
result.failure_count += 1
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Error applying filename template to %s: %s", model_name, exc,
|
||||
exc_info=True,
|
||||
)
|
||||
self._add_result(result, model_name, False, f"Error: {exc}")
|
||||
result.failure_count += 1
|
||||
|
||||
async def _emit_progress(
|
||||
self,
|
||||
progress_callback: Optional[ProgressCallback],
|
||||
result: AutoOrganizeResult,
|
||||
status: str,
|
||||
) -> None:
|
||||
if not progress_callback:
|
||||
return
|
||||
await progress_callback.on_progress(
|
||||
{
|
||||
"type": _PROGRESS_TYPE,
|
||||
"status": status,
|
||||
"total": result.total,
|
||||
"processed": result.processed,
|
||||
"success": result.success_count,
|
||||
"failures": result.failure_count,
|
||||
"skipped": result.skipped_count,
|
||||
"operation_type": result.operation_type,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _add_result(
|
||||
result: AutoOrganizeResult,
|
||||
model_name: str,
|
||||
success: bool,
|
||||
message: str,
|
||||
) -> None:
|
||||
"""Add a result entry if under the limit (mirrors ModelFileService)."""
|
||||
if len(result.results) < 100:
|
||||
result.results.append(
|
||||
{"model": model_name, "success": success, "message": message}
|
||||
)
|
||||
elif len(result.results) == 100:
|
||||
result.results_truncated = True
|
||||
result.sample_results = result.results[:50]
|
||||
@@ -20,6 +20,8 @@ class WebSocketManager:
|
||||
self._last_init_progress: Dict[str, Dict[str, Any]] = {}
|
||||
# Add auto-organize progress tracking
|
||||
self._auto_organize_progress: Optional[Dict[str, Any]] = None
|
||||
# Add filename template progress tracking
|
||||
self._filename_template_progress: Optional[Dict[str, Any]] = None
|
||||
# Add recipe rematch progress tracking
|
||||
self._recipe_rematch_progress: Optional[Dict[str, Any]] = None
|
||||
self._auto_organize_lock = asyncio.Lock()
|
||||
@@ -205,6 +207,26 @@ class WebSocketManager:
|
||||
def cleanup_auto_organize_progress(self):
|
||||
"""Clear auto-organize progress data"""
|
||||
self._auto_organize_progress = None
|
||||
|
||||
async def broadcast_filename_template_progress(self, data: Dict[str, Any]):
|
||||
"""Broadcast filename template progress to connected clients"""
|
||||
self._filename_template_progress = data
|
||||
await self.broadcast(data)
|
||||
|
||||
def get_filename_template_progress(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get current filename template progress"""
|
||||
return self._filename_template_progress
|
||||
|
||||
def cleanup_filename_template_progress(self):
|
||||
"""Clear filename template progress data"""
|
||||
self._filename_template_progress = None
|
||||
|
||||
def is_filename_template_running(self) -> bool:
|
||||
"""Check if a filename template operation is currently running"""
|
||||
if not self._filename_template_progress:
|
||||
return False
|
||||
status = self._filename_template_progress.get('status')
|
||||
return status in ['started', 'processing']
|
||||
|
||||
async def broadcast_recipe_rematch_progress(self, data: Dict[str, Any]):
|
||||
"""Broadcast recipe rematch progress to connected clients"""
|
||||
|
||||
@@ -21,6 +21,14 @@ class WebSocketProgressCallback(ProgressCallback):
|
||||
await ws_manager.broadcast_auto_organize_progress(progress_data)
|
||||
|
||||
|
||||
class WebSocketFilenameTemplateProgressCallback(ProgressCallback):
|
||||
"""WebSocket progress callback for filename template operations."""
|
||||
|
||||
async def on_progress(self, progress_data: Dict[str, Any]) -> None:
|
||||
"""Send filename template progress via WebSocket."""
|
||||
await ws_manager.broadcast_filename_template_progress(progress_data)
|
||||
|
||||
|
||||
class WebSocketBroadcastCallback:
|
||||
"""Generic WebSocket progress callback broadcasting to all clients."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user