alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-03-24 14:42:11 -03:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement batch import recipe functionality (frontend + backend fixes)

Browse Source 
Backend fixes:
- Add missing API route for /api/lm/recipes/batch-import/progress (GET)
- Add missing API route for /api/lm/recipes/batch-import/directory (POST)
- Add missing API route for /api/lm/recipes/browse-directory (POST)
- Register WebSocket endpoint for batch import progress
- Fix skip_no_metadata default value (True -> False) to allow no-LoRA imports
- Add items array to BatchImportProgress.to_dict() for detailed results

Frontend implementation:
- Create BatchImportManager.js with complete batch import workflow
- Add directory browser UI for selecting folders
- Add batch import modal with URL list and directory input modes
- Implement real-time progress tracking (WebSocket + HTTP polling)
- Add results summary with success/failed/skipped statistics
- Add expandable details view showing individual item status
- Auto-refresh recipe list after import completion

UI improvements:
- Add spinner animation for importing status
- Simplify results summary UI to match progress stats styling
- Fix current item text alignment
- Fix dark theme styling for directory browser button
- Fix batch import button styling consistency

Translations:
- Add batch import related i18n keys to all locale files
- Run sync_translation_keys.py to sync all translations

Fixes:
- Batch import now allows images without LoRAs (matches single import behavior)
- Progress endpoint now returns complete items array with status details
- Results view correctly displays skipped items with error messages
This commit is contained in:
Will Miao
2026-03-14 21:17:36 +08:00
parent f86651652c
commit ee466113d5
24 changed files with 2791 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
py/routes/handlers/recipe_handlers.py
View File

@@ -9,6 +9,7 @@ import re
import asyncio
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional
from aiohttp import web
@@ -89,6 +90,8 @@ class RecipeHandlerSet:
"start_batch_import": self.batch_import.start_batch_import,
"get_batch_import_progress": self.batch_import.get_batch_import_progress,
"cancel_batch_import": self.batch_import.cancel_batch_import,
"start_directory_import": self.batch_import.start_directory_import,
"browse_directory": self.batch_import.browse_directory,
}
@@ -1426,7 +1429,7 @@ class BatchImportHandler:
data = await request.json()
items = data.get("items", [])
tags = data.get("tags", [])
skip_no_metadata = data.get("skip_no_metadata", True)
skip_no_metadata = data.get("skip_no_metadata", False)
if not items:
return web.json_response(
@@ -1564,3 +1567,136 @@ class BatchImportHandler:
except Exception as exc:
self._logger.error("Error cancelling batch import: %s", exc, exc_info=True)
return web.json_response({"success": False, "error": str(exc)}, status=500)
async def browse_directory(self, request: web.Request) -> web.Response:
"""Browse a directory and return its contents (subdirectories and files)."""
try:
data = await request.json()
directory_path = data.get("path", "")
if not directory_path:
return web.json_response(
{"success": False, "error": "Directory path is required"},
status=400,
)
# Normalize the path
path = Path(directory_path).expanduser().resolve()
# Security check: ensure path is within allowed directories
# Allow common image/model directories
allowed_roots = [
Path.home(),
Path("/"), # Allow browsing from root for flexibility
]
# Check if path is within any allowed root
is_allowed = False
for root in allowed_roots:
try:
path.relative_to(root)
is_allowed = True
break
except ValueError:
continue
if not is_allowed:
return web.json_response(
{"success": False, "error": "Access denied to this directory"},
status=403,
)
if not path.exists():
return web.json_response(
{"success": False, "error": "Directory does not exist"},
status=404,
)
if not path.is_dir():
return web.json_response(
{"success": False, "error": "Path is not a directory"},
status=400,
)
# List directory contents
directories = []
image_files = []
image_extensions = {
".jpg",
".jpeg",
".png",
".gif",
".webp",
".bmp",
".tiff",
".tif",
}
try:
for item in path.iterdir():
try:
if item.is_dir():
# Skip hidden directories and common system folders
if not item.name.startswith(".") and item.name not in [
"__pycache__",
"node_modules",
]:
directories.append(
{
"name": item.name,
"path": str(item),
"is_parent": False,
}
)
elif item.is_file() and item.suffix.lower() in image_extensions:
image_files.append(
{
"name": item.name,
"path": str(item),
"size": item.stat().st_size,
}
)
except (PermissionError, OSError):
# Skip files/directories we can't access
continue
# Sort directories and files alphabetically
directories.sort(key=lambda x: x["name"].lower())
image_files.sort(key=lambda x: x["name"].lower())
# Add parent directory if not at root
parent_path = path.parent
show_parent = str(path) != str(path.root)
return web.json_response(
{
"success": True,
"current_path": str(path),
"parent_path": str(parent_path) if show_parent else None,
"directories": directories,
"image_files": image_files,
"image_count": len(image_files),
"directory_count": len(directories),
}
)
except PermissionError:
return web.json_response(
{"success": False, "error": "Permission denied"},
status=403,
)
except OSError as exc:
return web.json_response(
{"success": False, "error": f"Error reading directory: {str(exc)}"},
status=500,
)
except json.JSONDecodeError:
return web.json_response(
{"success": False, "error": "Invalid JSON"},
status=400,
)
except Exception as exc:
self._logger.error("Error browsing directory: %s", exc, exc_info=True)
return web.json_response({"success": False, "error": str(exc)}, status=500)

4
py/routes/recipe_route_registrar.py
View File

@@ -63,6 +63,10 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
RouteDefinition(
"POST", "/api/lm/recipes/batch-import/cancel", "cancel_batch_import"
),
RouteDefinition(
"POST", "/api/lm/recipes/batch-import/directory", "start_directory_import"
),
RouteDefinition("POST", "/api/lm/recipes/browse-directory", "browse_directory"),
)

22
py/services/batch_import_service.py
View File

@@ -69,7 +69,7 @@ class BatchImportProgress:
finished_at: Optional[float] = None
items: List[BatchImportItem] = field(default_factory=list)
tags: List[str] = field(default_factory=list)
skip_no_metadata: bool = True
skip_no_metadata: bool = False
skip_duplicates: bool = False
def to_dict(self) -> Dict[str, Any]:
@@ -87,6 +87,19 @@ class BatchImportProgress:
"progress_percent": round((self.completed / self.total) * 100, 1)
if self.total > 0
else 0,
"items": [
{
"id": item.id,
"source": item.source,
"item_type": item.item_type.value,
"status": item.status.value,
"error_message": item.error_message,
"recipe_name": item.recipe_name,
"recipe_id": item.recipe_id,
"duration": item.duration,
}
for item in self.items
],
}
@@ -226,7 +239,7 @@ class BatchImportService:
civitai_client_getter: Callable[[], Any],
items: List[Dict[str, str]],
tags: Optional[List[str]] = None,
skip_no_metadata: bool = True,
skip_no_metadata: bool = False,
skip_duplicates: bool = False,
) -> str:
operation_id = str(uuid.uuid4())
@@ -278,7 +291,7 @@ class BatchImportService:
directory: str,
recursive: bool = True,
tags: Optional[List[str]] = None,
skip_no_metadata: bool = True,
skip_no_metadata: bool = False,
skip_duplicates: bool = False,
) -> str:
image_paths = await self._discover_images(directory, recursive)
@@ -494,7 +507,8 @@ class BatchImportService:
"skipped": True,
"error": "No LoRAs found in image",
}
return {"success": False, "error": "No LoRAs found in image"}
# When skip_no_metadata is False, allow importing images without LoRAs
# Continue with empty loras list
recipe_name = self._generate_recipe_name(item, payload)
all_tags = list(set(tags + (payload.get("tags", []) or [])))

43
py/services/download_manager.py
View File

@@ -10,7 +10,11 @@ import uuid
from typing import Dict, List, Optional, Set, Tuple
from urllib.parse import urlparse
from ..utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
from ..utils.constants import CARD_PREVIEW_WIDTH, DIFFUSION_MODEL_BASE_MODELS, VALID_LORA_TYPES
from ..utils.constants import (
CARD_PREVIEW_WIDTH,
DIFFUSION_MODEL_BASE_MODELS,
VALID_LORA_TYPES,
)
from ..utils.civitai_utils import rewrite_preview_url
from ..utils.preview_selection import select_preview_media
from ..utils.utils import sanitize_folder_name
@@ -352,10 +356,12 @@ class DownloadManager:
# Check if this checkpoint should be treated as a diffusion model based on baseModel
is_diffusion_model = False
if model_type == "checkpoint":
base_model_value = version_info.get('baseModel', '')
base_model_value = version_info.get("baseModel", "")
if base_model_value in DIFFUSION_MODEL_BASE_MODELS:
is_diffusion_model = True
logger.info(f"baseModel '{base_model_value}' is a known diffusion model, routing to unet folder")
logger.info(
f"baseModel '{base_model_value}' is a known diffusion model, routing to unet folder"
)
# Case 2: model_version_id was None, check after getting version_info
if model_version_id is None:
@@ -464,7 +470,7 @@ class DownloadManager:
# 2. Get file information
files = version_info.get("files", [])
file_info = None
# If file_params is provided, try to find matching file
if file_params and model_version_id:
target_type = file_params.get("type", "Model")
@@ -472,23 +478,28 @@ class DownloadManager:
target_size = file_params.get("size", "full")
target_fp = file_params.get("fp")
is_primary = file_params.get("isPrimary", False)
if is_primary:
# Find primary file
file_info = next(
(f for f in files if f.get("primary") and f.get("type") in ("Model", "Negative")),
None
(
f
for f in files
if f.get("primary")
and f.get("type") in ("Model", "Negative")
),
None,
)
else:
# Match by metadata
for f in files:
f_type = f.get("type", "")
f_meta = f.get("metadata", {})
# Check type match
if f_type != target_type:
continue
# Check metadata match
if f_meta.get("format") != target_format:
continue
@@ -496,10 +507,10 @@ class DownloadManager:
continue
if target_fp and f_meta.get("fp") != target_fp:
continue
file_info = f
break
# Fallback to primary file if no match found
if not file_info:
file_info = next(
@@ -510,7 +521,7 @@ class DownloadManager:
),
None,
)
if not file_info:
return {"success": False, "error": "No suitable file found in metadata"}
mirrors = file_info.get("mirrors") or []
@@ -1220,7 +1231,13 @@ class DownloadManager:
entries: List = []
for index, file_path in enumerate(file_paths):
entry = base_metadata if index == 0 else copy.deepcopy(base_metadata)
entry.update_file_info(file_path)
# Update file paths without modifying size and modified timestamps
# modified should remain as the download start time (import time)
# size will be updated below to reflect actual downloaded file size
entry.file_path = file_path.replace(os.sep, "/")
entry.file_name = os.path.splitext(os.path.basename(file_path))[0]
# Update size to actual downloaded file size
entry.size = os.path.getsize(file_path)
entry.sha256 = await calculate_sha256(file_path)
entries.append(entry)

227
py/utils/models.py
View File

@@ -4,32 +4,40 @@ from datetime import datetime
import os
from .model_utils import determine_base_model
@dataclass
class BaseModelMetadata:
"""Base class for all model metadata structures"""
file_name: str # The filename without extension
model_name: str # The model's name defined by the creator
file_path: str # Full path to the model file
size: int # File size in bytes
modified: float # Timestamp when the model was added to the management system
sha256: str # SHA256 hash of the file
base_model: str # Base model type (SD1.5/SD2.1/SDXL/etc.)
preview_url: str # Preview image URL
preview_nsfw_level: int = 0 # NSFW level of the preview image
notes: str = "" # Additional notes
from_civitai: bool = True # Whether from Civitai
civitai: Dict[str, Any] = field(default_factory=dict) # Civitai API data if available
tags: List[str] = None # Model tags
file_name: str # The filename without extension
model_name: str # The model's name defined by the creator
file_path: str # Full path to the model file
size: int # File size in bytes
modified: float # Timestamp when the model was added to the management system
sha256: str # SHA256 hash of the file
base_model: str # Base model type (SD1.5/SD2.1/SDXL/etc.)
preview_url: str # Preview image URL
preview_nsfw_level: int = 0 # NSFW level of the preview image
notes: str = "" # Additional notes
from_civitai: bool = True # Whether from Civitai
civitai: Dict[str, Any] = field(
default_factory=dict
) # Civitai API data if available
tags: List[str] = None # Model tags
modelDescription: str = "" # Full model description
civitai_deleted: bool = False # Whether deleted from Civitai
favorite: bool = False # Whether the model is a favorite
exclude: bool = False # Whether to exclude this model from the cache
db_checked: bool = False # Whether checked in archive DB
skip_metadata_refresh: bool = False # Whether to skip this model during bulk metadata refresh
favorite: bool = False # Whether the model is a favorite
exclude: bool = False # Whether to exclude this model from the cache
db_checked: bool = False # Whether checked in archive DB
skip_metadata_refresh: bool = (
False # Whether to skip this model during bulk metadata refresh
)
metadata_source: Optional[str] = None # Last provider that supplied metadata
last_checked_at: float = 0 # Last checked timestamp
hash_status: str = "completed" # Hash calculation status: pending | calculating | completed | failed
_unknown_fields: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False) # Store unknown fields
_unknown_fields: Dict[str, Any] = field(
default_factory=dict, repr=False, compare=False
) # Store unknown fields
def __post_init__(self):
# Initialize empty lists to avoid mutable default parameter issue
@@ -40,211 +48,238 @@ class BaseModelMetadata:
self.tags = []
@classmethod
def from_dict(cls, data: Dict) -> 'BaseModelMetadata':
def from_dict(cls, data: Dict) -> "BaseModelMetadata":
"""Create instance from dictionary"""
data_copy = data.copy()
# Use cached fields if available, otherwise compute them
if not hasattr(cls, '_known_fields_cache'):
if not hasattr(cls, "_known_fields_cache"):
known_fields = set()
for c in cls.mro():
if hasattr(c, '__annotations__'):
if hasattr(c, "__annotations__"):
known_fields.update(c.__annotations__.keys())
cls._known_fields_cache = known_fields
known_fields = cls._known_fields_cache
# Extract fields that match our class attributes
fields_to_use = {k: v for k, v in data_copy.items() if k in known_fields}
# Store unknown fields separately
unknown_fields = {k: v for k, v in data_copy.items() if k not in known_fields and not k.startswith('_')}
unknown_fields = {
k: v
for k, v in data_copy.items()
if k not in known_fields and not k.startswith("_")
}
# Create instance with known fields
instance = cls(**fields_to_use)
# Add unknown fields as a separate attribute
instance._unknown_fields = unknown_fields
return instance
def to_dict(self) -> Dict:
"""Convert to dictionary for JSON serialization"""
result = asdict(self)
# Remove private fields
result = {k: v for k, v in result.items() if not k.startswith('_')}
result = {k: v for k, v in result.items() if not k.startswith("_")}
# Add back unknown fields if they exist
if hasattr(self, '_unknown_fields'):
if hasattr(self, "_unknown_fields"):
result.update(self._unknown_fields)
return result
def update_civitai_info(self, civitai_data: Dict) -> None:
"""Update Civitai information"""
self.civitai = civitai_data
def update_file_info(self, file_path: str) -> None:
"""Update metadata with actual file information"""
def update_file_info(self, file_path: str, update_timestamps: bool = False) -> None:
"""
Update metadata with actual file information.
Args:
file_path: Path to the model file
update_timestamps: If True, update size and modified from filesystem.
If False (default), only update file_path and file_name.
Set to True only when file has been moved/relocated.
"""
if os.path.exists(file_path):
self.size = os.path.getsize(file_path)
self.modified = os.path.getmtime(file_path)
self.file_path = file_path.replace(os.sep, '/')
# Update file_name when file_path changes
if update_timestamps:
# Only update size and modified when file has been relocated
self.size = os.path.getsize(file_path)
self.modified = os.path.getmtime(file_path)
# Always update paths when this method is called
self.file_path = file_path.replace(os.sep, "/")
self.file_name = os.path.splitext(os.path.basename(file_path))[0]
@staticmethod
def generate_unique_filename(target_dir: str, base_name: str, extension: str, hash_provider: callable = None) -> str:
def generate_unique_filename(
target_dir: str, base_name: str, extension: str, hash_provider: callable = None
) -> str:
"""Generate a unique filename to avoid conflicts
Args:
target_dir: Target directory path
base_name: Base filename without extension
extension: File extension including the dot
hash_provider: A callable that returns the SHA256 hash when needed
Returns:
str: Unique filename that doesn't conflict with existing files
"""
original_filename = f"{base_name}{extension}"
target_path = os.path.join(target_dir, original_filename)
# If no conflict, return original filename
if not os.path.exists(target_path):
return original_filename
# Only compute hash when needed
if hash_provider:
sha256_hash = hash_provider()
else:
sha256_hash = "0000"
# Generate short hash (first 4 characters of SHA256)
short_hash = sha256_hash[:4] if sha256_hash else "0000"
# Try with short hash suffix
unique_filename = f"{base_name}-{short_hash}{extension}"
unique_path = os.path.join(target_dir, unique_filename)
# If still conflicts, add incremental number
counter = 1
while os.path.exists(unique_path):
unique_filename = f"{base_name}-{short_hash}-{counter}{extension}"
unique_path = os.path.join(target_dir, unique_filename)
counter += 1
return unique_filename
@dataclass
class LoraMetadata(BaseModelMetadata):
"""Represents the metadata structure for a Lora model"""
usage_tips: str = "{}" # Usage tips for the model, json string
usage_tips: str = "{}" # Usage tips for the model, json string
@classmethod
def from_civitai_info(cls, version_info: Dict, file_info: Dict, save_path: str) -> 'LoraMetadata':
def from_civitai_info(
cls, version_info: Dict, file_info: Dict, save_path: str
) -> "LoraMetadata":
"""Create LoraMetadata instance from Civitai version info"""
file_name = file_info.get('name', '')
base_model = determine_base_model(version_info.get('baseModel', ''))
file_name = file_info.get("name", "")
base_model = determine_base_model(version_info.get("baseModel", ""))
# Extract tags and description if available
tags = []
description = ""
model_data = version_info.get('model') or {}
if 'tags' in model_data:
tags = model_data['tags']
if 'description' in model_data:
description = model_data['description']
model_data = version_info.get("model") or {}
if "tags" in model_data:
tags = model_data["tags"]
if "description" in model_data:
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get('name', os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, '/'),
size=file_info.get('sizeKB', 0) * 1024,
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
sha256=(file_info.get('hashes') or {}).get('SHA256', '').lower(),
sha256=(file_info.get("hashes") or {}).get("SHA256", "").lower(),
base_model=base_model,
preview_url='', # Will be updated after preview download
preview_nsfw_level=0, # Will be updated after preview download
preview_url="", # Will be updated after preview download
preview_nsfw_level=0, # Will be updated after preview download
from_civitai=True,
civitai=version_info,
tags=tags,
modelDescription=description
modelDescription=description,
)
@dataclass
class CheckpointMetadata(BaseModelMetadata):
"""Represents the metadata structure for a Checkpoint model"""
sub_type: str = "checkpoint" # Model sub-type (checkpoint, diffusion_model, etc.)
@classmethod
def from_civitai_info(cls, version_info: Dict, file_info: Dict, save_path: str) -> 'CheckpointMetadata':
def from_civitai_info(
cls, version_info: Dict, file_info: Dict, save_path: str
) -> "CheckpointMetadata":
"""Create CheckpointMetadata instance from Civitai version info"""
file_name = file_info.get('name', '')
base_model = determine_base_model(version_info.get('baseModel', ''))
sub_type = version_info.get('type', 'checkpoint')
file_name = file_info.get("name", "")
base_model = determine_base_model(version_info.get("baseModel", ""))
sub_type = version_info.get("type", "checkpoint")
# Extract tags and description if available
tags = []
description = ""
model_data = version_info.get('model') or {}
if 'tags' in model_data:
tags = model_data['tags']
if 'description' in model_data:
description = model_data['description']
model_data = version_info.get("model") or {}
if "tags" in model_data:
tags = model_data["tags"]
if "description" in model_data:
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get('name', os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, '/'),
size=file_info.get('sizeKB', 0) * 1024,
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
sha256=(file_info.get('hashes') or {}).get('SHA256', '').lower(),
sha256=(file_info.get("hashes") or {}).get("SHA256", "").lower(),
base_model=base_model,
preview_url='', # Will be updated after preview download
preview_url="", # Will be updated after preview download
preview_nsfw_level=0,
from_civitai=True,
civitai=version_info,
sub_type=sub_type,
tags=tags,
modelDescription=description
modelDescription=description,
)
@dataclass
class EmbeddingMetadata(BaseModelMetadata):
"""Represents the metadata structure for an Embedding model"""
sub_type: str = "embedding"
@classmethod
def from_civitai_info(cls, version_info: Dict, file_info: Dict, save_path: str) -> 'EmbeddingMetadata':
def from_civitai_info(
cls, version_info: Dict, file_info: Dict, save_path: str
) -> "EmbeddingMetadata":
"""Create EmbeddingMetadata instance from Civitai version info"""
file_name = file_info.get('name', '')
base_model = determine_base_model(version_info.get('baseModel', ''))
sub_type = version_info.get('type', 'embedding')
file_name = file_info.get("name", "")
base_model = determine_base_model(version_info.get("baseModel", ""))
sub_type = version_info.get("type", "embedding")
# Extract tags and description if available
tags = []
description = ""
model_data = version_info.get('model') or {}
if 'tags' in model_data:
tags = model_data['tags']
if 'description' in model_data:
description = model_data['description']
model_data = version_info.get("model") or {}
if "tags" in model_data:
tags = model_data["tags"]
if "description" in model_data:
description = model_data["description"]
return cls(
file_name=os.path.splitext(file_name)[0],
model_name=model_data.get('name', os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, '/'),
size=file_info.get('sizeKB', 0) * 1024,
model_name=model_data.get("name", os.path.splitext(file_name)[0]),
file_path=save_path.replace(os.sep, "/"),
size=file_info.get("sizeKB", 0) * 1024,
modified=datetime.now().timestamp(),
sha256=(file_info.get('hashes') or {}).get('SHA256', '').lower(),
sha256=(file_info.get("hashes") or {}).get("SHA256", "").lower(),
base_model=base_model,
preview_url='', # Will be updated after preview download
preview_url="", # Will be updated after preview download
preview_nsfw_level=0,
from_civitai=True,
civitai=version_info,
sub_type=sub_type,
tags=tags,
modelDescription=description
modelDescription=description,
)