mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-06-09 12:39:23 -03:00
perf(recipe): skip CivitAI API calls for locally-known models in create-from-example (#945)
Build a local_cache from the scanner cache before calling the metadata parser. When a resource hash is found in the cache, populate the entry directly from cached civitai metadata instead of calling CivitAI's /model-versions/by-hash endpoint. This eliminates redundant API calls and retries for the common case where the example image only uses the parent model plus a checkpoint.
This commit is contained in:
Will Miao
@@ -6,6 +6,7 @@ from typing import Dict, Any, Union
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -73,7 +74,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
return False
|
||||
|
||||
async def parse_metadata( # type: ignore[override]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||
local_cache: dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Parse metadata from Civitai image format
|
||||
|
||||
@@ -81,6 +83,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
user_comment: The metadata from the image (dict)
|
||||
recipe_scanner: Optional recipe scanner service
|
||||
civitai_client: Optional Civitai API client (deprecated, use metadata_provider instead)
|
||||
local_cache: Optional dict mapping sha256/autov3 hash → scanner cache item.
|
||||
When provided, matching models skip CivitAI API calls.
|
||||
|
||||
Returns:
|
||||
Dict containing parsed recipe data
|
||||
@@ -210,35 +214,45 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Try to look up base model from the checkpoint hash
|
||||
if checkpoint_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(
|
||||
checkpoint_entry["hash"]
|
||||
cp_hash = checkpoint_entry.get("hash")
|
||||
if cp_hash and metadata_provider:
|
||||
local_cached = local_cache.get(cp_hash) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
checkpoint_entry, local_cached
|
||||
)
|
||||
bm = checkpoint_entry.get("baseModel", "")
|
||||
if bm and not result["base_model"]:
|
||||
result["base_model"] = bm
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(
|
||||
cp_hash
|
||||
)
|
||||
)
|
||||
civitai_data, error_msg = (
|
||||
(civitai_info, None)
|
||||
if not isinstance(civitai_info, tuple)
|
||||
else civitai_info
|
||||
)
|
||||
if civitai_data and error_msg != "Model not found":
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint_entry['name'] = civitai_data['model']['name']
|
||||
checkpoint_entry['id'] = civitai_data.get('id', 0)
|
||||
checkpoint_entry['modelId'] = civitai_data.get('modelId', 0)
|
||||
if 'name' in civitai_data:
|
||||
checkpoint_entry['version'] = civitai_data['name']
|
||||
base_model = civitai_data.get('baseModel', '')
|
||||
if base_model:
|
||||
checkpoint_entry['baseModel'] = base_model
|
||||
if not result['base_model']:
|
||||
result['base_model'] = base_model
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching checkpoint info for hash "
|
||||
f"{cp_hash}: {e}"
|
||||
)
|
||||
)
|
||||
civitai_data, error_msg = (
|
||||
(civitai_info, None)
|
||||
if not isinstance(civitai_info, tuple)
|
||||
else civitai_info
|
||||
)
|
||||
if civitai_data and error_msg != "Model not found":
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint_entry['name'] = civitai_data['model']['name']
|
||||
checkpoint_entry['id'] = civitai_data.get('id', 0)
|
||||
checkpoint_entry['modelId'] = civitai_data.get('modelId', 0)
|
||||
if 'name' in civitai_data:
|
||||
checkpoint_entry['version'] = civitai_data['name']
|
||||
base_model = civitai_data.get('baseModel', '')
|
||||
if base_model:
|
||||
checkpoint_entry['baseModel'] = base_model
|
||||
if not result['base_model']:
|
||||
result['base_model'] = base_model
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching checkpoint info for hash "
|
||||
f"{checkpoint_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
if result["model"] is None:
|
||||
result["model"] = checkpoint_entry
|
||||
@@ -279,34 +293,45 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
if lora_hash and metadata_provider:
|
||||
local_cached = local_cache.get(lora_hash) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
lora_entry, local_cached
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
# Track by version ID for deduplication
|
||||
if lora_entry.get("id"):
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -684,3 +709,41 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing Civitai image metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
|
||||
@staticmethod
|
||||
def _populate_entry_from_cache(
|
||||
entry: dict[str, Any],
|
||||
cache_item: dict[str, Any],
|
||||
) -> None:
|
||||
"""Fill a lora/checkpoint entry from a scanner cache item.
|
||||
|
||||
Avoids CivitAI API calls for models that exist locally.
|
||||
Mirrors the population logic in
|
||||
``RecipeMetadataParser.populate_lora_from_civitai()`` but operates
|
||||
entirely on cached data.
|
||||
"""
|
||||
civ = cache_item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
if civ.get("id") is not None:
|
||||
entry["id"] = civ["id"]
|
||||
if civ.get("modelId") is not None:
|
||||
entry["modelId"] = civ["modelId"]
|
||||
if civ.get("name"):
|
||||
entry["version"] = civ["name"]
|
||||
cached_name = cache_item.get("model_name")
|
||||
if cached_name:
|
||||
entry["name"] = cached_name
|
||||
entry["existsLocally"] = True
|
||||
local_path = cache_item.get("file_path")
|
||||
if local_path:
|
||||
entry["localPath"] = local_path
|
||||
sha256 = cache_item.get("sha256")
|
||||
if sha256:
|
||||
entry["hash"] = sha256
|
||||
if "preview_url" in cache_item:
|
||||
entry["thumbnailUrl"] = config.get_preview_static_url(
|
||||
cache_item["preview_url"]
|
||||
)
|
||||
base_model = cache_item.get("base_model", "")
|
||||
if base_model:
|
||||
entry["baseModel"] = base_model
|
||||
|
||||
@@ -1718,49 +1718,75 @@ class RecipeManagementHandler:
|
||||
parsed_input = {**image_data, **inner_meta}
|
||||
parsed_input.pop("meta", None)
|
||||
|
||||
# Build a local cache of {hash → cache_item} so the parser can
|
||||
# skip CivitAI API calls for models that exist on disk.
|
||||
local_cache: Dict[str, Dict[str, Any]] = {}
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner and model_hash:
|
||||
try:
|
||||
parent_cache_data = await lora_scanner.get_cached_data()
|
||||
for item in getattr(parent_cache_data, "raw_data", []):
|
||||
if item.get("sha256", "").lower() == model_hash.lower():
|
||||
local_cache[model_hash.lower()] = item
|
||||
# Compute AutoV3 so the parser can also match on
|
||||
# that hash type (CivitAI metadata resources use
|
||||
# AutoV3).
|
||||
file_path = item.get("file_path")
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
from ...utils.file_utils import (
|
||||
calculate_autov3,
|
||||
)
|
||||
autov3 = calculate_autov3(file_path)
|
||||
if autov3:
|
||||
local_cache[autov3.lower()] = item
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
parser = self._analysis_service._recipe_parser_factory.create_parser(
|
||||
parsed_input
|
||||
)
|
||||
if not parser:
|
||||
raise RecipeValidationError("Unable to parse image metadata")
|
||||
|
||||
parsed = await parser.parse_metadata(
|
||||
parsed_input, recipe_scanner=recipe_scanner
|
||||
)
|
||||
from ...recipes.parsers.civitai_image import CivitaiApiMetadataParser
|
||||
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
parsed = await parser.parse_metadata(
|
||||
parsed_input,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
parsed = await parser.parse_metadata(
|
||||
parsed_input, recipe_scanner=recipe_scanner
|
||||
)
|
||||
|
||||
loras = list(parsed.get("loras") or [])
|
||||
checkpoint = parsed.get("model")
|
||||
is_lora_type = model_type.startswith("lora")
|
||||
is_ckpt_type = model_type.startswith("checkpoint")
|
||||
|
||||
# Look up parent model's cached CivitAI metadata (version ID,
|
||||
# version name, model ID) from the scanner cache. Used to fix
|
||||
# isDeleted entries and enrich auto-populated ones.
|
||||
# Extract parent model metadata from local_cache (used below to
|
||||
# reconcile isDeleted entries and enrich auto-populated ones).
|
||||
parent_civitai_id: int | None = None
|
||||
parent_model_id: int | None = None
|
||||
parent_version_name: str | None = None
|
||||
parent_model_name: str | None = None
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner and model_hash:
|
||||
try:
|
||||
parent_cache = await lora_scanner.get_cached_data()
|
||||
for item in getattr(parent_cache, "raw_data", []):
|
||||
if item.get("sha256", "").lower() == model_hash.lower():
|
||||
civ = item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
parent_civitai_id = civ.get("id")
|
||||
parent_model_id = civ.get("modelId")
|
||||
parent_version_name = civ.get("name")
|
||||
# model_name is a flat SQLite column holding the
|
||||
# CivitAI model display name (not nested under
|
||||
# civitai.model which only stores type).
|
||||
parent_model_name = item.get("model_name")
|
||||
|
||||
break
|
||||
else:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
# Prefer sha256 key; fall back to any cached entry.
|
||||
parent_item = local_cache.get(model_hash.lower()) if model_hash else None
|
||||
if parent_item is None and local_cache:
|
||||
parent_item = next(iter(local_cache.values()))
|
||||
if parent_item:
|
||||
civ = parent_item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
parent_civitai_id = civ.get("id")
|
||||
parent_model_id = civ.get("modelId")
|
||||
parent_version_name = civ.get("name")
|
||||
parent_model_name = parent_item.get("model_name")
|
||||
|
||||
# Reconcile isDeleted entries against the parent model.
|
||||
# When the CivitAI hash lookup fails (known issue — hashes not
|
||||
|
||||
Reference in New Issue
Block a user