From fb443ed6ae064fae461c9d922e3ca01fd3963c0b Mon Sep 17 00:00:00 2001 From: Will Miao Date: Wed, 3 Jun 2026 18:10:48 +0800 Subject: [PATCH] perf(recipe): skip CivitAI API calls for locally-known models in create-from-example (#945) Build a local_cache from the scanner cache before calling the metadata parser. When a resource hash is found in the cache, populate the entry directly from cached civitai metadata instead of calling CivitAI's /model-versions/by-hash endpoint. This eliminates redundant API calls and retries for the common case where the example image only uses the parent model plus a checkpoint. --- py/recipes/parsers/civitai_image.py | 169 ++++++++++++++++++-------- py/routes/handlers/recipe_handlers.py | 80 ++++++++---- 2 files changed, 169 insertions(+), 80 deletions(-) diff --git a/py/recipes/parsers/civitai_image.py b/py/recipes/parsers/civitai_image.py index c419e39b..331568a3 100644 --- a/py/recipes/parsers/civitai_image.py +++ b/py/recipes/parsers/civitai_image.py @@ -6,6 +6,7 @@ from typing import Dict, Any, Union from ..base import RecipeMetadataParser from ..constants import GEN_PARAM_KEYS from ...services.metadata_service import get_default_metadata_provider +from ...config import config logger = logging.getLogger(__name__) @@ -73,7 +74,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser): return False async def parse_metadata( # type: ignore[override] - self, user_comment, recipe_scanner=None, civitai_client=None + self, user_comment, recipe_scanner=None, civitai_client=None, + local_cache: dict[str, Any] | None = None, ) -> Dict[str, Any]: """Parse metadata from Civitai image format @@ -81,6 +83,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser): user_comment: The metadata from the image (dict) recipe_scanner: Optional recipe scanner service civitai_client: Optional Civitai API client (deprecated, use metadata_provider instead) + local_cache: Optional dict mapping sha256/autov3 hash → scanner cache item. + When provided, matching models skip CivitAI API calls. Returns: Dict containing parsed recipe data @@ -210,35 +214,45 @@ class CivitaiApiMetadataParser(RecipeMetadataParser): } # Try to look up base model from the checkpoint hash - if checkpoint_entry["hash"] and metadata_provider: - try: - civitai_info = ( - await metadata_provider.get_model_by_hash( - checkpoint_entry["hash"] + cp_hash = checkpoint_entry.get("hash") + if cp_hash and metadata_provider: + local_cached = local_cache.get(cp_hash) if local_cache else None + if local_cached: + self._populate_entry_from_cache( + checkpoint_entry, local_cached + ) + bm = checkpoint_entry.get("baseModel", "") + if bm and not result["base_model"]: + result["base_model"] = bm + else: + try: + civitai_info = ( + await metadata_provider.get_model_by_hash( + cp_hash + ) + ) + civitai_data, error_msg = ( + (civitai_info, None) + if not isinstance(civitai_info, tuple) + else civitai_info + ) + if civitai_data and error_msg != "Model not found": + if 'model' in civitai_data and 'name' in civitai_data['model']: + checkpoint_entry['name'] = civitai_data['model']['name'] + checkpoint_entry['id'] = civitai_data.get('id', 0) + checkpoint_entry['modelId'] = civitai_data.get('modelId', 0) + if 'name' in civitai_data: + checkpoint_entry['version'] = civitai_data['name'] + base_model = civitai_data.get('baseModel', '') + if base_model: + checkpoint_entry['baseModel'] = base_model + if not result['base_model']: + result['base_model'] = base_model + except Exception as e: + logger.error( + f"Error fetching checkpoint info for hash " + f"{cp_hash}: {e}" ) - ) - civitai_data, error_msg = ( - (civitai_info, None) - if not isinstance(civitai_info, tuple) - else civitai_info - ) - if civitai_data and error_msg != "Model not found": - if 'model' in civitai_data and 'name' in civitai_data['model']: - checkpoint_entry['name'] = civitai_data['model']['name'] - checkpoint_entry['id'] = civitai_data.get('id', 0) - checkpoint_entry['modelId'] = civitai_data.get('modelId', 0) - if 'name' in civitai_data: - checkpoint_entry['version'] = civitai_data['name'] - base_model = civitai_data.get('baseModel', '') - if base_model: - checkpoint_entry['baseModel'] = base_model - if not result['base_model']: - result['base_model'] = base_model - except Exception as e: - logger.error( - f"Error fetching checkpoint info for hash " - f"{checkpoint_entry['hash']}: {e}" - ) if result["model"] is None: result["model"] = checkpoint_entry @@ -279,34 +293,45 @@ class CivitaiApiMetadataParser(RecipeMetadataParser): } # Try to get info from Civitai if hash is available - if lora_entry["hash"] and metadata_provider: - try: - civitai_info = ( - await metadata_provider.get_model_by_hash(lora_hash) + if lora_hash and metadata_provider: + local_cached = local_cache.get(lora_hash) if local_cache else None + if local_cached: + self._populate_entry_from_cache( + lora_entry, local_cached ) - - populated_entry = await self.populate_lora_from_civitai( - lora_entry, - civitai_info, - recipe_scanner, - base_model_counts, - lora_hash, - ) - - if populated_entry is None: - continue # Skip invalid LoRA types - - lora_entry = populated_entry - - # If we have a version ID from Civitai, track it for deduplication - if "id" in lora_entry and lora_entry["id"]: + # Track by version ID for deduplication + if lora_entry.get("id"): added_loras[str(lora_entry["id"])] = len( result["loras"] ) - except Exception as e: - logger.error( - f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}" - ) + else: + try: + civitai_info = ( + await metadata_provider.get_model_by_hash(lora_hash) + ) + + populated_entry = await self.populate_lora_from_civitai( + lora_entry, + civitai_info, + recipe_scanner, + base_model_counts, + lora_hash, + ) + + if populated_entry is None: + continue # Skip invalid LoRA types + + lora_entry = populated_entry + + # If we have a version ID from Civitai, track it for deduplication + if "id" in lora_entry and lora_entry["id"]: + added_loras[str(lora_entry["id"])] = len( + result["loras"] + ) + except Exception as e: + logger.error( + f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}" + ) # Track by hash if we have it if lora_hash: @@ -684,3 +709,41 @@ class CivitaiApiMetadataParser(RecipeMetadataParser): except Exception as e: logger.error(f"Error parsing Civitai image metadata: {e}", exc_info=True) return {"error": str(e), "loras": []} + + @staticmethod + def _populate_entry_from_cache( + entry: dict[str, Any], + cache_item: dict[str, Any], + ) -> None: + """Fill a lora/checkpoint entry from a scanner cache item. + + Avoids CivitAI API calls for models that exist locally. + Mirrors the population logic in + ``RecipeMetadataParser.populate_lora_from_civitai()`` but operates + entirely on cached data. + """ + civ = cache_item.get("civitai") or {} + if isinstance(civ, dict): + if civ.get("id") is not None: + entry["id"] = civ["id"] + if civ.get("modelId") is not None: + entry["modelId"] = civ["modelId"] + if civ.get("name"): + entry["version"] = civ["name"] + cached_name = cache_item.get("model_name") + if cached_name: + entry["name"] = cached_name + entry["existsLocally"] = True + local_path = cache_item.get("file_path") + if local_path: + entry["localPath"] = local_path + sha256 = cache_item.get("sha256") + if sha256: + entry["hash"] = sha256 + if "preview_url" in cache_item: + entry["thumbnailUrl"] = config.get_preview_static_url( + cache_item["preview_url"] + ) + base_model = cache_item.get("base_model", "") + if base_model: + entry["baseModel"] = base_model diff --git a/py/routes/handlers/recipe_handlers.py b/py/routes/handlers/recipe_handlers.py index 33d8aebe..e9402b97 100644 --- a/py/routes/handlers/recipe_handlers.py +++ b/py/routes/handlers/recipe_handlers.py @@ -1718,49 +1718,75 @@ class RecipeManagementHandler: parsed_input = {**image_data, **inner_meta} parsed_input.pop("meta", None) + # Build a local cache of {hash → cache_item} so the parser can + # skip CivitAI API calls for models that exist on disk. + local_cache: Dict[str, Dict[str, Any]] = {} + lora_scanner = getattr(recipe_scanner, "_lora_scanner", None) + if lora_scanner and model_hash: + try: + parent_cache_data = await lora_scanner.get_cached_data() + for item in getattr(parent_cache_data, "raw_data", []): + if item.get("sha256", "").lower() == model_hash.lower(): + local_cache[model_hash.lower()] = item + # Compute AutoV3 so the parser can also match on + # that hash type (CivitAI metadata resources use + # AutoV3). + file_path = item.get("file_path") + if file_path and os.path.exists(file_path): + try: + from ...utils.file_utils import ( + calculate_autov3, + ) + autov3 = calculate_autov3(file_path) + if autov3: + local_cache[autov3.lower()] = item + except Exception: + pass + break + except Exception: + pass + parser = self._analysis_service._recipe_parser_factory.create_parser( parsed_input ) if not parser: raise RecipeValidationError("Unable to parse image metadata") - parsed = await parser.parse_metadata( - parsed_input, recipe_scanner=recipe_scanner - ) + from ...recipes.parsers.civitai_image import CivitaiApiMetadataParser + + if isinstance(parser, CivitaiApiMetadataParser): + parsed = await parser.parse_metadata( + parsed_input, + recipe_scanner=recipe_scanner, + local_cache=local_cache, + ) + else: + parsed = await parser.parse_metadata( + parsed_input, recipe_scanner=recipe_scanner + ) loras = list(parsed.get("loras") or []) checkpoint = parsed.get("model") is_lora_type = model_type.startswith("lora") is_ckpt_type = model_type.startswith("checkpoint") - # Look up parent model's cached CivitAI metadata (version ID, - # version name, model ID) from the scanner cache. Used to fix - # isDeleted entries and enrich auto-populated ones. + # Extract parent model metadata from local_cache (used below to + # reconcile isDeleted entries and enrich auto-populated ones). parent_civitai_id: int | None = None parent_model_id: int | None = None parent_version_name: str | None = None parent_model_name: str | None = None - lora_scanner = getattr(recipe_scanner, "_lora_scanner", None) - if lora_scanner and model_hash: - try: - parent_cache = await lora_scanner.get_cached_data() - for item in getattr(parent_cache, "raw_data", []): - if item.get("sha256", "").lower() == model_hash.lower(): - civ = item.get("civitai") or {} - if isinstance(civ, dict): - parent_civitai_id = civ.get("id") - parent_model_id = civ.get("modelId") - parent_version_name = civ.get("name") - # model_name is a flat SQLite column holding the - # CivitAI model display name (not nested under - # civitai.model which only stores type). - parent_model_name = item.get("model_name") - - break - else: - pass - except Exception: - pass + # Prefer sha256 key; fall back to any cached entry. + parent_item = local_cache.get(model_hash.lower()) if model_hash else None + if parent_item is None and local_cache: + parent_item = next(iter(local_cache.values())) + if parent_item: + civ = parent_item.get("civitai") or {} + if isinstance(civ, dict): + parent_civitai_id = civ.get("id") + parent_model_id = civ.get("modelId") + parent_version_name = civ.get("name") + parent_model_name = parent_item.get("model_name") # Reconcile isDeleted entries against the parent model. # When the CivitAI hash lookup fails (known issue — hashes not