From 007883b7d1c84121acef3936fba72104ddd91fe6 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sat, 8 Aug 2026 21:51:34 +0800 Subject: [PATCH] fix(recipes): backfill lora cache item by autov2/autov3 hash too --- py/recipes/base.py | 14 +- tests/services/test_civitai_image_parser.py | 162 ++++++++++++++++++++ 2 files changed, 173 insertions(+), 3 deletions(-) diff --git a/py/recipes/base.py b/py/recipes/base.py index 5bff6628..14f2cc62 100644 --- a/py/recipes/base.py +++ b/py/recipes/base.py @@ -179,10 +179,18 @@ class RecipeMetadataParser(ABC): lora_entry['localPath'] = local_path lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0] - # Get thumbnail from local preview if available + # Get thumbnail from local preview if available. + # Match the cache item by local path first (get_path_by_hash + # cascade: 10-char autov2 / 12-char autov3), then by hash. lora_cache = await lora_scanner.get_cached_data() - lora_item = next((item for item in lora_cache.raw_data - if item['sha256'].lower() == lora_entry['hash'].lower()), None) + h = (lora_entry.get("hash") or "").lower() + lora_item = next((item for item in lora_cache.raw_data + if (item.get("file_path") or "") == local_path), None) + if lora_item is None: + lora_item = next((item for item in lora_cache.raw_data + if (item.get("sha256") or "").lower() == h + or (item.get("autov3") or "").lower() == h + or (item.get("sha256") or "")[:10].lower() == h), None) if lora_item and 'preview_url' in lora_item: lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url']) except Exception as e: diff --git a/tests/services/test_civitai_image_parser.py b/tests/services/test_civitai_image_parser.py index a4b8498a..629c10cb 100644 --- a/tests/services/test_civitai_image_parser.py +++ b/tests/services/test_civitai_image_parser.py @@ -1,5 +1,7 @@ import pytest +from py.config import config +from py.recipes.base import RecipeMetadataParser from py.recipes.parsers.civitai_image import CivitaiApiMetadataParser @@ -408,3 +410,163 @@ async def test_parse_metadata_resources_model_type_does_not_duplicate_checkpoint # Checkpoint must be in result["model"] assert result["model"] is not None assert result["model"]["name"] == "My Checkpoint" + + +def _make_lora_civitai_info(hash_value): + """Build a minimal Civitai response for populate_lora_from_civitai. + + The files entry carries no SHA256 so the entry hash comes from the + hash_value fallback, letting each test control the hash form (autov3, + autov2 prefix, or full sha256) directly. + """ + return { + "id": 300, + "modelId": 400, + "model": {"name": "Style LoRA", "type": "LORA"}, + "name": "v1", + "images": [{"url": "https://image.civitai.com/lora/original=true"}], + "baseModel": "SDXL", + "downloadUrl": "https://civitai.com/api/download/300", + "files": [ + { + "type": "Model", + "primary": True, + "sizeKB": 512, + "name": "style.safetensors", + "hashes": {}, + } + ], + } + + +class _FakeCache: + def __init__(self, entries): + self.raw_data = entries + + +class _FakeLoraScanner: + def __init__(self, cache, local_path): + self._cache = cache + self._local_path = local_path + + def has_hash(self, sha256): + return True + + def get_path_by_hash(self, sha256): + return self._local_path + + async def get_cached_data(self): + return self._cache + + +class _FakeRecipeScanner: + def __init__(self, lora_scanner): + self._lora_scanner = lora_scanner + + +async def _run_backfill(cached_items, hash_value, local_path="/loras/style.safetensors"): + """Drive populate_lora_from_civitai through the local-exists backfill block.""" + lora_scanner = _FakeLoraScanner(_FakeCache(cached_items), local_path) + lora_entry = {"file_name": "style"} + return await RecipeMetadataParser.populate_lora_from_civitai( + lora_entry, + _make_lora_civitai_info(hash_value), + recipe_scanner=_FakeRecipeScanner(lora_scanner), + hash_value=hash_value, + ) + + +@pytest.mark.asyncio +async def test_backfill_lora_item_by_file_path(): + # The primary resolution: the cache item's file_path equals the local + # path resolved by get_path_by_hash, so it is found without hashing. + autov3_hash = "a1b2c3d4e5f6" + cached_item = { + "file_path": "/loras/style.safetensors", + "file_name": "Style LoRA", + "preview_url": "/previews/style.png", + } + result = await _run_backfill([cached_item], autov3_hash) + assert result is not None + assert result["existsLocally"] is True + assert result["localPath"] == "/loras/style.safetensors" + assert result["thumbnailUrl"] == config.get_preview_static_url( + cached_item["preview_url"] + ) + + +@pytest.mark.asyncio +async def test_backfill_lora_item_by_autov3_hash(): + # 12-char autov3 hash that does NOT match the cache item's sha256, + # but matches its stored autov3 — would fail with the sha256-only lookup. + autov3_hash = "a1b2c3d4e5f6" + cached_item = { + "file_path": "/loras/style_stored.safetensors", + "file_name": "Style LoRA", + "sha256": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "autov3": autov3_hash, + "preview_url": "/previews/style.png", + } + result = await _run_backfill([cached_item], autov3_hash) + assert result is not None + assert result["existsLocally"] is True + assert result["localPath"] == "/loras/style.safetensors" + assert result["thumbnailUrl"] == config.get_preview_static_url( + cached_item["preview_url"] + ) + + +@pytest.mark.asyncio +async def test_backfill_lora_item_by_autov2_prefix(): + # 10-char autov2 hash matches the cache item's sha256 prefix. + autov2_hash = "0123456789" + cached_item = { + "file_path": "/loras/style_stored.safetensors", + "file_name": "Style LoRA", + "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "autov3": "", + "preview_url": "/previews/style.png", + } + result = await _run_backfill([cached_item], autov2_hash) + assert result is not None + assert result["existsLocally"] is True + assert result["thumbnailUrl"] == config.get_preview_static_url( + cached_item["preview_url"] + ) + + +@pytest.mark.asyncio +async def test_backfill_lora_item_by_full_sha256(): + # Full sha256 hash matches the cache item as before the change. + sha256_hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + cached_item = { + "file_path": "/loras/style_stored.safetensors", + "file_name": "Style LoRA", + "sha256": sha256_hash, + "preview_url": "/previews/style.png", + } + result = await _run_backfill([cached_item], sha256_hash) + assert result is not None + assert result["existsLocally"] is True + assert result["thumbnailUrl"] == config.get_preview_static_url( + cached_item["preview_url"] + ) + + +@pytest.mark.asyncio +async def test_backfill_lora_cache_item_without_sha256_does_not_crash(): + # Cache item missing the sha256 field: no KeyError, no match, no crash. + autov3_hash = "a1b2c3d4e5f6" + cached_item = { + "file_path": "/loras/unrelated.safetensors", + "file_name": "Unrelated", + } + result = await _run_backfill([cached_item], autov3_hash) + assert result is not None + assert result["existsLocally"] is True + # No match, so thumbnailUrl stays the CivitAI image URL. + assert result["thumbnailUrl"] != config.get_preview_static_url( + cached_item.get("preview_url", "/previews/unrelated.png") + ) + assert result["thumbnailUrl"].startswith("https://image.civitai.com/") +