diff --git a/py/recipes/parsers/automatic.py b/py/recipes/parsers/automatic.py index 74503261..2ad1236c 100644 --- a/py/recipes/parsers/automatic.py +++ b/py/recipes/parsers/automatic.py @@ -8,6 +8,7 @@ from typing import Dict, Any from ..base import RecipeMetadataParser from ..constants import GEN_PARAM_KEYS from ...services.metadata_service import get_default_metadata_provider +from ...utils.constants import is_empty_placeholder_hash logger = logging.getLogger(__name__) @@ -524,6 +525,26 @@ class AutomaticMetadataParser(RecipeMetadataParser): weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0 lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash) + if is_empty_placeholder_hash(lora_hash): + # The empty-hash placeholder (SHA256 of an empty byte + # string) is not a real hash: never look it up in the + # local hash index or on CivitAI. Match by filename; + # otherwise keep the item as unresolved (no hash, flagged + # hashInvalid so the UI shows the unresolvable-hash state + # and offers reconnect instead of download) rather than + # dropping it. + if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames: + local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model) + if local_lora: + local_entry = self.populate_lora_from_local(lora_entry, local_lora) + merge_or_append_local(local_entry) + continue + lora_entry['hash'] = '' + lora_entry['hashInvalid'] = True + if not resource_lora_count: + loras.append(lora_entry) + continue + if lora_hash and recipe_scanner and lora_type == 'lora': local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash) if local_lora: diff --git a/py/services/civitai_client.py b/py/services/civitai_client.py index 81f2f297..fbcdaa06 100644 --- a/py/services/civitai_client.py +++ b/py/services/civitai_client.py @@ -21,7 +21,7 @@ from .model_metadata_provider import ( from .downloader import get_downloader from .errors import RateLimitError, ResourceNotFoundError from ..utils.civitai_utils import resolve_license_payload -from ..utils.constants import MODEL_WEIGHT_FILE_TYPES +from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash logger = logging.getLogger(__name__) @@ -180,6 +180,11 @@ class CivitaiClient: async def get_model_by_hash( self, model_hash: str ) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + if is_empty_placeholder_hash(model_hash): + # The empty-hash placeholder (SHA256 of an empty byte string) + # matches no real file; CivitAI's by-hash index can contain + # polluted entries for it, so never resolve it. + return None, "Model not found" try: success, version = await self._make_request( "GET", @@ -503,6 +508,8 @@ class CivitaiClient: async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]: if not model_hash: return None + if is_empty_placeholder_hash(model_hash): + return None success, version = await self._make_request( "GET", diff --git a/py/services/model_hash_index.py b/py/services/model_hash_index.py index 103f39ae..e8a6f572 100644 --- a/py/services/model_hash_index.py +++ b/py/services/model_hash_index.py @@ -1,6 +1,8 @@ from typing import Dict, Optional, Set, List import os +from ..utils.constants import is_empty_placeholder_hash + class ModelHashIndex: """Index for looking up models by hash or filename""" @@ -81,6 +83,8 @@ class ModelHashIndex: # mapping. First-time registrations stay O(1). if autov3: autov3 = autov3.lower() + if is_empty_placeholder_hash(autov3): + autov3 = None if is_re_registration and (existing_hash != sha256 or autov3): stale_autov3_keys = [ key for key, mapped_path in self._autov3_to_path.items() @@ -93,7 +97,7 @@ class ModelHashIndex: def add_autov3(self, autov3: str, file_path: str) -> None: """Add or update an AutoV3-only index entry (used when only AutoV3 is known)""" - if not autov3: + if not autov3 or is_empty_placeholder_hash(autov3): return autov3 = autov3.lower() self._autov3_to_path[autov3] = file_path @@ -250,6 +254,8 @@ class ModelHashIndex: def has_hash(self, hash_value: str) -> bool: """Check if hash exists in index (SHA256, AutoV2, or AutoV3)""" + if is_empty_placeholder_hash(hash_value): + return False normalized = hash_value.lower() if normalized in self._hash_to_path: return True @@ -261,6 +267,8 @@ class ModelHashIndex: def get_path(self, hash_value: str) -> Optional[str]: """Get file path for a hash (SHA256, AutoV2, or AutoV3)""" + if is_empty_placeholder_hash(hash_value): + return None normalized = hash_value.lower() path = self._hash_to_path.get(normalized) if path is not None: diff --git a/py/utils/constants.py b/py/utils/constants.py index 974537a2..73325aef 100644 --- a/py/utils/constants.py +++ b/py/utils/constants.py @@ -1,3 +1,5 @@ +from typing import Any + NSFW_LEVELS = { "PG": 1, "PG13": 2, @@ -99,11 +101,30 @@ DEFAULT_HASH_CHUNK_SIZE_MB = 4 # absurd 64-bit header length from forcing a multi-GB allocation during scan. MAX_SAFETENSORS_HEADER_BYTES = 64 * 1024 * 1024 -# First 12 chars of the SHA256 of an empty byte string. Some (re-packaging) -# training tools write this placeholder into safetensors metadata instead of a -# real hash; it must never be treated as a valid AutoV3 — several broken -# models sharing it would collide in the hash index and falsely match recipes. -INVALID_AUTOV3_EMPTY_HASH = "e3b0c44298fc" +# SHA256 of an empty byte string. Some (re-packaging) training tools write a +# truncated form of this placeholder into safetensors metadata (as +# ``modelspec.hash_sha256`` / ``sshs_model_hash``), and hashing an empty or +# unreadable file produces it directly. It must never be treated as a valid +# hash: several broken models share it, CivitAI's by-hash index can contain +# such polluted entries, and matching it falsely attributes recipes. +EMPTY_HASH_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +INVALID_AUTOV3_EMPTY_HASH = EMPTY_HASH_SHA256[:12] +INVALID_AUTOV2_EMPTY_HASH = EMPTY_HASH_SHA256[:10] + + +def is_empty_placeholder_hash(value: Any) -> bool: + """True for a 10/12/64-hex-char spelling of the empty-hash placeholder. + + These are the AutoV2, AutoV3 and full-SHA256 forms of the placeholder; + such values identify no real model and must never be resolved against + local files or CivitAI. + """ + if not isinstance(value, str): + return False + v = value.strip().lower() + if len(v) not in (10, 12, 64): + return False + return v == EMPTY_HASH_SHA256[: len(v)] # Auto-organize settings AUTO_ORGANIZE_BATCH_SIZE = ( diff --git a/tests/services/test_automatic_metadata_parser.py b/tests/services/test_automatic_metadata_parser.py index a337985b..9bc3f7c1 100644 --- a/tests/services/test_automatic_metadata_parser.py +++ b/tests/services/test_automatic_metadata_parser.py @@ -503,3 +503,64 @@ async def test_parse_metadata_extracts_checkpoint_from_model_hash(monkeypatch): assert result["model"] == checkpoint assert result["base_model"] == "flux" assert result["loras"] == [] + + +@pytest.mark.asyncio +async def test_parse_metadata_keeps_empty_placeholder_hash_lora_unresolved(monkeypatch): + """A LoRA hash equal to the SHA256("") placeholder must never be resolved + against CivitAI or the local hash index, but the LoRA item itself must be + kept: matched by filename locally when present, otherwise kept as an + unresolved entry (no hash) instead of being dropped.""" + queried_hashes = [] + + async def fake_metadata_provider(): + class Provider: + async def get_model_by_hash(self, model_hash): + queried_hashes.append(model_hash) + return None, "Model not found" + + async def get_model_version_info(self, version_id): + raise AssertionError("get_model_version_info should not be called") + + return Provider() + + monkeypatch.setattr( + "py.recipes.parsers.automatic.get_default_metadata_provider", + fake_metadata_provider, + ) + + parser = AutomaticMetadataParser() + metadata_text = ( + "photo of a DeLorean DMC12, , at night\n" + "Steps: 20, Sampler: Euler, CFG scale: 1, Seed: 2242760352, Size: 1280x720, " + "Model: flux1-dev, Model hash: 3f97fdc57a, " + 'Lora hashes: "dmc12bttf: e3b0c44298fc"' + ) + + # Local file with the same name: the item is matched by filename. + scanner_with_local = LocalRecipeScanner({"dmc12bttf": local_lora("dmc12bttf")}) + result = await parser.parse_metadata(metadata_text, recipe_scanner=scanner_with_local) + + assert "e3b0c44298fc" not in queried_hashes + assert "e3b0c44298" not in queried_hashes + assert scanner_with_local.hash_queries == [] + assert scanner_with_local.queries == ["dmc12bttf"] + assert len(result["loras"]) == 1 + assert result["loras"][0]["file_name"] == "dmc12bttf" + assert result["loras"][0]["weight"] == 1.2 + assert result["loras"][0]["existsLocally"] is True + assert result["loras"][0]["isDeleted"] is False + + # No local file: the item is kept as unresolved (empty hash, flagged + # hashInvalid so the UI renders the unresolvable-hash badge). + scanner_without_local = LocalRecipeScanner({}) + result = await parser.parse_metadata(metadata_text, recipe_scanner=scanner_without_local) + + assert len(result["loras"]) == 1 + lora = result["loras"][0] + assert lora["file_name"] == "dmc12bttf" + assert lora["weight"] == 1.2 + assert lora["hash"] == "" + assert lora["hashInvalid"] is True + assert lora["existsLocally"] is False + assert lora["isDeleted"] is False diff --git a/tests/services/test_civitai_client.py b/tests/services/test_civitai_client.py index 63f2ab93..b07cabdd 100644 --- a/tests/services/test_civitai_client.py +++ b/tests/services/test_civitai_client.py @@ -789,3 +789,32 @@ async def test_get_creator_model_count_never_raises(downloader): client = await CivitaiClient.get_instance() assert await client.get_creator_model_count("pixel") is None + + +@pytest.mark.parametrize( + "placeholder_hash", + [ + "e3b0c44298", # AutoV2 (10 chars) + "e3b0c44298fc", # AutoV3 (12 chars) + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", # full SHA256 + ], +) +async def test_get_model_by_hash_rejects_empty_placeholder_without_request(downloader, placeholder_hash): + """The empty-hash placeholder must never be resolved via the by-hash API: + CivitAI's index can contain polluted entries for it (e.g. a broken SD 1.5 + LoRA whose AutoV3 equals the placeholder).""" + requested = [] + + async def fake_make_request(method, url, use_auth=True, **kwargs): + requested.append(url) + return True, {} + + downloader.make_request = fake_make_request + + client = await CivitaiClient.get_instance() + + result, error = await client.get_model_by_hash(placeholder_hash) + + assert result is None + assert error == "Model not found" + assert requested == [] diff --git a/tests/services/test_model_hash_index.py b/tests/services/test_model_hash_index.py index fdb592e9..4667da3c 100644 --- a/tests/services/test_model_hash_index.py +++ b/tests/services/test_model_hash_index.py @@ -1,5 +1,6 @@ import pytest from py.services.model_hash_index import ModelHashIndex +from py.utils.constants import EMPTY_HASH_SHA256 class TestModelHashIndexRemoveByPath: @@ -253,3 +254,38 @@ class TestModelHashIndexAutov3: assert index.has_hash("abcdef123456") is False assert index.get_path("fedcba654321") == "/models/ckpt.safetensors" assert index.get_all_autov3() == {"fedcba654321": "/models/ckpt.safetensors"} + + +class TestModelHashIndexEmptyPlaceholder: + def test_add_autov3_rejects_empty_placeholder(self): + index = ModelHashIndex() + index.add_autov3("e3b0c44298fc", "/models/lora.safetensors") + assert "e3b0c44298fc" not in index.get_all_autov3() + + def test_add_entry_rejects_empty_placeholder_autov3(self): + index = ModelHashIndex() + index.add_entry("abc123", "/models/lora.safetensors", autov3="e3b0c44298fc") + assert "e3b0c44298fc" not in index.get_all_autov3() + + def test_has_hash_false_for_placeholder(self): + index = ModelHashIndex() + index.add_entry("abc123", "/models/lora.safetensors") + assert not index.has_hash("e3b0c44298") + assert not index.has_hash("e3b0c44298fc") + assert not index.has_hash(EMPTY_HASH_SHA256) + + def test_get_path_none_for_placeholder(self): + index = ModelHashIndex() + index.add_entry("abc123", "/models/lora.safetensors") + assert index.get_path("e3b0c44298") is None + assert index.get_path("e3b0c44298fc") is None + assert index.get_path(EMPTY_HASH_SHA256) is None + + def test_placeholder_autov3_does_not_clobber_existing_mapping(self): + # Registering a path with a placeholder autov3 must not replace or + # clear autov3 mappings already registered for other paths. + index = ModelHashIndex() + index.add_entry("a" * 64, "/models/real.safetensors", autov3="abcdef123456") + index.add_entry("b" * 64, "/models/other.safetensors", autov3="e3b0c44298fc") + assert index.get_path("abcdef123456") == "/models/real.safetensors" + assert index.get_all_autov3() == {"abcdef123456": "/models/real.safetensors"} diff --git a/tests/utils/test_constants.py b/tests/utils/test_constants.py new file mode 100644 index 00000000..d6616451 --- /dev/null +++ b/tests/utils/test_constants.py @@ -0,0 +1,45 @@ +"""Tests for the empty-hash placeholder predicate in constants.""" + +from py.utils.constants import ( + EMPTY_HASH_SHA256, + INVALID_AUTOV2_EMPTY_HASH, + INVALID_AUTOV3_EMPTY_HASH, + is_empty_placeholder_hash, +) + + +class TestIsEmptyPlaceholderHash: + def test_full_length_sha256(self): + assert is_empty_placeholder_hash(EMPTY_HASH_SHA256) + + def test_autov3_length(self): + assert is_empty_placeholder_hash("e3b0c44298fc") + + def test_autov2_length(self): + assert is_empty_placeholder_hash("e3b0c44298") + + def test_case_insensitive(self): + assert is_empty_placeholder_hash("E3B0C44298FC") + assert is_empty_placeholder_hash(EMPTY_HASH_SHA256.upper()) + + def test_derived_constants_are_prefixes(self): + assert INVALID_AUTOV2_EMPTY_HASH == EMPTY_HASH_SHA256[:10] + assert INVALID_AUTOV3_EMPTY_HASH == EMPTY_HASH_SHA256[:12] + + def test_rejects_other_lengths(self): + # 8-char AutoV1-style prefix and non-placeholder lengths are not it + assert not is_empty_placeholder_hash("e3b0c442") + assert not is_empty_placeholder_hash("e3b0c44298fc1c") + assert not is_empty_placeholder_hash("") + + def test_rejects_real_hashes_that_share_the_prefix(self): + # A real hash whose first characters coincide must not be rejected + assert not is_empty_placeholder_hash("e3b0c44298aa") + assert not is_empty_placeholder_hash("e3b0c44298fc" + "a" * 52) + assert not is_empty_placeholder_hash("915a9a1f5f") + assert not is_empty_placeholder_hash("915a9a1f5f58") + assert not is_empty_placeholder_hash("a" * 64) + + def test_rejects_non_strings(self): + assert not is_empty_placeholder_hash(None) + assert not is_empty_placeholder_hash(123) \ No newline at end of file