mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-20 18:51:26 -03:00
fix(recipes): reject the empty-hash placeholder when resolving LoRA hashes
The SHA256 of an empty byte string (written by repackaging tools into safetensors metadata, or produced by hashing an empty/unreadable file) was previously resolved against CivitAI's by-hash API, which can contain polluted entries for it (e.g. a broken SD 1.5 LoRA whose AutoV3 equals the placeholder) and falsely attributed the wrong model to a recipe. Guard all lookup paths for the 10/12/64-char AutoV2/AutoV3/full-SHA256 spellings: CivitaiClient.get_model_by_hash/_fetch_version_by_hash return not-found without a request, and ModelHashIndex ignores the placeholder in has_hash/get_path/add_autov3. The Automatic1111 metadata parser keeps the LoRA item itself when its hash is the placeholder: it matches by filename locally, or retains the entry with an empty hash flagged hashInvalid (unresolvable-hash state in the UI, with reconnect as the remedy) instead of dropping it or resolving it to a polluted CivitAI entry.
This commit is contained in:
@@ -8,6 +8,7 @@ from typing import Dict, Any
|
|||||||
from ..base import RecipeMetadataParser
|
from ..base import RecipeMetadataParser
|
||||||
from ..constants import GEN_PARAM_KEYS
|
from ..constants import GEN_PARAM_KEYS
|
||||||
from ...services.metadata_service import get_default_metadata_provider
|
from ...services.metadata_service import get_default_metadata_provider
|
||||||
|
from ...utils.constants import is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -524,6 +525,26 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
||||||
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
||||||
|
|
||||||
|
if is_empty_placeholder_hash(lora_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte
|
||||||
|
# string) is not a real hash: never look it up in the
|
||||||
|
# local hash index or on CivitAI. Match by filename;
|
||||||
|
# otherwise keep the item as unresolved (no hash, flagged
|
||||||
|
# hashInvalid so the UI shows the unresolvable-hash state
|
||||||
|
# and offers reconnect instead of download) rather than
|
||||||
|
# dropping it.
|
||||||
|
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
continue
|
||||||
|
lora_entry['hash'] = ''
|
||||||
|
lora_entry['hashInvalid'] = True
|
||||||
|
if not resource_lora_count:
|
||||||
|
loras.append(lora_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
if lora_hash and recipe_scanner and lora_type == 'lora':
|
if lora_hash and recipe_scanner and lora_type == 'lora':
|
||||||
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
||||||
if local_lora:
|
if local_lora:
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from .model_metadata_provider import (
|
|||||||
from .downloader import get_downloader
|
from .downloader import get_downloader
|
||||||
from .errors import RateLimitError, ResourceNotFoundError
|
from .errors import RateLimitError, ResourceNotFoundError
|
||||||
from ..utils.civitai_utils import resolve_license_payload
|
from ..utils.civitai_utils import resolve_license_payload
|
||||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -180,6 +180,11 @@ class CivitaiClient:
|
|||||||
async def get_model_by_hash(
|
async def get_model_by_hash(
|
||||||
self, model_hash: str
|
self, model_hash: str
|
||||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte string)
|
||||||
|
# matches no real file; CivitAI's by-hash index can contain
|
||||||
|
# polluted entries for it, so never resolve it.
|
||||||
|
return None, "Model not found"
|
||||||
try:
|
try:
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
@@ -503,6 +508,8 @@ class CivitaiClient:
|
|||||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||||
if not model_hash:
|
if not model_hash:
|
||||||
return None
|
return None
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
return None
|
||||||
|
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
from typing import Dict, Optional, Set, List
|
from typing import Dict, Optional, Set, List
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from ..utils.constants import is_empty_placeholder_hash
|
||||||
|
|
||||||
class ModelHashIndex:
|
class ModelHashIndex:
|
||||||
"""Index for looking up models by hash or filename"""
|
"""Index for looking up models by hash or filename"""
|
||||||
|
|
||||||
@@ -81,6 +83,8 @@ class ModelHashIndex:
|
|||||||
# mapping. First-time registrations stay O(1).
|
# mapping. First-time registrations stay O(1).
|
||||||
if autov3:
|
if autov3:
|
||||||
autov3 = autov3.lower()
|
autov3 = autov3.lower()
|
||||||
|
if is_empty_placeholder_hash(autov3):
|
||||||
|
autov3 = None
|
||||||
if is_re_registration and (existing_hash != sha256 or autov3):
|
if is_re_registration and (existing_hash != sha256 or autov3):
|
||||||
stale_autov3_keys = [
|
stale_autov3_keys = [
|
||||||
key for key, mapped_path in self._autov3_to_path.items()
|
key for key, mapped_path in self._autov3_to_path.items()
|
||||||
@@ -93,7 +97,7 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def add_autov3(self, autov3: str, file_path: str) -> None:
|
def add_autov3(self, autov3: str, file_path: str) -> None:
|
||||||
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
||||||
if not autov3:
|
if not autov3 or is_empty_placeholder_hash(autov3):
|
||||||
return
|
return
|
||||||
autov3 = autov3.lower()
|
autov3 = autov3.lower()
|
||||||
self._autov3_to_path[autov3] = file_path
|
self._autov3_to_path[autov3] = file_path
|
||||||
@@ -250,6 +254,8 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def has_hash(self, hash_value: str) -> bool:
|
def has_hash(self, hash_value: str) -> bool:
|
||||||
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
||||||
|
if is_empty_placeholder_hash(hash_value):
|
||||||
|
return False
|
||||||
normalized = hash_value.lower()
|
normalized = hash_value.lower()
|
||||||
if normalized in self._hash_to_path:
|
if normalized in self._hash_to_path:
|
||||||
return True
|
return True
|
||||||
@@ -261,6 +267,8 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def get_path(self, hash_value: str) -> Optional[str]:
|
def get_path(self, hash_value: str) -> Optional[str]:
|
||||||
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
||||||
|
if is_empty_placeholder_hash(hash_value):
|
||||||
|
return None
|
||||||
normalized = hash_value.lower()
|
normalized = hash_value.lower()
|
||||||
path = self._hash_to_path.get(normalized)
|
path = self._hash_to_path.get(normalized)
|
||||||
if path is not None:
|
if path is not None:
|
||||||
|
|||||||
+26
-5
@@ -1,3 +1,5 @@
|
|||||||
|
from typing import Any
|
||||||
|
|
||||||
NSFW_LEVELS = {
|
NSFW_LEVELS = {
|
||||||
"PG": 1,
|
"PG": 1,
|
||||||
"PG13": 2,
|
"PG13": 2,
|
||||||
@@ -99,11 +101,30 @@ DEFAULT_HASH_CHUNK_SIZE_MB = 4
|
|||||||
# absurd 64-bit header length from forcing a multi-GB allocation during scan.
|
# absurd 64-bit header length from forcing a multi-GB allocation during scan.
|
||||||
MAX_SAFETENSORS_HEADER_BYTES = 64 * 1024 * 1024
|
MAX_SAFETENSORS_HEADER_BYTES = 64 * 1024 * 1024
|
||||||
|
|
||||||
# First 12 chars of the SHA256 of an empty byte string. Some (re-packaging)
|
# SHA256 of an empty byte string. Some (re-packaging) training tools write a
|
||||||
# training tools write this placeholder into safetensors metadata instead of a
|
# truncated form of this placeholder into safetensors metadata (as
|
||||||
# real hash; it must never be treated as a valid AutoV3 — several broken
|
# ``modelspec.hash_sha256`` / ``sshs_model_hash``), and hashing an empty or
|
||||||
# models sharing it would collide in the hash index and falsely match recipes.
|
# unreadable file produces it directly. It must never be treated as a valid
|
||||||
INVALID_AUTOV3_EMPTY_HASH = "e3b0c44298fc"
|
# hash: several broken models share it, CivitAI's by-hash index can contain
|
||||||
|
# such polluted entries, and matching it falsely attributes recipes.
|
||||||
|
EMPTY_HASH_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||||
|
INVALID_AUTOV3_EMPTY_HASH = EMPTY_HASH_SHA256[:12]
|
||||||
|
INVALID_AUTOV2_EMPTY_HASH = EMPTY_HASH_SHA256[:10]
|
||||||
|
|
||||||
|
|
||||||
|
def is_empty_placeholder_hash(value: Any) -> bool:
|
||||||
|
"""True for a 10/12/64-hex-char spelling of the empty-hash placeholder.
|
||||||
|
|
||||||
|
These are the AutoV2, AutoV3 and full-SHA256 forms of the placeholder;
|
||||||
|
such values identify no real model and must never be resolved against
|
||||||
|
local files or CivitAI.
|
||||||
|
"""
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return False
|
||||||
|
v = value.strip().lower()
|
||||||
|
if len(v) not in (10, 12, 64):
|
||||||
|
return False
|
||||||
|
return v == EMPTY_HASH_SHA256[: len(v)]
|
||||||
|
|
||||||
# Auto-organize settings
|
# Auto-organize settings
|
||||||
AUTO_ORGANIZE_BATCH_SIZE = (
|
AUTO_ORGANIZE_BATCH_SIZE = (
|
||||||
|
|||||||
@@ -503,3 +503,64 @@ async def test_parse_metadata_extracts_checkpoint_from_model_hash(monkeypatch):
|
|||||||
assert result["model"] == checkpoint
|
assert result["model"] == checkpoint
|
||||||
assert result["base_model"] == "flux"
|
assert result["base_model"] == "flux"
|
||||||
assert result["loras"] == []
|
assert result["loras"] == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_metadata_keeps_empty_placeholder_hash_lora_unresolved(monkeypatch):
|
||||||
|
"""A LoRA hash equal to the SHA256("") placeholder must never be resolved
|
||||||
|
against CivitAI or the local hash index, but the LoRA item itself must be
|
||||||
|
kept: matched by filename locally when present, otherwise kept as an
|
||||||
|
unresolved entry (no hash) instead of being dropped."""
|
||||||
|
queried_hashes = []
|
||||||
|
|
||||||
|
async def fake_metadata_provider():
|
||||||
|
class Provider:
|
||||||
|
async def get_model_by_hash(self, model_hash):
|
||||||
|
queried_hashes.append(model_hash)
|
||||||
|
return None, "Model not found"
|
||||||
|
|
||||||
|
async def get_model_version_info(self, version_id):
|
||||||
|
raise AssertionError("get_model_version_info should not be called")
|
||||||
|
|
||||||
|
return Provider()
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"py.recipes.parsers.automatic.get_default_metadata_provider",
|
||||||
|
fake_metadata_provider,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser = AutomaticMetadataParser()
|
||||||
|
metadata_text = (
|
||||||
|
"photo of a DeLorean DMC12, <lora:dmc12bttf:1.2>, at night\n"
|
||||||
|
"Steps: 20, Sampler: Euler, CFG scale: 1, Seed: 2242760352, Size: 1280x720, "
|
||||||
|
"Model: flux1-dev, Model hash: 3f97fdc57a, "
|
||||||
|
'Lora hashes: "dmc12bttf: e3b0c44298fc"'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Local file with the same name: the item is matched by filename.
|
||||||
|
scanner_with_local = LocalRecipeScanner({"dmc12bttf": local_lora("dmc12bttf")})
|
||||||
|
result = await parser.parse_metadata(metadata_text, recipe_scanner=scanner_with_local)
|
||||||
|
|
||||||
|
assert "e3b0c44298fc" not in queried_hashes
|
||||||
|
assert "e3b0c44298" not in queried_hashes
|
||||||
|
assert scanner_with_local.hash_queries == []
|
||||||
|
assert scanner_with_local.queries == ["dmc12bttf"]
|
||||||
|
assert len(result["loras"]) == 1
|
||||||
|
assert result["loras"][0]["file_name"] == "dmc12bttf"
|
||||||
|
assert result["loras"][0]["weight"] == 1.2
|
||||||
|
assert result["loras"][0]["existsLocally"] is True
|
||||||
|
assert result["loras"][0]["isDeleted"] is False
|
||||||
|
|
||||||
|
# No local file: the item is kept as unresolved (empty hash, flagged
|
||||||
|
# hashInvalid so the UI renders the unresolvable-hash badge).
|
||||||
|
scanner_without_local = LocalRecipeScanner({})
|
||||||
|
result = await parser.parse_metadata(metadata_text, recipe_scanner=scanner_without_local)
|
||||||
|
|
||||||
|
assert len(result["loras"]) == 1
|
||||||
|
lora = result["loras"][0]
|
||||||
|
assert lora["file_name"] == "dmc12bttf"
|
||||||
|
assert lora["weight"] == 1.2
|
||||||
|
assert lora["hash"] == ""
|
||||||
|
assert lora["hashInvalid"] is True
|
||||||
|
assert lora["existsLocally"] is False
|
||||||
|
assert lora["isDeleted"] is False
|
||||||
|
|||||||
@@ -789,3 +789,32 @@ async def test_get_creator_model_count_never_raises(downloader):
|
|||||||
|
|
||||||
client = await CivitaiClient.get_instance()
|
client = await CivitaiClient.get_instance()
|
||||||
assert await client.get_creator_model_count("pixel") is None
|
assert await client.get_creator_model_count("pixel") is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"placeholder_hash",
|
||||||
|
[
|
||||||
|
"e3b0c44298", # AutoV2 (10 chars)
|
||||||
|
"e3b0c44298fc", # AutoV3 (12 chars)
|
||||||
|
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", # full SHA256
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_get_model_by_hash_rejects_empty_placeholder_without_request(downloader, placeholder_hash):
|
||||||
|
"""The empty-hash placeholder must never be resolved via the by-hash API:
|
||||||
|
CivitAI's index can contain polluted entries for it (e.g. a broken SD 1.5
|
||||||
|
LoRA whose AutoV3 equals the placeholder)."""
|
||||||
|
requested = []
|
||||||
|
|
||||||
|
async def fake_make_request(method, url, use_auth=True, **kwargs):
|
||||||
|
requested.append(url)
|
||||||
|
return True, {}
|
||||||
|
|
||||||
|
downloader.make_request = fake_make_request
|
||||||
|
|
||||||
|
client = await CivitaiClient.get_instance()
|
||||||
|
|
||||||
|
result, error = await client.get_model_by_hash(placeholder_hash)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
assert error == "Model not found"
|
||||||
|
assert requested == []
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from py.services.model_hash_index import ModelHashIndex
|
from py.services.model_hash_index import ModelHashIndex
|
||||||
|
from py.utils.constants import EMPTY_HASH_SHA256
|
||||||
|
|
||||||
|
|
||||||
class TestModelHashIndexRemoveByPath:
|
class TestModelHashIndexRemoveByPath:
|
||||||
@@ -253,3 +254,38 @@ class TestModelHashIndexAutov3:
|
|||||||
assert index.has_hash("abcdef123456") is False
|
assert index.has_hash("abcdef123456") is False
|
||||||
assert index.get_path("fedcba654321") == "/models/ckpt.safetensors"
|
assert index.get_path("fedcba654321") == "/models/ckpt.safetensors"
|
||||||
assert index.get_all_autov3() == {"fedcba654321": "/models/ckpt.safetensors"}
|
assert index.get_all_autov3() == {"fedcba654321": "/models/ckpt.safetensors"}
|
||||||
|
|
||||||
|
|
||||||
|
class TestModelHashIndexEmptyPlaceholder:
|
||||||
|
def test_add_autov3_rejects_empty_placeholder(self):
|
||||||
|
index = ModelHashIndex()
|
||||||
|
index.add_autov3("e3b0c44298fc", "/models/lora.safetensors")
|
||||||
|
assert "e3b0c44298fc" not in index.get_all_autov3()
|
||||||
|
|
||||||
|
def test_add_entry_rejects_empty_placeholder_autov3(self):
|
||||||
|
index = ModelHashIndex()
|
||||||
|
index.add_entry("abc123", "/models/lora.safetensors", autov3="e3b0c44298fc")
|
||||||
|
assert "e3b0c44298fc" not in index.get_all_autov3()
|
||||||
|
|
||||||
|
def test_has_hash_false_for_placeholder(self):
|
||||||
|
index = ModelHashIndex()
|
||||||
|
index.add_entry("abc123", "/models/lora.safetensors")
|
||||||
|
assert not index.has_hash("e3b0c44298")
|
||||||
|
assert not index.has_hash("e3b0c44298fc")
|
||||||
|
assert not index.has_hash(EMPTY_HASH_SHA256)
|
||||||
|
|
||||||
|
def test_get_path_none_for_placeholder(self):
|
||||||
|
index = ModelHashIndex()
|
||||||
|
index.add_entry("abc123", "/models/lora.safetensors")
|
||||||
|
assert index.get_path("e3b0c44298") is None
|
||||||
|
assert index.get_path("e3b0c44298fc") is None
|
||||||
|
assert index.get_path(EMPTY_HASH_SHA256) is None
|
||||||
|
|
||||||
|
def test_placeholder_autov3_does_not_clobber_existing_mapping(self):
|
||||||
|
# Registering a path with a placeholder autov3 must not replace or
|
||||||
|
# clear autov3 mappings already registered for other paths.
|
||||||
|
index = ModelHashIndex()
|
||||||
|
index.add_entry("a" * 64, "/models/real.safetensors", autov3="abcdef123456")
|
||||||
|
index.add_entry("b" * 64, "/models/other.safetensors", autov3="e3b0c44298fc")
|
||||||
|
assert index.get_path("abcdef123456") == "/models/real.safetensors"
|
||||||
|
assert index.get_all_autov3() == {"abcdef123456": "/models/real.safetensors"}
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
"""Tests for the empty-hash placeholder predicate in constants."""
|
||||||
|
|
||||||
|
from py.utils.constants import (
|
||||||
|
EMPTY_HASH_SHA256,
|
||||||
|
INVALID_AUTOV2_EMPTY_HASH,
|
||||||
|
INVALID_AUTOV3_EMPTY_HASH,
|
||||||
|
is_empty_placeholder_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsEmptyPlaceholderHash:
|
||||||
|
def test_full_length_sha256(self):
|
||||||
|
assert is_empty_placeholder_hash(EMPTY_HASH_SHA256)
|
||||||
|
|
||||||
|
def test_autov3_length(self):
|
||||||
|
assert is_empty_placeholder_hash("e3b0c44298fc")
|
||||||
|
|
||||||
|
def test_autov2_length(self):
|
||||||
|
assert is_empty_placeholder_hash("e3b0c44298")
|
||||||
|
|
||||||
|
def test_case_insensitive(self):
|
||||||
|
assert is_empty_placeholder_hash("E3B0C44298FC")
|
||||||
|
assert is_empty_placeholder_hash(EMPTY_HASH_SHA256.upper())
|
||||||
|
|
||||||
|
def test_derived_constants_are_prefixes(self):
|
||||||
|
assert INVALID_AUTOV2_EMPTY_HASH == EMPTY_HASH_SHA256[:10]
|
||||||
|
assert INVALID_AUTOV3_EMPTY_HASH == EMPTY_HASH_SHA256[:12]
|
||||||
|
|
||||||
|
def test_rejects_other_lengths(self):
|
||||||
|
# 8-char AutoV1-style prefix and non-placeholder lengths are not it
|
||||||
|
assert not is_empty_placeholder_hash("e3b0c442")
|
||||||
|
assert not is_empty_placeholder_hash("e3b0c44298fc1c")
|
||||||
|
assert not is_empty_placeholder_hash("")
|
||||||
|
|
||||||
|
def test_rejects_real_hashes_that_share_the_prefix(self):
|
||||||
|
# A real hash whose first characters coincide must not be rejected
|
||||||
|
assert not is_empty_placeholder_hash("e3b0c44298aa")
|
||||||
|
assert not is_empty_placeholder_hash("e3b0c44298fc" + "a" * 52)
|
||||||
|
assert not is_empty_placeholder_hash("915a9a1f5f")
|
||||||
|
assert not is_empty_placeholder_hash("915a9a1f5f58")
|
||||||
|
assert not is_empty_placeholder_hash("a" * 64)
|
||||||
|
|
||||||
|
def test_rejects_non_strings(self):
|
||||||
|
assert not is_empty_placeholder_hash(None)
|
||||||
|
assert not is_empty_placeholder_hash(123)
|
||||||
Reference in New Issue
Block a user