feat(metadata): resolve AutoV3 at download time without waiting for backfill

- Read AutoV3 directly from the downloaded file's own file_info hashes
  (no SHA256 cross-matching against version_info.files, so the value is
  captured even when the API omits SHA256)
- Extract normalize_autov3() validation helper shared with the
  sha256-matching autov3_from_civitai_files path
- Fall back to the embedded safetensors header hash at download
  completion; mark '' (checked-unavailable) so the startup backfill
  query (autov3 IS NULL) never revisits the row
- Clear archive-level AutoV3 for zip-extracted models so per-file
  header resolution applies to every extracted model
This commit is contained in:
Will Miao
2026-08-08 15:20:43 +08:00
parent 97b9b1f62b
commit 6fcdeb799d
6 changed files with 197 additions and 18 deletions

View File

@@ -18,7 +18,7 @@ from ..utils.constants import (
VALID_LORA_TYPES,
)
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
from ..utils.file_utils import calculate_sha256
from ..utils.file_utils import calculate_sha256, calculate_autov3
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
from ..utils.utils import sanitize_folder_name
from ..utils.exif_utils import ExifUtils
@@ -2160,6 +2160,10 @@ class DownloadManager:
"error": f"Zip archive does not contain any supported model files ({supported_text})",
}
actual_file_paths = extracted_paths
# The archive entry's AutoV3 (if any) describes the zip itself,
# not the extracted models; clear it so per-file header
# resolution applies to every extracted model.
metadata.autov3 = None
try:
os.remove(save_path)
except OSError as exc:
@@ -2374,6 +2378,16 @@ class DownloadManager:
sha256 = await calculate_sha256(file_path)
if sha256:
entry.sha256 = sha256.lower()
# AutoV3: the Civitai-reported value for the downloaded file (set
# by from_civitai_info) takes precedence. Only the un-checked
# state (None) triggers a header read; '' (checked-unavailable)
# is never re-read, honoring the three-state contract so rows
# marked at download time stay untouched by later passes.
if entry.autov3 is None:
autov3 = await asyncio.get_running_loop().run_in_executor(
None, calculate_autov3, file_path
)
entry.autov3 = (autov3 or "").lower()
entries.append(entry)
return entries

View File

@@ -6,15 +6,29 @@ from .constants import INVALID_AUTOV3_EMPTY_HASH
from .model_utils import determine_base_model
def normalize_autov3(value: Any) -> Optional[str]:
"""Normalize a raw Civitai AutoV3 value to the canonical 12-char form.
Returns the first 12 characters, lowercased, when ``value`` is a string
of at least 12 characters. The empty-string SHA256 placeholder
(``e3b0c44298fc``) is rejected — it is a repackaging-tool artifact, not a
real hash.
Returns ``None`` when the value is unusable.
"""
if isinstance(value, str) and len(value) >= 12:
candidate = value[:12].lower()
if candidate != INVALID_AUTOV3_EMPTY_HASH:
return candidate
return None
def autov3_from_civitai_files(civitai_data: Optional[Dict], sha256: str) -> Optional[str]:
"""Extract the AutoV3 hash from Civitai metadata for the matching file.
Civitai versions can ship multiple files; the AutoV3 hash is only valid
for the file whose ``hashes.SHA256`` equals the local model's sha256.
Matching is case-insensitive. The value is the first 12 characters of
Civitai's AutoV3 hash, lowercased. The empty-string SHA256 placeholder
(``e3b0c44298fc``) is rejected — it is a repackaging-tool artifact, not a
real hash.
Matching is case-insensitive.
Returns ``None`` when no Civitai data, no matching file, or no usable
AutoV3 hash is available.
@@ -28,11 +42,7 @@ def autov3_from_civitai_files(civitai_data: Optional[Dict], sha256: str) -> Opti
hashes = file_info.get("hashes") or {}
file_sha = (hashes.get("SHA256") or "").lower()
if file_sha and file_sha == target_sha:
auto_v3 = hashes.get("AutoV3")
if isinstance(auto_v3, str) and len(auto_v3) >= 12:
candidate = auto_v3[:12].lower()
if candidate != INVALID_AUTOV3_EMPTY_HASH:
return candidate
return normalize_autov3(hashes.get("AutoV3"))
return None
@@ -264,7 +274,8 @@ class LoraMetadata(BaseModelMetadata):
civitai=version_info,
tags=tags,
modelDescription=description,
autov3=autov3_from_civitai_files(version_info, sha256_value),
# Direct read: the downloaded file IS file_info, no SHA256 matching.
autov3=normalize_autov3((file_info.get("hashes") or {}).get("AutoV3")),
)
@@ -308,7 +319,8 @@ class CheckpointMetadata(BaseModelMetadata):
sub_type=sub_type,
tags=tags,
modelDescription=description,
autov3=autov3_from_civitai_files(version_info, sha256_value),
# Direct read: the downloaded file IS file_info, no SHA256 matching.
autov3=normalize_autov3((file_info.get("hashes") or {}).get("AutoV3")),
)
@@ -352,5 +364,6 @@ class EmbeddingMetadata(BaseModelMetadata):
sub_type=sub_type,
tags=tags,
modelDescription=description,
autov3=autov3_from_civitai_files(version_info, sha256_value),
# Direct read: the downloaded file IS file_info, no SHA256 matching.
autov3=normalize_autov3((file_info.get("hashes") or {}).get("AutoV3")),
)