feat(metadata): add CivitAI AutoV3 hash support across all storage layers

- Three-state autov3 field (not-checked / checked-unavailable / 12-hex value)
  in .metadata.json sidecars, in-memory ModelHashIndex, and SQLite
  (models.autov3 column + autov3_index table) with column-presence migration
- Background self-terminating backfill for legacy rows: per-model-type
  concurrency guard, executor-offloaded I/O, Civitai-first resolution
  (SHA256-matched version file) falling back to the embedded safetensors
  header hash
- Civitai-first propagation on metadata refresh, scan, and download paths;
  reject the empty-string SHA256 placeholder and strip OneTrainer 0x prefix
- List API hash filters and hash index lookups accept 12-char AutoV3
- Cap safetensors header reads at 64 MiB to prevent crafted-file allocation
- Prevent stale AutoV3 mappings on file replacement while preserving them on
  same-file re-registration (lazy-hash completion)
This commit is contained in:
Will Miao
2026-08-08 14:30:34 +08:00
parent 4bf9a4b640
commit 97b9b1f62b
23 changed files with 1918 additions and 50 deletions

View File

@@ -112,6 +112,68 @@ async def test_update_model_metadata_merges_and_persists():
)
@pytest.mark.asyncio
async def test_update_model_metadata_propagates_civitai_autov3():
helpers = build_service()
local = {
"sha256": "111aabbf94dd9e59c05d842fccf57bec915b2a3c237f6b54f8d614e40858d717",
"autov3": "",
"model_name": "Local",
}
remote = {
"source": "api",
"model": {"name": "Remote Model", "description": "", "tags": []},
"images": [],
"files": [
{
"name": "other.safetensors",
"hashes": {"SHA256": "ZZZ999"},
},
{
"name": "model.safetensors",
"hashes": {
"SHA256": "111aabbf94dd9e59c05d842fccf57bec915b2a3c237f6b54f8d614e40858d717",
"AutoV3": "8A582E901D7F",
},
},
],
}
result = await helpers.service.update_model_metadata(
"path/to/model.metadata.json",
local,
remote,
helpers.default_provider,
)
# Civitai-first: the '' (checked-unavailable) state is upgraded in-session
# by the freshly fetched metadata, without any header re-read.
assert result["autov3"] == "8a582e901d7f"
@pytest.mark.asyncio
async def test_update_model_metadata_keeps_autov3_without_matching_file():
helpers = build_service()
local = {"sha256": "abc123", "autov3": "", "model_name": "Local"}
remote = {
"source": "api",
"model": {"name": "Remote Model", "description": "", "tags": []},
"images": [],
"files": [{"name": "other.safetensors", "hashes": {"SHA256": "ZZZ999", "AutoV3": "ABCDEF123456"}}],
}
result = await helpers.service.update_model_metadata(
"path/to/model.metadata.json",
local,
remote,
helpers.default_provider,
)
assert result["autov3"] == ""
@pytest.mark.asyncio
async def test_fetch_and_update_model_success_updates_cache(tmp_path):
helpers = build_service()