feat(metadata): improve model ID redirect logic and provider ordering

- Fix CivArchive model ID redirect logic to only follow redirects when context points to original model
- Rename CivitaiModelMetadataProvider to CivArchiveModelMetadataProvider for consistency
- Reorder fallback metadata providers to prioritize Civitai API over CivArchive API for better metadata quality
- Remove unused asyncio import and redundant logging from metadata sync service
This commit is contained in:
Will Miao
2025-10-11 16:11:13 +08:00
parent c3a66ecf28
commit 1e8bd88e28
5 changed files with 17 additions and 49 deletions

View File

@@ -392,7 +392,15 @@ class CivArchiveClient:
)
return None
actual_model_id = version_data.get("modelId")
if actual_model_id is not None and str(actual_model_id) != str(model_id):
context_model_id = context.get("id")
# CivArchive can respond with data for a different model id while already
# returning the fully resolved model context. Only follow the redirect when
# the context itself still points to the original (wrong) model.
if (
actual_model_id is not None
and str(actual_model_id) != str(model_id)
and (context_model_id is None or str(context_model_id) != str(actual_model_id))
):
return await self.get_model_version(actual_model_id, version_id)
return self._transform_version(context, version_data, fallback_files)

View File

@@ -58,7 +58,7 @@ async def initialize_metadata_providers():
# Register CivArchive provider, and all add to fallback providers
try:
civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivitaiModelMetadataProvider(civarchive_client)
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)")
@@ -67,16 +67,15 @@ async def initialize_metadata_providers():
# Set up fallback provider based on available providers
if len(providers) > 1:
# Always use Civarchive, then Civitai API, then Archive DB
# Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB
ordered_providers = []
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite'])
if ordered_providers:
fallback_provider = FallbackMetadataProvider(ordered_providers)
provider_manager.register_provider('fallback', fallback_provider, is_default=True)
logger.info(f"Fallback metadata provider registered with {len(ordered_providers)} providers, Civarchive first")
elif len(providers) == 1:
# Only one provider available, set it as default
provider_name, provider = providers[0]

View File

@@ -5,7 +5,6 @@ from __future__ import annotations
import json
import logging
import os
import asyncio
from datetime import datetime
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
@@ -170,46 +169,6 @@ class MetadataSyncService:
enable_archive = self._settings.get("enable_metadata_archive_db", False)
try:
metadata_provider = await self._get_provider("civarchive_api")
tryagain = True
delay = 5
while tryagain:
civitai_metadata, error = await metadata_provider.get_model_by_hash(sha256)
tryagain = False
if not civitai_metadata or error:
if error == "HTTP 429":
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
delay = delay * 2
await asyncio.sleep(delay)
tryagain = True
continue
if error == "Model not found":
model_data["from_civitai"] = False
model_data["civitai_deleted"] = True
#model_data["db_checked"] = enable_archive
model_data["last_checked_at"] = datetime.now().timestamp()
data_to_save = model_data.copy()
data_to_save.pop("folder", None)
await self._metadata_manager.save_metadata(file_path, data_to_save)
await asyncio.sleep(1)
if error == "No version data found":
error_msg = (f"Error - No civitai version found: (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
error = False
if civitai_metadata.get('files'):
for file in civitai_metadata['files']:
logger.error(f"{file}")
if 'tensorart' in file['url'] or "seaart" in file['url']:
civitai_metadata, error = await metadata_provider.get_model_by_hash(file['url'])
error_msg = (f"Error fetching metadata: {error} {civitai_metadata}")
logger.error(error_msg)
if error or not civitai_metadata:
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
return False, error_msg
if model_data.get("civitai_deleted") is True:
if not enable_archive or model_data.get("db_checked") is True:
if not enable_archive:
@@ -241,8 +200,8 @@ class MetadataSyncService:
return False, error_msg
model_data["from_civitai"] = True
model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db"
model_data["db_checked"] = enable_archive
model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" or civitai_metadata.get("source") == "civarchive"
model_data["db_checked"] = enable_archive and civitai_metadata.get("source") == "archive_db"
model_data["last_checked_at"] = datetime.now().timestamp()
local_metadata = model_data.copy()

View File

@@ -120,7 +120,7 @@ async def test_get_model_by_hash_transforms_payload(downloader):
file_meta = result["files"][0]
assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3"
assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567"
assert file_meta["primary"] is False
assert file_meta["primary"] is True
assert result["source"] == "civarchive"
assert result["images"][0]["url"] == "https://img.genur.art/example.png"

View File

@@ -108,6 +108,7 @@ def metadata_provider(monkeypatch):
"creator": {"username": "Author"},
"files": [
{
"type": "Model",
"primary": True,
"downloadUrl": "https://example.invalid/file.safetensors",
"name": "file.safetensors",
@@ -206,6 +207,7 @@ async def test_download_uses_active_mirrors(monkeypatch, scanners, metadata_prov
"creator": {"username": "Author"},
"files": [
{
"type": "Model",
"primary": True,
"downloadUrl": "https://example.invalid/file.safetensors",
"mirrors": [