feat(metadata): improve model ID redirect logic and provider ordering

- Fix CivArchive model ID redirect logic to only follow redirects when context points to original model
- Rename CivitaiModelMetadataProvider to CivArchiveModelMetadataProvider for consistency
- Reorder fallback metadata providers to prioritize Civitai API over CivArchive API for better metadata quality
- Remove unused asyncio import and redundant logging from metadata sync service
This commit is contained in:
Will Miao
2025-10-11 16:11:13 +08:00
parent c3a66ecf28
commit 1e8bd88e28
5 changed files with 17 additions and 49 deletions

View File

@@ -392,7 +392,15 @@ class CivArchiveClient:
) )
return None return None
actual_model_id = version_data.get("modelId") actual_model_id = version_data.get("modelId")
if actual_model_id is not None and str(actual_model_id) != str(model_id): context_model_id = context.get("id")
# CivArchive can respond with data for a different model id while already
# returning the fully resolved model context. Only follow the redirect when
# the context itself still points to the original (wrong) model.
if (
actual_model_id is not None
and str(actual_model_id) != str(model_id)
and (context_model_id is None or str(context_model_id) != str(actual_model_id))
):
return await self.get_model_version(actual_model_id, version_id) return await self.get_model_version(actual_model_id, version_id)
return self._transform_version(context, version_data, fallback_files) return self._transform_version(context, version_data, fallback_files)

View File

@@ -58,7 +58,7 @@ async def initialize_metadata_providers():
# Register CivArchive provider, and all add to fallback providers # Register CivArchive provider, and all add to fallback providers
try: try:
civarchive_client = await ServiceRegistry.get_civarchive_client() civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivitaiModelMetadataProvider(civarchive_client) civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider) provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider)) providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)") logger.debug("CivArchive metadata provider registered (also included in fallback)")
@@ -67,16 +67,15 @@ async def initialize_metadata_providers():
# Set up fallback provider based on available providers # Set up fallback provider based on available providers
if len(providers) > 1: if len(providers) > 1:
# Always use Civarchive, then Civitai API, then Archive DB # Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB
ordered_providers = [] ordered_providers = []
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api']) ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite']) ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite'])
if ordered_providers: if ordered_providers:
fallback_provider = FallbackMetadataProvider(ordered_providers) fallback_provider = FallbackMetadataProvider(ordered_providers)
provider_manager.register_provider('fallback', fallback_provider, is_default=True) provider_manager.register_provider('fallback', fallback_provider, is_default=True)
logger.info(f"Fallback metadata provider registered with {len(ordered_providers)} providers, Civarchive first")
elif len(providers) == 1: elif len(providers) == 1:
# Only one provider available, set it as default # Only one provider available, set it as default
provider_name, provider = providers[0] provider_name, provider = providers[0]

View File

@@ -5,7 +5,6 @@ from __future__ import annotations
import json import json
import logging import logging
import os import os
import asyncio
from datetime import datetime from datetime import datetime
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
@@ -170,46 +169,6 @@ class MetadataSyncService:
enable_archive = self._settings.get("enable_metadata_archive_db", False) enable_archive = self._settings.get("enable_metadata_archive_db", False)
try: try:
metadata_provider = await self._get_provider("civarchive_api")
tryagain = True
delay = 5
while tryagain:
civitai_metadata, error = await metadata_provider.get_model_by_hash(sha256)
tryagain = False
if not civitai_metadata or error:
if error == "HTTP 429":
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
delay = delay * 2
await asyncio.sleep(delay)
tryagain = True
continue
if error == "Model not found":
model_data["from_civitai"] = False
model_data["civitai_deleted"] = True
#model_data["db_checked"] = enable_archive
model_data["last_checked_at"] = datetime.now().timestamp()
data_to_save = model_data.copy()
data_to_save.pop("folder", None)
await self._metadata_manager.save_metadata(file_path, data_to_save)
await asyncio.sleep(1)
if error == "No version data found":
error_msg = (f"Error - No civitai version found: (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
error = False
if civitai_metadata.get('files'):
for file in civitai_metadata['files']:
logger.error(f"{file}")
if 'tensorart' in file['url'] or "seaart" in file['url']:
civitai_metadata, error = await metadata_provider.get_model_by_hash(file['url'])
error_msg = (f"Error fetching metadata: {error} {civitai_metadata}")
logger.error(error_msg)
if error or not civitai_metadata:
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
logger.error(error_msg)
return False, error_msg
if model_data.get("civitai_deleted") is True: if model_data.get("civitai_deleted") is True:
if not enable_archive or model_data.get("db_checked") is True: if not enable_archive or model_data.get("db_checked") is True:
if not enable_archive: if not enable_archive:
@@ -241,8 +200,8 @@ class MetadataSyncService:
return False, error_msg return False, error_msg
model_data["from_civitai"] = True model_data["from_civitai"] = True
model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" or civitai_metadata.get("source") == "civarchive"
model_data["db_checked"] = enable_archive model_data["db_checked"] = enable_archive and civitai_metadata.get("source") == "archive_db"
model_data["last_checked_at"] = datetime.now().timestamp() model_data["last_checked_at"] = datetime.now().timestamp()
local_metadata = model_data.copy() local_metadata = model_data.copy()

View File

@@ -120,7 +120,7 @@ async def test_get_model_by_hash_transforms_payload(downloader):
file_meta = result["files"][0] file_meta = result["files"][0]
assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3" assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3"
assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567" assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567"
assert file_meta["primary"] is False assert file_meta["primary"] is True
assert result["source"] == "civarchive" assert result["source"] == "civarchive"
assert result["images"][0]["url"] == "https://img.genur.art/example.png" assert result["images"][0]["url"] == "https://img.genur.art/example.png"

View File

@@ -108,6 +108,7 @@ def metadata_provider(monkeypatch):
"creator": {"username": "Author"}, "creator": {"username": "Author"},
"files": [ "files": [
{ {
"type": "Model",
"primary": True, "primary": True,
"downloadUrl": "https://example.invalid/file.safetensors", "downloadUrl": "https://example.invalid/file.safetensors",
"name": "file.safetensors", "name": "file.safetensors",
@@ -206,6 +207,7 @@ async def test_download_uses_active_mirrors(monkeypatch, scanners, metadata_prov
"creator": {"username": "Author"}, "creator": {"username": "Author"},
"files": [ "files": [
{ {
"type": "Model",
"primary": True, "primary": True,
"downloadUrl": "https://example.invalid/file.safetensors", "downloadUrl": "https://example.invalid/file.safetensors",
"mirrors": [ "mirrors": [