diff --git a/py/services/civarchive_client.py b/py/services/civarchive_client.py index 60813d6f..06b0310d 100644 --- a/py/services/civarchive_client.py +++ b/py/services/civarchive_client.py @@ -392,7 +392,15 @@ class CivArchiveClient: ) return None actual_model_id = version_data.get("modelId") - if actual_model_id is not None and str(actual_model_id) != str(model_id): + context_model_id = context.get("id") + # CivArchive can respond with data for a different model id while already + # returning the fully resolved model context. Only follow the redirect when + # the context itself still points to the original (wrong) model. + if ( + actual_model_id is not None + and str(actual_model_id) != str(model_id) + and (context_model_id is None or str(context_model_id) != str(actual_model_id)) + ): return await self.get_model_version(actual_model_id, version_id) return self._transform_version(context, version_data, fallback_files) diff --git a/py/services/metadata_service.py b/py/services/metadata_service.py index 5cfd716d..730e45b0 100644 --- a/py/services/metadata_service.py +++ b/py/services/metadata_service.py @@ -58,7 +58,7 @@ async def initialize_metadata_providers(): # Register CivArchive provider, and all add to fallback providers try: civarchive_client = await ServiceRegistry.get_civarchive_client() - civarchive_provider = CivitaiModelMetadataProvider(civarchive_client) + civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client) provider_manager.register_provider('civarchive_api', civarchive_provider) providers.append(('civarchive_api', civarchive_provider)) logger.debug("CivArchive metadata provider registered (also included in fallback)") @@ -67,16 +67,15 @@ async def initialize_metadata_providers(): # Set up fallback provider based on available providers if len(providers) > 1: - # Always use Civarchive, then Civitai API, then Archive DB + # Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB ordered_providers = [] - ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api']) ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api']) + ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api']) ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite']) if ordered_providers: fallback_provider = FallbackMetadataProvider(ordered_providers) provider_manager.register_provider('fallback', fallback_provider, is_default=True) - logger.info(f"Fallback metadata provider registered with {len(ordered_providers)} providers, Civarchive first") elif len(providers) == 1: # Only one provider available, set it as default provider_name, provider = providers[0] diff --git a/py/services/metadata_sync_service.py b/py/services/metadata_sync_service.py index def0a010..0a228079 100644 --- a/py/services/metadata_sync_service.py +++ b/py/services/metadata_sync_service.py @@ -5,7 +5,6 @@ from __future__ import annotations import json import logging import os -import asyncio from datetime import datetime from typing import Any, Awaitable, Callable, Dict, Iterable, Optional @@ -170,46 +169,6 @@ class MetadataSyncService: enable_archive = self._settings.get("enable_metadata_archive_db", False) try: - metadata_provider = await self._get_provider("civarchive_api") - tryagain = True - delay = 5 - - while tryagain: - civitai_metadata, error = await metadata_provider.get_model_by_hash(sha256) - tryagain = False - if not civitai_metadata or error: - if error == "HTTP 429": - error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})") - logger.error(error_msg) - delay = delay * 2 - await asyncio.sleep(delay) - tryagain = True - continue - if error == "Model not found": - model_data["from_civitai"] = False - model_data["civitai_deleted"] = True - #model_data["db_checked"] = enable_archive - model_data["last_checked_at"] = datetime.now().timestamp() - data_to_save = model_data.copy() - data_to_save.pop("folder", None) - await self._metadata_manager.save_metadata(file_path, data_to_save) - await asyncio.sleep(1) - if error == "No version data found": - error_msg = (f"Error - No civitai version found: (model_name={model_data.get('model_name', '')} sha256={sha256})") - logger.error(error_msg) - error = False - if civitai_metadata.get('files'): - for file in civitai_metadata['files']: - logger.error(f"{file}") - if 'tensorart' in file['url'] or "seaart" in file['url']: - civitai_metadata, error = await metadata_provider.get_model_by_hash(file['url']) - error_msg = (f"Error fetching metadata: {error} {civitai_metadata}") - logger.error(error_msg) - if error or not civitai_metadata: - error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})") - logger.error(error_msg) - return False, error_msg - if model_data.get("civitai_deleted") is True: if not enable_archive or model_data.get("db_checked") is True: if not enable_archive: @@ -241,8 +200,8 @@ class MetadataSyncService: return False, error_msg model_data["from_civitai"] = True - model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" - model_data["db_checked"] = enable_archive + model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" or civitai_metadata.get("source") == "civarchive" + model_data["db_checked"] = enable_archive and civitai_metadata.get("source") == "archive_db" model_data["last_checked_at"] = datetime.now().timestamp() local_metadata = model_data.copy() diff --git a/tests/services/test_civarchive_client.py b/tests/services/test_civarchive_client.py index fe4fa647..6c62f878 100644 --- a/tests/services/test_civarchive_client.py +++ b/tests/services/test_civarchive_client.py @@ -120,7 +120,7 @@ async def test_get_model_by_hash_transforms_payload(downloader): file_meta = result["files"][0] assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3" assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567" - assert file_meta["primary"] is False + assert file_meta["primary"] is True assert result["source"] == "civarchive" assert result["images"][0]["url"] == "https://img.genur.art/example.png" diff --git a/tests/services/test_download_manager.py b/tests/services/test_download_manager.py index 48b425af..fad366f0 100644 --- a/tests/services/test_download_manager.py +++ b/tests/services/test_download_manager.py @@ -108,6 +108,7 @@ def metadata_provider(monkeypatch): "creator": {"username": "Author"}, "files": [ { + "type": "Model", "primary": True, "downloadUrl": "https://example.invalid/file.safetensors", "name": "file.safetensors", @@ -206,6 +207,7 @@ async def test_download_uses_active_mirrors(monkeypatch, scanners, metadata_prov "creator": {"username": "Author"}, "files": [ { + "type": "Model", "primary": True, "downloadUrl": "https://example.invalid/file.safetensors", "mirrors": [