mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-03-25 15:15:44 -03:00
feat(metadata): improve model ID redirect logic and provider ordering
- Fix CivArchive model ID redirect logic to only follow redirects when context points to original model - Rename CivitaiModelMetadataProvider to CivArchiveModelMetadataProvider for consistency - Reorder fallback metadata providers to prioritize Civitai API over CivArchive API for better metadata quality - Remove unused asyncio import and redundant logging from metadata sync service
This commit is contained in:
@@ -392,7 +392,15 @@ class CivArchiveClient:
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
actual_model_id = version_data.get("modelId")
|
actual_model_id = version_data.get("modelId")
|
||||||
if actual_model_id is not None and str(actual_model_id) != str(model_id):
|
context_model_id = context.get("id")
|
||||||
|
# CivArchive can respond with data for a different model id while already
|
||||||
|
# returning the fully resolved model context. Only follow the redirect when
|
||||||
|
# the context itself still points to the original (wrong) model.
|
||||||
|
if (
|
||||||
|
actual_model_id is not None
|
||||||
|
and str(actual_model_id) != str(model_id)
|
||||||
|
and (context_model_id is None or str(context_model_id) != str(actual_model_id))
|
||||||
|
):
|
||||||
return await self.get_model_version(actual_model_id, version_id)
|
return await self.get_model_version(actual_model_id, version_id)
|
||||||
|
|
||||||
return self._transform_version(context, version_data, fallback_files)
|
return self._transform_version(context, version_data, fallback_files)
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ async def initialize_metadata_providers():
|
|||||||
# Register CivArchive provider, and all add to fallback providers
|
# Register CivArchive provider, and all add to fallback providers
|
||||||
try:
|
try:
|
||||||
civarchive_client = await ServiceRegistry.get_civarchive_client()
|
civarchive_client = await ServiceRegistry.get_civarchive_client()
|
||||||
civarchive_provider = CivitaiModelMetadataProvider(civarchive_client)
|
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
|
||||||
provider_manager.register_provider('civarchive_api', civarchive_provider)
|
provider_manager.register_provider('civarchive_api', civarchive_provider)
|
||||||
providers.append(('civarchive_api', civarchive_provider))
|
providers.append(('civarchive_api', civarchive_provider))
|
||||||
logger.debug("CivArchive metadata provider registered (also included in fallback)")
|
logger.debug("CivArchive metadata provider registered (also included in fallback)")
|
||||||
@@ -67,16 +67,15 @@ async def initialize_metadata_providers():
|
|||||||
|
|
||||||
# Set up fallback provider based on available providers
|
# Set up fallback provider based on available providers
|
||||||
if len(providers) > 1:
|
if len(providers) > 1:
|
||||||
# Always use Civarchive, then Civitai API, then Archive DB
|
# Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB
|
||||||
ordered_providers = []
|
ordered_providers = []
|
||||||
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
|
|
||||||
ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api'])
|
ordered_providers.extend([p[1] for p in providers if p[0] == 'civitai_api'])
|
||||||
|
ordered_providers.extend([p[1] for p in providers if p[0] == 'civarchive_api'])
|
||||||
ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite'])
|
ordered_providers.extend([p[1] for p in providers if p[0] == 'sqlite'])
|
||||||
|
|
||||||
if ordered_providers:
|
if ordered_providers:
|
||||||
fallback_provider = FallbackMetadataProvider(ordered_providers)
|
fallback_provider = FallbackMetadataProvider(ordered_providers)
|
||||||
provider_manager.register_provider('fallback', fallback_provider, is_default=True)
|
provider_manager.register_provider('fallback', fallback_provider, is_default=True)
|
||||||
logger.info(f"Fallback metadata provider registered with {len(ordered_providers)} providers, Civarchive first")
|
|
||||||
elif len(providers) == 1:
|
elif len(providers) == 1:
|
||||||
# Only one provider available, set it as default
|
# Only one provider available, set it as default
|
||||||
provider_name, provider = providers[0]
|
provider_name, provider = providers[0]
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import asyncio
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
||||||
|
|
||||||
@@ -170,46 +169,6 @@ class MetadataSyncService:
|
|||||||
enable_archive = self._settings.get("enable_metadata_archive_db", False)
|
enable_archive = self._settings.get("enable_metadata_archive_db", False)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
metadata_provider = await self._get_provider("civarchive_api")
|
|
||||||
tryagain = True
|
|
||||||
delay = 5
|
|
||||||
|
|
||||||
while tryagain:
|
|
||||||
civitai_metadata, error = await metadata_provider.get_model_by_hash(sha256)
|
|
||||||
tryagain = False
|
|
||||||
if not civitai_metadata or error:
|
|
||||||
if error == "HTTP 429":
|
|
||||||
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
|
|
||||||
logger.error(error_msg)
|
|
||||||
delay = delay * 2
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
tryagain = True
|
|
||||||
continue
|
|
||||||
if error == "Model not found":
|
|
||||||
model_data["from_civitai"] = False
|
|
||||||
model_data["civitai_deleted"] = True
|
|
||||||
#model_data["db_checked"] = enable_archive
|
|
||||||
model_data["last_checked_at"] = datetime.now().timestamp()
|
|
||||||
data_to_save = model_data.copy()
|
|
||||||
data_to_save.pop("folder", None)
|
|
||||||
await self._metadata_manager.save_metadata(file_path, data_to_save)
|
|
||||||
await asyncio.sleep(1)
|
|
||||||
if error == "No version data found":
|
|
||||||
error_msg = (f"Error - No civitai version found: (model_name={model_data.get('model_name', '')} sha256={sha256})")
|
|
||||||
logger.error(error_msg)
|
|
||||||
error = False
|
|
||||||
if civitai_metadata.get('files'):
|
|
||||||
for file in civitai_metadata['files']:
|
|
||||||
logger.error(f"{file}")
|
|
||||||
if 'tensorart' in file['url'] or "seaart" in file['url']:
|
|
||||||
civitai_metadata, error = await metadata_provider.get_model_by_hash(file['url'])
|
|
||||||
error_msg = (f"Error fetching metadata: {error} {civitai_metadata}")
|
|
||||||
logger.error(error_msg)
|
|
||||||
if error or not civitai_metadata:
|
|
||||||
error_msg = (f"Error fetching metadata: {error} (model_name={model_data.get('model_name', '')} sha256={sha256})")
|
|
||||||
logger.error(error_msg)
|
|
||||||
return False, error_msg
|
|
||||||
|
|
||||||
if model_data.get("civitai_deleted") is True:
|
if model_data.get("civitai_deleted") is True:
|
||||||
if not enable_archive or model_data.get("db_checked") is True:
|
if not enable_archive or model_data.get("db_checked") is True:
|
||||||
if not enable_archive:
|
if not enable_archive:
|
||||||
@@ -241,8 +200,8 @@ class MetadataSyncService:
|
|||||||
return False, error_msg
|
return False, error_msg
|
||||||
|
|
||||||
model_data["from_civitai"] = True
|
model_data["from_civitai"] = True
|
||||||
model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db"
|
model_data["civitai_deleted"] = civitai_metadata.get("source") == "archive_db" or civitai_metadata.get("source") == "civarchive"
|
||||||
model_data["db_checked"] = enable_archive
|
model_data["db_checked"] = enable_archive and civitai_metadata.get("source") == "archive_db"
|
||||||
model_data["last_checked_at"] = datetime.now().timestamp()
|
model_data["last_checked_at"] = datetime.now().timestamp()
|
||||||
|
|
||||||
local_metadata = model_data.copy()
|
local_metadata = model_data.copy()
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ async def test_get_model_by_hash_transforms_payload(downloader):
|
|||||||
file_meta = result["files"][0]
|
file_meta = result["files"][0]
|
||||||
assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3"
|
assert file_meta["hashes"]["SHA256"] == "E2B7A280D6539556F23F380B3F71E4E22BC4524445C4C96526E117C6005C6AD3"
|
||||||
assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567"
|
assert file_meta["mirrors"][0]["url"] == "https://civitai.com/api/download/models/1976567"
|
||||||
assert file_meta["primary"] is False
|
assert file_meta["primary"] is True
|
||||||
assert result["source"] == "civarchive"
|
assert result["source"] == "civarchive"
|
||||||
assert result["images"][0]["url"] == "https://img.genur.art/example.png"
|
assert result["images"][0]["url"] == "https://img.genur.art/example.png"
|
||||||
|
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ def metadata_provider(monkeypatch):
|
|||||||
"creator": {"username": "Author"},
|
"creator": {"username": "Author"},
|
||||||
"files": [
|
"files": [
|
||||||
{
|
{
|
||||||
|
"type": "Model",
|
||||||
"primary": True,
|
"primary": True,
|
||||||
"downloadUrl": "https://example.invalid/file.safetensors",
|
"downloadUrl": "https://example.invalid/file.safetensors",
|
||||||
"name": "file.safetensors",
|
"name": "file.safetensors",
|
||||||
@@ -206,6 +207,7 @@ async def test_download_uses_active_mirrors(monkeypatch, scanners, metadata_prov
|
|||||||
"creator": {"username": "Author"},
|
"creator": {"username": "Author"},
|
||||||
"files": [
|
"files": [
|
||||||
{
|
{
|
||||||
|
"type": "Model",
|
||||||
"primary": True,
|
"primary": True,
|
||||||
"downloadUrl": "https://example.invalid/file.safetensors",
|
"downloadUrl": "https://example.invalid/file.safetensors",
|
||||||
"mirrors": [
|
"mirrors": [
|
||||||
|
|||||||
Reference in New Issue
Block a user