feat(metadata): enhance model processing with CivitAI metadata checks and new fields for archive DB status

This commit is contained in:
Will Miao
2025-09-19 23:22:47 +08:00
parent f3544b3471
commit 3053b13fcb
4 changed files with 33 additions and 36 deletions

View File

@@ -626,18 +626,16 @@ class BaseModelRoutes(ABC):
# Prepare models to process, only those without CivitAI data
enable_metadata_archive_db = settings.get('enable_metadata_archive_db', False)
# Filter models that need CivitAI metadata update
to_process = [
model for model in cache.raw_data
if (
model.get('sha256')
and (
not model.get('civitai')
or not model['civitai'].get('id')
)
and (
(enable_metadata_archive_db)
or (not enable_metadata_archive_db and model.get('from_civitai') is True)
)
if model.get('sha256')
and (
not model.get('civitai') or not model['civitai'].get('id')
)
and (
(enable_metadata_archive_db and not model.get('db_checked', False))
or (not enable_metadata_archive_db and model.get('from_civitai') is True)
)
]
total_to_process = len(to_process)

View File

@@ -24,6 +24,8 @@ class BaseModelMetadata:
civitai_deleted: bool = False # Whether deleted from Civitai
favorite: bool = False # Whether the model is a favorite
exclude: bool = False # Whether to exclude this model from the cache
db_checked: bool = False # Whether checked in archive DB
last_checked_at: float = 0 # Last checked timestamp
_unknown_fields: Dict[str, Any] = field(default_factory=dict, repr=False, compare=False) # Store unknown fields
def __post_init__(self):

View File

@@ -3,6 +3,7 @@ import json
import logging
from typing import Dict, List, Callable, Awaitable
from aiohttp import web
from datetime import datetime
from .model_utils import determine_base_model
from .constants import PREVIEW_EXTENSIONS, CARD_PREVIEW_WIDTH
@@ -82,7 +83,6 @@ class ModelRouteUtils:
# Update local metadata with merged civitai data
local_metadata['civitai'] = merged_civitai
local_metadata['from_civitai'] = True
# Update model-related metadata from civitai_metadata.model
if 'model' in civitai_metadata and civitai_metadata['model']:
@@ -203,12 +203,11 @@ class ModelRouteUtils:
return False
metadata_path = os.path.splitext(file_path)[0] + '.metadata.json'
# Check if model metadata exists
local_metadata = await ModelRouteUtils.load_local_metadata(metadata_path)
enable_metadata_archive_db = settings.get('enable_metadata_archive_db', False)
if model_data.get('from_civitai') is False:
if not settings.get('enable_metadata_archive_db', False):
if model_data.get('civitai_deleted') is True and model_data.get('db_checked') is False:
# If CivitAI deleted flag is set, skip CivitAI API provider
if not enable_metadata_archive_db:
return False
# Likely deleted from CivitAI, use archive_db if available
metadata_provider = await get_metadata_provider('sqlite')
@@ -217,11 +216,24 @@ class ModelRouteUtils:
civitai_metadata, error = await metadata_provider.get_model_by_hash(sha256)
if not civitai_metadata:
# Mark as not from CivitAI if not found
local_metadata['from_civitai'] = False
model_data['from_civitai'] = False
await MetadataManager.save_metadata(file_path, local_metadata)
if error == "Model not found":
model_data['from_civitai'] = False
model_data['civitai_deleted'] = True
model_data['db_checked'] = enable_metadata_archive_db
model_data['last_checked_at'] = datetime.now().timestamp()
# Remove 'folder' key from model_data if present before saving
data_to_save = model_data.copy()
data_to_save.pop('folder', None)
await MetadataManager.save_metadata(file_path, data_to_save)
return False
model_data['from_civitai'] = True
model_data['civitai_deleted'] = civitai_metadata.get('source') == 'archive_db'
model_data['db_checked'] = enable_metadata_archive_db
local_metadata = model_data.copy()
local_metadata.pop('folder', None) # Remove 'folder' key if present
# Update metadata
await ModelRouteUtils.update_model_metadata(
@@ -235,8 +247,7 @@ class ModelRouteUtils:
update_dict = {
'model_name': local_metadata.get('model_name'),
'preview_url': local_metadata.get('preview_url'),
'from_civitai': True,
'civitai': civitai_metadata
'civitai': local_metadata.get('civitai'),
}
model_data.update(update_dict)

View File

@@ -33,20 +33,6 @@ export async function loadExampleImages(images, modelHash) {
const params = `model_hash=${modelHash}`;
const response = await fetch(`${endpoint}?${params}`);
if (!response.ok) {
// Try to parse error message from backend
let errorMsg = `HTTP error ${response.status}`;
try {
const errorData = await response.json();
if (errorData && errorData.error) {
errorMsg = errorData.error;
}
} catch (e) {
// Ignore JSON parse error
}
console.warn("Failed to get example files:", errorMsg);
return;
}
const result = await response.json();
if (result.success) {