feat(metadata): add OpenModelDB metadata provider and model source for upscalers

Add OpenModelDB (openmodeldb.info) as a metadata and download source for
the existing upscaler model type.

Metadata:
- New OpenModelDBClient: fetches the site's bulk JSON dumps, caches them
  on disk (24h TTL + ETag revalidation), and builds a local sha256 index
- New OpenModelDBModelMetadataProvider adapts catalogue entries to the
  CivitAI-shaped version dict contract; registered in the fallback chain
  behind the enable_openmodeldb_api setting (default on), gated to the
  upscaler sub-type so other model types never trigger the dump download
- Persisted provenance uses metadata_source "openmodeldb" plus a nested
  openmodeldb block (page URL, architecture, scale, license)

Images: paired-image LR/SR URLs are ephemeral imgdiff.net sessions, so
displayable images come from the site-hosted auto-generated thumbnails
(model-level cover leads images[], per-image thumbs for the rest); the
original comparison URL is kept in meta.comparisonUrl.

Downloads:
- New OpenModelDBSource (flat model ids, omdb: group prefix) with
  resource filename derivation that recovers names hidden mid-path
  (mediafire) or synthesizes {id}.{type} for folder links
- HTML-gateway mirrors (mediafire/mega/drive) are rejected with a clear
  manual-download hint instead of silently saving an HTML page as .pth
- ModelSource base gains is_valid_source_id / default_subdir_parts /
  resolve_download_url hooks so flat-id sources need no platform branches

UI: "View on OpenModelDB" link in the model modal (downloaded and
hash-enriched models), settings toggle next to the CivArchive one.
This commit is contained in:
Will Miao
2026-10-03 21:13:07 +08:00
parent 515469054c
commit 35f1ced41a
39 changed files with 2787 additions and 37 deletions
+29 -3
View File
@@ -10,6 +10,7 @@ from .model_metadata_provider import (
SQLiteModelMetadataProvider,
CivitaiModelMetadataProvider,
CivArchiveModelMetadataProvider,
OpenModelDBModelMetadataProvider,
FallbackMetadataProvider,
RateLimitRetryingProvider,
)
@@ -22,12 +23,18 @@ logger = logging.getLogger(__name__)
_PROVIDER_DISPLAY_NAMES = {
"civitai_api": "CivitAI",
"civarchive_api": "CivArchive",
"openmodeldb_api": "OpenModelDB",
"sqlite": "Archive DB",
}
# Preset fallback chains. civitai_api is always first (richest metadata).
# openmodeldb_api sits right after it: its lookups are local index hits over a
# cached bulk dump (no rate-limit budget spent), and it covers upscalers that
# CivArchive only has when they once existed on CivitAI. Providers that are not
# registered (disabled/unavailable) are skipped, so presets degrade gracefully.
_PRESET_PROVIDER_ORDERS = {
"civitai_archive_sqlite": ["civitai_api", "civarchive_api", "sqlite"],
"civitai_sqlite_archive": ["civitai_api", "sqlite", "civarchive_api"],
"civitai_archive_sqlite": ["civitai_api", "openmodeldb_api", "civarchive_api", "sqlite"],
"civitai_sqlite_archive": ["civitai_api", "openmodeldb_api", "sqlite", "civarchive_api"],
}
async def initialize_metadata_providers():
@@ -42,6 +49,7 @@ async def initialize_metadata_providers():
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
enable_openmodeldb_api = settings_manager.get('enable_openmodeldb_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
providers = []
@@ -92,6 +100,22 @@ async def initialize_metadata_providers():
else:
logger.debug("CivArchive metadata provider disabled by setting 'enable_civarchive_api'")
# Register the OpenModelDB provider when enabled. It only covers upscaler
# models (hash-matched against its catalogue dump), so it complements
# rather than replaces the CivitAI-family providers; disabling it avoids
# the one-time bulk dump download entirely.
if enable_openmodeldb_api:
try:
openmodeldb_client = await ServiceRegistry.get_openmodeldb_client()
openmodeldb_provider = OpenModelDBModelMetadataProvider(openmodeldb_client)
provider_manager.register_provider('openmodeldb_api', openmodeldb_provider)
providers.append(('openmodeldb_api', openmodeldb_provider))
logger.debug("OpenModelDB metadata provider registered (also included in fallback)")
except Exception as e:
logger.error(f"Failed to initialize OpenModelDB metadata provider: {e}")
else:
logger.debug("OpenModelDB metadata provider disabled by setting 'enable_openmodeldb_api'")
# Preset fallback orderings (see module-level _PRESET_PROVIDER_ORDERS).
# civitai_api is always first (better metadata); the remaining providers
# are arranged by the configured preset. Providers that are not
@@ -135,6 +159,7 @@ async def update_metadata_providers():
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
enable_openmodeldb_api = settings_manager.get('enable_openmodeldb_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
# Reinitialize all providers with new settings
@@ -153,9 +178,10 @@ async def update_metadata_providers():
)
logger.info(
"Updated metadata providers: archive_db=%s, civarchive_api=%s, chain=%s",
"Updated metadata providers: archive_db=%s, civarchive_api=%s, openmodeldb_api=%s, chain=%s",
enable_archive_db,
enable_civarchive_api,
enable_openmodeldb_api,
chain,
)
return provider_manager