feat(metadata): add OpenModelDB metadata provider and model source for upscalers

Add OpenModelDB (openmodeldb.info) as a metadata and download source for
the existing upscaler model type.

Metadata:
- New OpenModelDBClient: fetches the site's bulk JSON dumps, caches them
  on disk (24h TTL + ETag revalidation), and builds a local sha256 index
- New OpenModelDBModelMetadataProvider adapts catalogue entries to the
  CivitAI-shaped version dict contract; registered in the fallback chain
  behind the enable_openmodeldb_api setting (default on), gated to the
  upscaler sub-type so other model types never trigger the dump download
- Persisted provenance uses metadata_source "openmodeldb" plus a nested
  openmodeldb block (page URL, architecture, scale, license)

Images: paired-image LR/SR URLs are ephemeral imgdiff.net sessions, so
displayable images come from the site-hosted auto-generated thumbnails
(model-level cover leads images[], per-image thumbs for the rest); the
original comparison URL is kept in meta.comparisonUrl.

Downloads:
- New OpenModelDBSource (flat model ids, omdb: group prefix) with
  resource filename derivation that recovers names hidden mid-path
  (mediafire) or synthesizes {id}.{type} for folder links
- HTML-gateway mirrors (mediafire/mega/drive) are rejected with a clear
  manual-download hint instead of silently saving an HTML page as .pth
- ModelSource base gains is_valid_source_id / default_subdir_parts /
  resolve_download_url hooks so flat-id sources need no platform branches

UI: "View on OpenModelDB" link in the model modal (downloaded and
hash-enriched models), settings toggle next to the CivArchive one.
This commit is contained in:
Will Miao
2026-10-03 21:13:07 +08:00
parent 515469054c
commit 35f1ced41a
39 changed files with 2787 additions and 37 deletions
+60 -1
View File
@@ -112,7 +112,10 @@ class _RateLimitRetryHelper:
# Labels of providers that are free to consult even while a network provider
# is rate-limited (local lookups, no vendor cost).
_LOCAL_PROVIDER_LABELS = frozenset({"sqlite"})
# "openmodeldb_api" qualifies because its lookups hit a local index built from
# a cached bulk dump; the underlying site is a static host (GitHub Pages), so
# even a cold cache refresh is a single cheap GET against a different vendor.
_LOCAL_PROVIDER_LABELS = frozenset({"sqlite", "openmodeldb_api"})
class ModelMetadataProvider(ABC):
@@ -480,6 +483,36 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
except json.JSONDecodeError:
return None
class OpenModelDBModelMetadataProvider(ModelMetadataProvider):
"""Provider that serves upscaler metadata from the OpenModelDB catalogue.
Only hash lookups are supported: OpenModelDB has no per-model or version
API, so the remaining provider surface intentionally returns None and lets
the fallback chain continue to the next provider.
"""
def __init__(self, openmodeldb_client):
self.client = openmodeldb_client
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
return await self.client.get_model_by_hash(model_hash)
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
"""Not supported: OpenModelDB models have no version history API."""
return None
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
"""Not supported: OpenModelDB models have no version history API."""
return None
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
"""Not supported: OpenModelDB models have no version history API."""
return None, "Model not found"
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""Not supported by the OpenModelDB provider."""
return None
class FallbackMetadataProvider(ModelMetadataProvider):
"""Try providers in order, return first successful result.
@@ -750,6 +783,32 @@ class FallbackMetadataProvider(ModelMetadataProvider):
def _iter_providers(self):
return zip(self.providers, self._provider_labels)
def excluding(self, labels: "frozenset[str] | set[str]") -> "FallbackMetadataProvider":
"""Return a copy of this chain without the providers named in *labels*.
Used by the metadata sync service to skip providers that cannot apply
to a given model (e.g. OpenModelDB only indexes upscalers), so their
cold-start cost (a bulk dump download) is never paid pointlessly.
"""
kept = [
(label, provider)
for provider, label in self._iter_providers()
if label not in labels
]
if len(kept) == len(self.providers):
return self
if not kept:
# Never produce an empty chain; the caller still needs a provider
# that can at least report "Model not found".
return self
return FallbackMetadataProvider(
kept,
rate_limit_retry_limit=self._rate_limit_retry_limit,
rate_limit_base_delay=self._rate_limit_base_delay,
rate_limit_max_delay=self._rate_limit_max_delay,
rate_limit_jitter_ratio=self._rate_limit_jitter_ratio,
)
async def _call_with_rate_limit(self, label: str, func, *args, **kwargs):
return await self._rate_limit_helper.run(label, func, *args, **kwargs)