feat(metadata): add OpenModelDB metadata provider and model source for upscalers

Add OpenModelDB (openmodeldb.info) as a metadata and download source for
the existing upscaler model type.

Metadata:
- New OpenModelDBClient: fetches the site's bulk JSON dumps, caches them
  on disk (24h TTL + ETag revalidation), and builds a local sha256 index
- New OpenModelDBModelMetadataProvider adapts catalogue entries to the
  CivitAI-shaped version dict contract; registered in the fallback chain
  behind the enable_openmodeldb_api setting (default on), gated to the
  upscaler sub-type so other model types never trigger the dump download
- Persisted provenance uses metadata_source "openmodeldb" plus a nested
  openmodeldb block (page URL, architecture, scale, license)

Images: paired-image LR/SR URLs are ephemeral imgdiff.net sessions, so
displayable images come from the site-hosted auto-generated thumbnails
(model-level cover leads images[], per-image thumbs for the rest); the
original comparison URL is kept in meta.comparisonUrl.

Downloads:
- New OpenModelDBSource (flat model ids, omdb: group prefix) with
  resource filename derivation that recovers names hidden mid-path
  (mediafire) or synthesizes {id}.{type} for folder links
- HTML-gateway mirrors (mediafire/mega/drive) are rejected with a clear
  manual-download hint instead of silently saving an HTML page as .pth
- ModelSource base gains is_valid_source_id / default_subdir_parts /
  resolve_download_url hooks so flat-id sources need no platform branches

UI: "View on OpenModelDB" link in the model modal (downloaded and
hash-enriched models), settings toggle next to the CivArchive one.
This commit is contained in:
Will Miao
2026-10-03 21:13:07 +08:00
parent 515469054c
commit 35f1ced41a
39 changed files with 2787 additions and 37 deletions
+123
View File
@@ -312,6 +312,7 @@ async def test_get_model_sources_lists_capabilities():
"modelscope",
"modelscope-ai",
"tensorart",
"openmodeldb",
}
assert by_platform["huggingface"]["supports_enrichment"] is True
assert by_platform["modelscope"]["supports_enrichment"] is True
@@ -1305,3 +1306,125 @@ async def test_download_model_source_sends_no_headers_without_hf_token(
assert response.status == 200
assert captured["custom_headers"] is None
# ---------------------------------------------------------------------------
# OpenModelDB downloads
# ---------------------------------------------------------------------------
def _seed_openmodeldb_client(tmp_path, monkeypatch) -> None:
"""Install a catalogue-loaded OpenModelDB client as the singleton."""
from py.services.openmodeldb_client import OpenModelDBClient
client = OpenModelDBClient(cache_dir=str(tmp_path / "omdb-cache"))
client._install_payloads(
{
"models": {
"4x-UltraSharp": {
"name": "4x UltraSharp",
"resources": [
{
"platform": "pytorch",
"type": "pth",
"size": 67_000_000,
"sha256": "a" * 64,
"urls": ["https://files.example.com/4x-UltraSharp.pth"],
}
],
}
},
"users": {},
"tags": {},
"architectures": {},
}
)
monkeypatch.setattr(
OpenModelDBClient, "get_instance", AsyncMock(return_value=client)
)
@pytest.mark.asyncio
async def test_download_model_source_openmodeldb_default_paths(tmp_path, monkeypatch):
captured = _stub_download_backend(monkeypatch)
saved = AsyncMock()
monkeypatch.setattr(model_source_handlers, "_save_source_metadata", saved)
_seed_openmodeldb_client(tmp_path, monkeypatch)
response = await ModelSourceHandler().download_model_source(
FakeRequest(
json_data={
"platform": "openmodeldb",
# Flat catalogue id — no owner/name split.
"repo": "4x-UltraSharp",
"filename": "4x-UltraSharp.pth",
"model_root": str(tmp_path),
"use_default_paths": True,
}
)
)
assert response.status == 200
assert captured["url"] == "https://files.example.com/4x-UltraSharp.pth"
# Flat layout: the site sub-directory only, no owner/repo namespaces.
assert captured["save_path"] == str(
tmp_path / "openmodeldb" / "4x-UltraSharp.pth"
)
ref = saved.await_args.args[1]
assert ref.platform == "openmodeldb"
assert ref.source_id == "4x-UltraSharp"
assert ref.url == "https://openmodeldb.info/models/4x-UltraSharp"
@pytest.mark.asyncio
async def test_download_model_source_openmodeldb_unknown_model_returns_404(
tmp_path, monkeypatch
):
_stub_download_backend(monkeypatch)
_seed_openmodeldb_client(tmp_path, monkeypatch)
response = await ModelSourceHandler().download_model_source(
FakeRequest(
json_data={
"platform": "openmodeldb",
"repo": "nope",
"filename": "f.pth",
"model_root": str(tmp_path),
}
)
)
assert response.status == 404
assert "not found" in _json_payload(response)["error"]
@pytest.mark.asyncio
async def test_download_model_source_openmodeldb_rejects_repo_style_id(tmp_path):
response = await ModelSourceHandler().download_model_source(
FakeRequest(
json_data={
"platform": "openmodeldb",
"repo": "owner/name",
"filename": "f.pth",
"model_root": str(tmp_path),
}
)
)
assert response.status == 400
assert "Invalid repo format" in _json_payload(response)["error"]
@pytest.mark.asyncio
async def test_list_model_source_files_openmodeldb(tmp_path, monkeypatch):
_seed_openmodeldb_client(tmp_path, monkeypatch)
response = await ModelSourceHandler().list_model_source_files(
FakeRequest(query={"platform": "openmodeldb", "repo": "4x-UltraSharp"})
)
assert response.status == 200
assert _json_payload(response) == [
{"filename": "4x-UltraSharp.pth", "size": 67_000_000}
]