mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-20 18:51:26 -03:00
feat(download): support ModelScope repositories in the URL downloader
ModelScope became a linkable source, but downloading from it was impossible:
the URL picker only recognised huggingface.co, the file listing hit a
huggingface-only endpoint, the resolve URL was hardcoded, and the default
path template always wrote into a `huggingface/` directory.
Move the download knowledge into the providers so the handlers stay generic:
- `ModelSource` gains `list_files()`, `file_download_url()`,
`default_revision` and `default_subdir`. `HuggingFaceSource` keeps the Hub
tree API (`/api/models/{id}/tree/{rev}`, LFS-aware sizes, `main`).
`ModelScopeSource` uses `/api/v1/models/{id}/repo/files?Revision=master`
— which reports real byte sizes for LFS files, so no HEAD probe is needed,
and which only accepts `master` (an HF-imported repo still 404s on `main`)
— and downloads through `/models/{id}/resolve/{rev}/{path}`. That URL
redirects to a CDN target carrying a time-limited `auth_key`, so it is
rebuilt on every request and never cached, which is also what keeps
resumable Range requests working.
- `hf_handlers.py`/`HfHandler` become `model_source_handlers.py`/
`ModelSourceHandler` with `list_model_source_files` and
`download_model_source`. New routes `/api/lm/model-source-files` and
`/api/lm/download-model-source`; the old `/api/lm/hf-repo-files` and
`/api/lm/download-hf-model` paths stay as aliases, and a payload without
`platform` still means Hugging Face, so existing callers are unaffected.
- A downloaded sidecar now records `source_platform` + `source_url` (with the
`hf_url` alias only for Hugging Face) instead of always writing `hf_url`,
and `use_default_paths` files ModelScope downloads under
`modelscope/<owner>/<repo>`. The now-unused shared HF aiohttp session and
its shutdown hook are gone; providers open short-lived sessions.
- Frontend: `detectUrlType` returns the platform-neutral
`model-source-repo` / `model-source-file` plus an explicit `platform`, the
DownloadManager's `hf*` state and methods are renamed to `source*`, every
`source === 'huggingface'` check becomes `isExternalModelSource()`, and
batch groups are keyed by `platform:repo` so the same `owner/name` on two
sites renders as two groups. A bare `owner/name` still means Hugging Face.
- `is_valid_source_id()` centralises repo-id validation (exactly
`owner/name`, no traversal, no leading dot). This also fixes the old HF
download check that rejected any dot in the name, i.e. legitimate repos
such as `black-forest-labs/FLUX.1-dev`.
Verified against the live APIs: the example repo lists 8 weight files with
correct sizes, and a ranged GET of the built resolve URL returns 206 after
following the redirect to the CDN. Backend 2853 passed; frontend 1143 JS +
91 Vue passed. The nine locales carry the refreshed download copy in the
next commit.
This commit is contained in:
@@ -2,20 +2,38 @@
|
||||
|
||||
ModelScope exposes the same "model card as README.md" convention as
|
||||
Hugging Face, including a YAML frontmatter block that often carries
|
||||
``base_model:`` and ``trigger_words:``. Two public endpoints are used,
|
||||
neither of which requires an API key for public models:
|
||||
``base_model:`` and ``trigger_words:``. Three public endpoints are used,
|
||||
none of which requires an API key for public models:
|
||||
|
||||
* ``/models/{owner}/{name}/resolve/{revision}/README.md`` — raw model card
|
||||
* ``/api/v1/models/{owner}/{name}/repo?Revision=..&FilePath=README.md`` —
|
||||
the same content through the API, used as a fallback when the resolve
|
||||
URL is unavailable.
|
||||
* ``/api/v1/models/{owner}/{name}/repo/files?Revision=..`` — the file
|
||||
listing backing the download picker. It reports real sizes for LFS
|
||||
files (not the pointer size), so no extra HEAD request is needed.
|
||||
|
||||
Downloads go through ``/models/{owner}/{name}/resolve/{revision}/{path}``,
|
||||
which redirects to a CDN URL carrying a time-limited ``auth_key``.
|
||||
Requesting the resolve URL fresh on every attempt (which the shared
|
||||
downloader does, including for resumable Range requests) keeps that key
|
||||
valid; the CDN URL must never be cached.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from .base import ModelSource, fetch_text
|
||||
from .base import (
|
||||
ModelSource,
|
||||
ModelSourceError,
|
||||
fetch_json,
|
||||
fetch_text,
|
||||
filter_weight_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_URL_PATTERN = re.compile(
|
||||
r"https?://(?:www\.)?modelscope\.(?:cn|com)/models/(?P<id>[^/?#\s]+/[^/?#\s]+)"
|
||||
@@ -41,7 +59,9 @@ class ModelScopeSource(ModelSource):
|
||||
platform = "modelscope"
|
||||
label = "ModelScope"
|
||||
supports_enrichment = True
|
||||
supports_download = False
|
||||
supports_download = True
|
||||
default_revision = "master"
|
||||
default_subdir = "modelscope"
|
||||
url_pattern = _URL_PATTERN
|
||||
strict_url_pattern = _STRICT_URL_PATTERN
|
||||
|
||||
@@ -49,7 +69,10 @@ class ModelScopeSource(ModelSource):
|
||||
return f"https://modelscope.cn/models/{source_id}"
|
||||
|
||||
def asset_base_url(self, source_id: str, revision: str = "") -> str:
|
||||
return f"https://modelscope.cn/models/{source_id}/resolve/{revision or 'master'}"
|
||||
return (
|
||||
f"https://modelscope.cn/models/{source_id}/resolve/"
|
||||
f"{self.resolve_revision(revision)}"
|
||||
)
|
||||
|
||||
async def fetch_model_card(self, source_id: str) -> str:
|
||||
"""Fetch the model card, preferring the raw resolve URL."""
|
||||
@@ -72,5 +95,50 @@ class ModelScopeSource(ModelSource):
|
||||
return text
|
||||
return ""
|
||||
|
||||
async def list_files(
|
||||
self, source_id: str, revision: str = ""
|
||||
) -> list[dict]:
|
||||
"""List weight files via the repo files API.
|
||||
|
||||
``master`` is the only branch name the API accepts — even repos
|
||||
imported from Hugging Face are addressed as ``master`` (``main``
|
||||
returns 404) — so no fallback probing is done here.
|
||||
"""
|
||||
|
||||
revision = self.resolve_revision(revision)
|
||||
status, payload = await fetch_json(
|
||||
"https://modelscope.cn/api/v1/models/"
|
||||
f"{source_id}/repo/files?Revision={revision}"
|
||||
)
|
||||
|
||||
if status == 404:
|
||||
raise ModelSourceError(f"Repository '{source_id}' not found", status=404)
|
||||
if status != 200 or not isinstance(payload, dict):
|
||||
raise ModelSourceError(
|
||||
f"ModelScope API error while listing '{source_id}' (HTTP {status})"
|
||||
)
|
||||
|
||||
entries = []
|
||||
for entry in (payload.get("Data") or {}).get("Files") or []:
|
||||
if not isinstance(entry, dict) or entry.get("Type") != "blob":
|
||||
continue
|
||||
entries.append((entry.get("Path", ""), entry.get("Size", 0) or 0))
|
||||
|
||||
return filter_weight_files(entries)
|
||||
|
||||
def file_download_url(
|
||||
self, source_id: str, filename: str, revision: str = ""
|
||||
) -> str:
|
||||
return (
|
||||
f"https://modelscope.cn/models/{source_id}/resolve/"
|
||||
f"{self.resolve_revision(revision)}/{filename}"
|
||||
)
|
||||
|
||||
def page_url_for_file(self, source_id: str, filename: str) -> str:
|
||||
return (
|
||||
f"https://modelscope.cn/models/{source_id}/file/view/"
|
||||
f"{self.default_revision}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ModelScopeSource"]
|
||||
|
||||
Reference in New Issue
Block a user