feat(download): support ModelScope repositories in the URL downloader

ModelScope became a linkable source, but downloading from it was impossible: the URL picker only recognised huggingface.co, the file listing hit a huggingface-only endpoint, the resolve URL was hardcoded, and the default path template always wrote into a `huggingface/` directory. Move the download knowledge into the providers so the handlers stay generic: - `ModelSource` gains `list_files()`, `file_download_url()`, `default_revision` and `default_subdir`. `HuggingFaceSource` keeps the Hub tree API (`/api/models/{id}/tree/{rev}`, LFS-aware sizes, `main`). `ModelScopeSource` uses `/api/v1/models/{id}/repo/files?Revision=master` — which reports real byte sizes for LFS files, so no HEAD probe is needed, and which only accepts `master` (an HF-imported repo still 404s on `main`) — and downloads through `/models/{id}/resolve/{rev}/{path}`. That URL redirects to a CDN target carrying a time-limited `auth_key`, so it is rebuilt on every request and never cached, which is also what keeps resumable Range requests working. - `hf_handlers.py`/`HfHandler` become `model_source_handlers.py`/ `ModelSourceHandler` with `list_model_source_files` and `download_model_source`. New routes `/api/lm/model-source-files` and `/api/lm/download-model-source`; the old `/api/lm/hf-repo-files` and `/api/lm/download-hf-model` paths stay as aliases, and a payload without `platform` still means Hugging Face, so existing callers are unaffected. - A downloaded sidecar now records `source_platform` + `source_url` (with the `hf_url` alias only for Hugging Face) instead of always writing `hf_url`, and `use_default_paths` files ModelScope downloads under `modelscope/<owner>/<repo>`. The now-unused shared HF aiohttp session and its shutdown hook are gone; providers open short-lived sessions. - Frontend: `detectUrlType` returns the platform-neutral `model-source-repo` / `model-source-file` plus an explicit `platform`, the DownloadManager's `hf*` state and methods are renamed to `source*`, every `source === 'huggingface'` check becomes `isExternalModelSource()`, and batch groups are keyed by `platform:repo` so the same `owner/name` on two sites renders as two groups. A bare `owner/name` still means Hugging Face. - `is_valid_source_id()` centralises repo-id validation (exactly `owner/name`, no traversal, no leading dot). This also fixes the old HF download check that rejected any dot in the name, i.e. legitimate repos such as `black-forest-labs/FLUX.1-dev`. Verified against the live APIs: the example repo lists 8 weight files with correct sizes, and a ranged GET of the built resolve URL returns 206 after following the redirect to the CDN. Backend 2853 passed; frontend 1143 JS + 91 Vue passed. The nine locales carry the refreshed download copy in the next commit.
2026-09-21 03:01:27 -03:00 · 2026-09-14 07:42:51 +08:00
parent b9bf006998
commit 38d4c59b4c
22 changed files with 1953 additions and 667 deletions
@@ -19,21 +19,35 @@ export const MODEL_SOURCES = [
    groupPrefix: 'hf',
    supportsEnrichment: true,
    supportsDownload: true,
+    defaultRevision: 'main',
+    defaultSubdir: 'huggingface',
    exampleUrl: 'https://huggingface.co/user/repo',
    placeholder: 'https://huggingface.co/user/repo',
    pattern: /^https?:\/\/(?:www\.)?huggingface\.co\/([^/?#\s]+\/[^/?#\s]+)/i,
+    // `blob` is the web preview page; it maps 1:1 to the `resolve` download URL.
+    filePattern:
+      /^https?:\/\/(?:www\.)?huggingface\.co\/([^/?#\s]+\/[^/?#\s]+)\/(?:resolve|blob)\/([^/?#\s]+)\/(.+)$/i,
    canonical: (id) => `https://huggingface.co/${id}`,
+    filePage: (id, filename) => `https://huggingface.co/${id}/blob/main/${filename}`,
+    // Bare `user/repo` has always meant Hugging Face; keep that meaning.
+    bareRepoPattern: /^([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)$/,
  },
  {
    platform: 'modelscope',
    label: 'ModelScope',
    groupPrefix: 'ms',
    supportsEnrichment: true,
-    supportsDownload: false,
+    supportsDownload: true,
+    defaultRevision: 'master',
+    defaultSubdir: 'modelscope',
    exampleUrl: 'https://modelscope.cn/models/user/repo',
    placeholder: 'https://modelscope.cn/models/user/repo',
    pattern: /^https?:\/\/(?:www\.)?modelscope\.(?:cn|com)\/models\/([^/?#\s]+\/[^/?#\s]+)/i,
+    filePattern:
+      /^https?:\/\/(?:www\.)?modelscope\.(?:cn|com)\/models\/([^/?#\s]+\/[^/?#\s]+)\/resolve\/([^/?#\s]+)\/(.+)$/i,
    canonical: (id) => `https://modelscope.cn/models/${id}`,
+    filePage: (id, filename) =>
+      `https://modelscope.cn/models/${id}/file/view/master/${filename}`,
  },
  {
    platform: 'tensorart',
@@ -41,10 +55,14 @@ export const MODEL_SOURCES = [
    groupPrefix: 'ta',
    supportsEnrichment: false,
    supportsDownload: false,
+    defaultRevision: '',
+    defaultSubdir: '',
    exampleUrl: 'https://tensor.art/models/827823520299086029',
    placeholder: 'https://tensor.art/models/827823520299086029',
    pattern: /^https?:\/\/(?:www\.)?(?:tensor\.art|tusi\.cn)\/models\/(\d+)/i,
+    filePattern: null,
    canonical: (id) => `https://tensor.art/models/${id}`,
+    filePage: null,
  },
 ];

@@ -171,3 +189,103 @@ export function openModelSource(url) {
  if (!url) return;
  window.open(url, '_blank', 'noopener,noreferrer');
 }
+
+// ---------------------------------------------------------------------------
+// Download support
+// ---------------------------------------------------------------------------
+
+/** Sources whose repositories the backend can download from. */
+export const DOWNLOADABLE_SOURCES = MODEL_SOURCES.filter((s) => s.supportsDownload);
+
+/**
+ * Whether a DownloadManager `source` value refers to an external repository
+ * download (as opposed to a CivitAI/CivArchive version or a direct link).
+ */
+export function isExternalModelSource(source) {
+  return DOWNLOADABLE_SOURCES.some((s) => s.platform === source);
+}
+
+/** Return the downloadable source descriptor for a platform, or null. */
+export function getDownloadSource(platform) {
+  const source = getModelSource(platform);
+  return source && source.supportsDownload ? source : null;
+}
+
+/** Normalise a repository id: reject traversal, exactly one slash. */
+export function isValidRepoId(repo) {
+  if (!repo || typeof repo !== 'string' || repo.split('/').length !== 2) return false;
+  return repo
+    .split('/')
+    .every((part) => part && part !== '.' && part !== '..' && /^[A-Za-z0-9_][\w.-]*$/.test(part));
+}
+
+/**
+ * Recognise a downloadable model-source URL.
+ *
+ * Handles both a repository page and a direct file (resolve) URL for every
+ * source that supports downloads, plus the historical bare `owner/name`
+ * shorthand, which only ever meant Hugging Face.
+ *
+ * @returns {{kind: 'repo'|'file', platform: string, label: string,
+ *   repo: string, revision?: string, filename?: string}|null}
+ */
+export function detectModelSourceDownloadUrl(url) {
+  if (!url || typeof url !== 'string') return null;
+  const candidate = url.trim();
+  if (!candidate) return null;
+
+  // Direct file URLs first: the repo pattern would match their prefix and
+  // lose the revision/filename.
+  for (const source of DOWNLOADABLE_SOURCES) {
+    if (!source.filePattern) continue;
+    const match = candidate.match(source.filePattern);
+    if (match) {
+      return {
+        kind: 'file',
+        platform: source.platform,
+        label: source.label,
+        repo: match[1],
+        revision: match[2],
+        filename: match[3],
+      };
+    }
+  }
+
+  for (const source of DOWNLOADABLE_SOURCES) {
+    const match = candidate.match(source.pattern);
+    if (match) {
+      return {
+        kind: 'repo',
+        platform: source.platform,
+        label: source.label,
+        repo: match[1],
+      };
+    }
+  }
+
+  if (!candidate.includes('://')) {
+    for (const source of DOWNLOADABLE_SOURCES) {
+      if (!source.bareRepoPattern) continue;
+      const match = candidate.match(source.bareRepoPattern);
+      if (match && isValidRepoId(match[1])) {
+        return {
+          kind: 'repo',
+          platform: source.platform,
+          label: source.label,
+          repo: match[1],
+        };
+      }
+    }
+  }
+
+  return null;
+}
+
+/** Human-facing page for one file of an external repository. */
+export function buildModelSourceFilePage({ platform, repo, filename }) {
+  const source = getModelSource(platform);
+  if (!source || !source.filePage || !filename) {
+    return source ? source.canonical(repo) : null;
+  }
+  return source.filePage(repo, filename);
+}