feat(links): support ModelScope and TensorArt as model sources

A model file could only ever be linked to huggingface.co: `set_hf_url` validated the URL with a huggingface-only regex, the agent fetched the card from a hardcoded HF URL, and the readme processor built every relative image path off `https://huggingface.co/{repo}/resolve/main`. ModelScope publishes the same model-card convention (README.md + YAML frontmatter, often carrying `base_model:` and `trigger_words:`) behind a public, key-less API, so the enrichment pipeline could already serve it - it was the plumbing that was HF-shaped, not the idea. Make the external source a first-class, provider-driven concept: - New `py/services/model_sources/` registry. A `ModelSource` owns URL recognition (lenient for stored values, strict for user input), the canonical page URL, model-card fetching, the asset base URL and the capability flags. `HuggingFaceSource` is the previous logic relocated; `ModelScopeSource` reads `/models/{o}/{n}/resolve/{master|main}/README.md` and falls back to `/api/v1/models/{o}/{n}/repo`. `TensorArtSource` is link-only on purpose: tensor.art answers plain HTTP clients with a Cloudflare challenge and its internal API (ap-east-1.tensorart.cloud / cn.tensorart.net) rejects every /v1/model/* route with "invalid authorization header", so it declares supports_enrichment=False rather than failing silently later. - Metadata gains `source_platform` + `source_url`; `hf_url` stays as a read/write alias, written only for Hugging Face, so existing sidecars, cached rows and third-party consumers keep working. Normalisation runs at the scanner, the persistent cache (both directions, plus two new columns behind an ALTER migration) and the linking handler - which is what stops a user who switches sources from leaving a stale `hf_url` on a ModelScope model. - The agent pipeline keys off the provider instead of `hf_url`: the fast-fail gate now explains *why* a model is skipped (no source / unknown source / source without a reachable card), the prompt context exposes source_url/source_id/source_label/asset_base_url while still filling the legacy hf_url/repo aliases, and the four README image extractors take a base_url (defaulting to HF) so relative paths resolve against the right site. Version grouping generalises to hf: / ms: / ta: keys. - `POST /api/lm/set-hf-url` keeps its path and its legacy payload keys but accepts `source_url`, validates against every provider and returns the platform. `GET /api/lm/model-sources` lets the UI render the supported-site list from the server. - Frontend: a `modelSourceHelpers` mirror of the registry drives the link dialog, the card/modal globe (branded "View on ModelScope/TensorArt"), the version-group key and the enrichment gate; the versions tab no longer sends ms:/ta: keys to the CivitAI API. TensorArt stays in the list because provenance is worth keeping even when the card is unreadable - the dialog says so plainly ("Sites that don't expose one (currently TensorArt) can only be linked") and the context menu disables enrichment with a matching tooltip, instead of the user getting "Unsupported URL". Verified against the real ModelScope API: jj3550945163/Krea-2-LORA returns a 1882-byte card whose frontmatter carries base_model/tags/trigger_words, and relative images resolve to .../resolve/master/.... Tests: backend 2815 passed; frontend 1130 JS + 91 Vue passed; pytest tests/i18n and a Jinja compile pass over templates/. The nine locales carry [TODO: Translate] for the new strings, completed in the next commit.
2026-09-21 03:01:27 -03:00 · 2026-09-14 07:24:08 +08:00
parent 84146b62fd
commit 5ab0e88abc
51 changed files with 2518 additions and 279 deletions
@@ -7,6 +7,8 @@ import { MODEL_CONFIG } from '../../api/apiConfig.js';
 import { translate } from '../../utils/i18nHelpers.js';
 import { getNsfwLevelSelector } from '../shared/NsfwLevelSelector.js';
 import { classifyModelRelinkUrl } from '../../utils/civitaiUtils.js';
+import { parseModelSourceUrl, getModelSourceInfo } from '../../utils/modelSourceHelpers.js';
+import { escapeHtml } from '../shared/utils.js';

 // Mixin with shared functionality for LoraContextMenu and CheckpointContextMenu
 export const ModelContextMenuMixin = {
@@ -211,7 +213,7 @@ export const ModelContextMenuMixin = {
        setTimeout(() => urlInput.focus(), 50);
    },

-    // HuggingFace linking methods
+    // External model source linking (Hugging Face / ModelScope / TensorArt)
    showLinkHfModal() {
        const filePath = this.currentCard.dataset.filepath;
        if (!filePath) return;
@@ -225,15 +227,23 @@ export const ModelContextMenuMixin = {
        }

        this._boundLinkHfHandler = async () => {
-            const hfUrl = urlInput.value.trim();
-            if (!hfUrl) {
-                errorDiv.textContent = 'Please enter a HuggingFace repository URL.';
+            const rawUrl = urlInput.value.trim();
+            if (!rawUrl) {
+                errorDiv.textContent = translate(
+                    'modals.linkModelSource.urlRequired',
+                    {},
+                    'Please enter a model page URL.'
+                );
                return;
            }

-            const hfPattern = /^https?:\/\/huggingface\.co\/([^/]+\/[^/]+)\/?$/;
-            if (!hfPattern.test(hfUrl)) {
-                errorDiv.textContent = 'Invalid URL format. Expected: https://huggingface.co/user/repo';
+            const sourceInfo = parseModelSourceUrl(rawUrl);
+            if (!sourceInfo) {
+                errorDiv.textContent = translate(
+                    'modals.linkModelSource.invalidUrl',
+                    {},
+                    'Unsupported URL. Supported sites: Hugging Face, ModelScope, TensorArt.'
+                );
                return;
            }

@@ -241,12 +251,14 @@ export const ModelContextMenuMixin = {
            modalManager.closeModal('linkHfModal');

            try {
-                state.loadingManager.showSimpleLoading('Linking to HuggingFace...');
+                state.loadingManager.showSimpleLoading(
+                    translate('modals.linkModelSource.linking', {}, 'Linking model source...')
+                );

                const response = await fetch('/api/lm/set-hf-url', {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({ file_path: filePath, hf_url: hfUrl }),
+                    body: JSON.stringify({ file_path: filePath, source_url: sourceInfo.url }),
                });

                if (!response.ok) {
@@ -262,7 +274,7 @@ export const ModelContextMenuMixin = {
                    throw new Error(data.error || 'Failed to link model');
                }
            } catch (error) {
-                console.error('Error linking model to HuggingFace:', error);
+                console.error('Error linking model source:', error);
                showToast('toast.contextMenu.linkHfFailed', { message: error.message }, 'error');
            } finally {
                state.loadingManager.hide();
@@ -276,18 +288,68 @@ export const ModelContextMenuMixin = {

        modalManager.showModal('linkHfModal');

+        this._renderSupportedSources();
+
        setTimeout(() => urlInput.focus(), 50);
    },

-    // HF metadata enrichment (AI agent) methods
+    /**
+     * Refresh the supported-site hints from the server so the dialog reflects
+     * whatever sources this backend build actually knows about. Falls back to
+     * the static markup in the template when the request fails.
+     */
+    async _renderSupportedSources() {
+        const container = document.getElementById('hfSupportedSources');
+        if (!container) return;
+
+        try {
+            const response = await fetch('/api/lm/model-sources');
+            if (!response.ok) return;
+            const sources = await response.json();
+            if (!Array.isArray(sources) || sources.length === 0) return;
+
+            const examples = sources
+                .map((source) => source?.example_url)
+                .filter((url) => typeof url === 'string' && url);
+            if (examples.length === 0) return;
+
+            container.innerHTML = examples
+                .map((url) => `<strong>${escapeHtml(url)}</strong>`)
+                .join('<br>');
+        } catch (error) {
+            console.debug('Failed to load supported model sources:', error);
+        }
+    },
+
+    // Model metadata enrichment (AI agent) methods
    updateEnrichMenuItem(card) {
        const enrichItem = this.menu?.querySelector('[data-action="enrich-hf-llm"]');
        if (!enrichItem) return;
-        const hasHfUrl = !!card.dataset.hf_url;
-        enrichItem.classList.toggle('disabled', !hasHfUrl);
-        enrichItem.title = hasHfUrl
-            ? ''
-            : 'Link this model to a HuggingFace repo first (Link Model → Link to HuggingFace)';
+
+        const model = {
+            source_url: card.dataset.source_url || '',
+            source_platform: card.dataset.source_platform || '',
+            hf_url: card.dataset.hf_url || '',
+        };
+        const sourceInfo = getModelSourceInfo(model);
+        const canEnrich = Boolean(sourceInfo && sourceInfo.supportsEnrichment);
+
+        enrichItem.classList.toggle('disabled', !canEnrich);
+        if (canEnrich) {
+            enrichItem.title = '';
+        } else if (!sourceInfo) {
+            enrichItem.title = translate(
+                'toast.contextMenu.enrichNeedsSource',
+                {},
+                'Link this model to a model source first (Link Model → Link to Model Source)'
+            );
+        } else {
+            enrichItem.title = translate(
+                'toast.contextMenu.enrichUnsupportedSource',
+                { source: sourceInfo.label },
+                `AI enrichment is not available for ${sourceInfo.label} models`
+            );
+        }
    },

    async enrichWithAgent(filePath) {