feat(download): support ModelScope repositories in the URL downloader

ModelScope became a linkable source, but downloading from it was impossible:
the URL picker only recognised huggingface.co, the file listing hit a
huggingface-only endpoint, the resolve URL was hardcoded, and the default
path template always wrote into a `huggingface/` directory.

Move the download knowledge into the providers so the handlers stay generic:

- `ModelSource` gains `list_files()`, `file_download_url()`,
  `default_revision` and `default_subdir`. `HuggingFaceSource` keeps the Hub
  tree API (`/api/models/{id}/tree/{rev}`, LFS-aware sizes, `main`).
  `ModelScopeSource` uses `/api/v1/models/{id}/repo/files?Revision=master`
  — which reports real byte sizes for LFS files, so no HEAD probe is needed,
  and which only accepts `master` (an HF-imported repo still 404s on `main`)
  — and downloads through `/models/{id}/resolve/{rev}/{path}`. That URL
  redirects to a CDN target carrying a time-limited `auth_key`, so it is
  rebuilt on every request and never cached, which is also what keeps
  resumable Range requests working.
- `hf_handlers.py`/`HfHandler` become `model_source_handlers.py`/
  `ModelSourceHandler` with `list_model_source_files` and
  `download_model_source`. New routes `/api/lm/model-source-files` and
  `/api/lm/download-model-source`; the old `/api/lm/hf-repo-files` and
  `/api/lm/download-hf-model` paths stay as aliases, and a payload without
  `platform` still means Hugging Face, so existing callers are unaffected.
- A downloaded sidecar now records `source_platform` + `source_url` (with the
  `hf_url` alias only for Hugging Face) instead of always writing `hf_url`,
  and `use_default_paths` files ModelScope downloads under
  `modelscope/<owner>/<repo>`. The now-unused shared HF aiohttp session and
  its shutdown hook are gone; providers open short-lived sessions.
- Frontend: `detectUrlType` returns the platform-neutral
  `model-source-repo` / `model-source-file` plus an explicit `platform`, the
  DownloadManager's `hf*` state and methods are renamed to `source*`, every
  `source === 'huggingface'` check becomes `isExternalModelSource()`, and
  batch groups are keyed by `platform:repo` so the same `owner/name` on two
  sites renders as two groups. A bare `owner/name` still means Hugging Face.
- `is_valid_source_id()` centralises repo-id validation (exactly
  `owner/name`, no traversal, no leading dot). This also fixes the old HF
  download check that rejected any dot in the name, i.e. legitimate repos
  such as `black-forest-labs/FLUX.1-dev`.

Verified against the live APIs: the example repo lists 8 weight files with
correct sizes, and a ranged GET of the built resolve URL returns 206 after
following the redirect to the CDN. Backend 2853 passed; frontend 1143 JS +
91 Vue passed. The nine locales carry the refreshed download copy in the
next commit.
This commit is contained in:
Will Miao
2026-09-14 07:42:51 +08:00
parent b9bf006998
commit 38d4c59b4c
22 changed files with 1953 additions and 667 deletions
@@ -26,6 +26,7 @@ const {
},
},
downloadModel: vi.fn(),
downloadModelSource: vi.fn(),
downloadHfModel: vi.fn(),
cancelDownload: vi.fn(),
getPageState: vi.fn(() => ({})),
@@ -158,7 +159,7 @@ describe('DownloadManager batch download summary flow', () => {
// Reset the shared mocks so mockResolvedValueOnce queues and call
// history never leak between tests.
mockApiClient.downloadModel.mockReset();
mockApiClient.downloadHfModel.mockReset();
mockApiClient.downloadModelSource.mockReset();
mockApiClient.cancelDownload.mockReset();
showToastMock.mockClear();
showDownloadBatchSummaryMock.mockClear();
@@ -406,14 +407,15 @@ describe('DownloadManager batch download summary flow', () => {
expect(showToastMock).toHaveBeenCalledWith('toast.loras.downloadCompleted', {}, 'success');
});
it('shows a summary for HF partial failure and retries only the failed files', async () => {
manager.hfRepoId = 'user/repo';
manager.hfSelectedFiles = ['a.safetensors', 'b.safetensors'];
mockApiClient.downloadHfModel
it('shows a summary for external repo partial failure and retries only the failed files', async () => {
manager.sourcePlatform = 'huggingface';
manager.sourceRepoId = 'user/repo';
manager.sourceSelectedFiles = ['a.safetensors', 'b.safetensors'];
mockApiClient.downloadModelSource
.mockResolvedValueOnce({ success: true })
.mockResolvedValueOnce({ success: false, error: 'denied' });
const result = await manager._downloadHfSingle({ modelRoot: '/m', useDefaultPaths: true });
const result = await manager._downloadExternalRepoFiles({ modelRoot: '/m', useDefaultPaths: true });
expect(result).toBe(false);
expect(showDownloadBatchSummaryMock).toHaveBeenCalledTimes(1);
@@ -428,7 +430,9 @@ describe('DownloadManager batch download summary flow', () => {
await summary.onRetry();
expect(mockApiClient.downloadHfModel).toHaveBeenCalledTimes(3);
expect(mockApiClient.downloadHfModel.mock.calls[2][0].filename).toBe('b.safetensors');
expect(mockApiClient.downloadModelSource).toHaveBeenCalledTimes(3);
const retryArgs = mockApiClient.downloadModelSource.mock.calls[2][0];
expect(retryArgs.filename).toBe('b.safetensors');
expect(retryArgs.platform).toBe('huggingface');
});
});
@@ -0,0 +1,269 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const {
DOWNLOAD_MANAGER_MODULE,
MODAL_MANAGER_MODULE,
UI_HELPERS_MODULE,
STATE_MODULE,
LOADING_MANAGER_MODULE,
API_FACTORY_MODULE,
STORAGE_HELPERS_MODULE,
FOLDER_TREE_MANAGER_MODULE,
I18N_HELPERS_MODULE,
SUMMARY_MODULE,
mockApiClient,
mockLoadingManager,
showDownloadBatchSummaryMock,
} = vi.hoisted(() => {
const mockApiClient = {
apiConfig: { config: { displayName: 'LoRA', singularName: 'lora' } },
downloadModel: vi.fn(),
downloadModelSource: vi.fn(),
fetchModelSourceFiles: vi.fn(),
cancelDownload: vi.fn(),
getPageState: vi.fn(() => ({})),
};
const mockLoadingManager = {
showSimpleLoading: vi.fn(),
hide: vi.fn(),
restoreProgressBar: vi.fn(),
showDownloadProgress: vi.fn(() => vi.fn()),
setStatus: vi.fn(),
showCancelButton: vi.fn(),
};
return {
DOWNLOAD_MANAGER_MODULE: new URL('../../../static/js/managers/DownloadManager.js', import.meta.url).pathname,
MODAL_MANAGER_MODULE: new URL('../../../static/js/managers/ModalManager.js', import.meta.url).pathname,
UI_HELPERS_MODULE: new URL('../../../static/js/utils/uiHelpers.js', import.meta.url).pathname,
STATE_MODULE: new URL('../../../static/js/state/index.js', import.meta.url).pathname,
LOADING_MANAGER_MODULE: new URL('../../../static/js/managers/LoadingManager.js', import.meta.url).pathname,
API_FACTORY_MODULE: new URL('../../../static/js/api/modelApiFactory.js', import.meta.url).pathname,
STORAGE_HELPERS_MODULE: new URL('../../../static/js/utils/storageHelpers.js', import.meta.url).pathname,
FOLDER_TREE_MANAGER_MODULE: new URL('../../../static/js/components/FolderTreeManager.js', import.meta.url).pathname,
I18N_HELPERS_MODULE: new URL('../../../static/js/utils/i18nHelpers.js', import.meta.url).pathname,
SUMMARY_MODULE: new URL('../../../static/js/components/DownloadBatchSummaryModal.js', import.meta.url).pathname,
mockApiClient,
mockLoadingManager,
showDownloadBatchSummaryMock: vi.fn(),
};
});
vi.mock(MODAL_MANAGER_MODULE, () => ({
modalManager: { showModal: vi.fn(), closeModal: vi.fn() },
}));
vi.mock(UI_HELPERS_MODULE, () => ({ showToast: vi.fn() }));
vi.mock(STATE_MODULE, () => ({
state: { global: { settings: {} }, loadingManager: mockLoadingManager },
}));
vi.mock(LOADING_MANAGER_MODULE, () => ({
LoadingManager: vi.fn(() => mockLoadingManager),
}));
vi.mock(API_FACTORY_MODULE, () => ({
getModelApiClient: vi.fn(() => mockApiClient),
resetAndReload: vi.fn(),
}));
vi.mock(STORAGE_HELPERS_MODULE, () => ({
getStorageItem: vi.fn((_key, defaultValue) => defaultValue),
setStorageItem: vi.fn(),
}));
vi.mock(FOLDER_TREE_MANAGER_MODULE, () => ({
FolderTreeManager: vi.fn(() => ({ clearSelection: vi.fn(), init: vi.fn() })),
}));
vi.mock(I18N_HELPERS_MODULE, () => ({
translate: vi.fn((_, __, fallback) => fallback ?? ''),
}));
vi.mock(SUMMARY_MODULE, () => ({
showDownloadBatchSummary: showDownloadBatchSummaryMock,
}));
class FakeWebSocket {
constructor(url) {
this.url = url;
this.onopen = null;
this.onmessage = null;
this.onerror = null;
this.close = vi.fn();
queueMicrotask(() => {
if (this.onopen) this.onopen();
});
}
}
const MS_REPO_URL = 'https://modelscope.cn/models/jj3550945163/Krea-2-LORA';
describe('DownloadManager external model source downloads', () => {
let DownloadManager;
let manager;
beforeEach(async () => {
document.body.innerHTML = '';
vi.stubGlobal('WebSocket', FakeWebSocket);
mockApiClient.downloadModelSource.mockReset();
mockApiClient.fetchModelSourceFiles.mockReset();
mockLoadingManager.showSimpleLoading.mockReset();
({ DownloadManager } = await import(DOWNLOAD_MANAGER_MODULE));
manager = new DownloadManager();
manager.apiClient = mockApiClient;
manager.showBatchPreviewStep = vi.fn();
manager.proceedToLocation = vi.fn();
});
it('loads a ModelScope repo as batch items on the master revision', async () => {
mockApiClient.fetchModelSourceFiles.mockResolvedValue([
{ filename: 'a.safetensors', size: 10 },
{ filename: 'sub/b.safetensors', size: 20 },
]);
const errorElement = { textContent: '' };
await manager._validateAndFetchExternalRepo([MS_REPO_URL], errorElement);
expect(mockApiClient.fetchModelSourceFiles).toHaveBeenCalledWith(
'jj3550945163/Krea-2-LORA',
'modelscope',
'master'
);
expect(errorElement.textContent).toBe('');
expect(manager.source).toBe('modelscope');
expect(manager.isBatchMode).toBe(true);
expect(manager.batchModels).toHaveLength(2);
expect(manager.batchModels[0]).toMatchObject({
source: 'modelscope',
platform: 'modelscope',
repo: 'jj3550945163/Krea-2-LORA',
revision: 'master',
filename: 'a.safetensors',
fileSizeBytes: 10,
displayName: 'a.safetensors',
});
expect(manager.showBatchPreviewStep).toHaveBeenCalled();
});
it('keeps Hugging Face on its own revision', async () => {
mockApiClient.fetchModelSourceFiles.mockResolvedValue([
{ filename: 'a.safetensors', size: 10 },
]);
await manager._validateAndFetchExternalRepo(
['https://huggingface.co/user/repo'],
{ textContent: '' }
);
expect(mockApiClient.fetchModelSourceFiles).toHaveBeenCalledWith(
'user/repo',
'huggingface',
'main'
);
expect(manager.batchModels[0].revision).toBe('main');
});
it('surfaces a listing failure on the URL field', async () => {
mockApiClient.fetchModelSourceFiles.mockRejectedValue(new Error('Repository not found'));
const errorElement = { textContent: '' };
await manager._validateAndFetchExternalRepo([MS_REPO_URL], errorElement);
expect(errorElement.textContent).toBe('Repository not found');
expect(manager.showBatchPreviewStep).not.toHaveBeenCalled();
});
it('skips file selection for a direct ModelScope file URL', async () => {
await manager._validateAndFetchExternalRepo(
[`${MS_REPO_URL}/resolve/master/Krea-2-LORA_c1-st1000.safetensors`],
{ textContent: '' }
);
expect(manager.isBatchMode).toBe(false);
expect(manager.sourcePlatform).toBe('modelscope');
expect(manager.sourceRepoId).toBe('jj3550945163/Krea-2-LORA');
expect(manager.sourceSelectedFiles).toEqual(['Krea-2-LORA_c1-st1000.safetensors']);
expect(manager.proceedToLocation).toHaveBeenCalled();
});
it('downloads a single ModelScope file through the generic endpoint', async () => {
mockApiClient.downloadModelSource.mockResolvedValue({ success: true });
manager.sourcePlatform = 'modelscope';
manager.sourceRepoId = 'jj3550945163/Krea-2-LORA';
manager.sourceSelectedFiles = ['Krea-2-LORA_c1-st1000.safetensors'];
await manager._downloadExternalRepoFiles({
modelRoot: '/models/loras',
targetFolder: '',
useDefaultPaths: true,
});
expect(mockApiClient.downloadModelSource).toHaveBeenCalledTimes(1);
expect(mockApiClient.downloadModelSource.mock.calls[0][0]).toMatchObject({
platform: 'modelscope',
repo: 'jj3550945163/Krea-2-LORA',
filename: 'Krea-2-LORA_c1-st1000.safetensors',
revision: 'master',
});
});
it('carries the platform through a batch download', async () => {
mockApiClient.downloadModelSource.mockResolvedValue({ success: true });
manager.showBatchPreviewStep = vi.fn();
await manager.executeBatchDownload(
[
{
source: 'modelscope',
platform: 'modelscope',
repo: 'u/r',
filename: 'f.safetensors',
revision: 'master',
displayName: 'f.safetensors',
checked: true,
},
],
{ modelRoot: '/models/loras', targetFolder: '', useDefaultPaths: true }
);
expect(mockApiClient.downloadModelSource).toHaveBeenCalledTimes(1);
expect(mockApiClient.downloadModelSource.mock.calls[0][0]).toMatchObject({
platform: 'modelscope',
repo: 'u/r',
filename: 'f.safetensors',
revision: 'master',
});
expect(mockApiClient.downloadModel).not.toHaveBeenCalled();
});
it('links failures to the ModelScope file page', async () => {
expect(
manager._buildSingleItemUrl({
source: 'modelscope',
repo: 'u/r',
filename: 'sub/f.safetensors',
})
).toBe('https://modelscope.cn/models/u/r/file/view/master/sub/f.safetensors');
expect(
manager._buildSingleItemUrl({
source: 'huggingface',
repo: 'u/r',
filename: 'f.safetensors',
})
).toBe('https://huggingface.co/u/r/blob/main/f.safetensors');
});
it('groups the same repo name on two platforms separately', () => {
const hf = { source: 'huggingface', repo: 'u/r' };
const ms = { source: 'modelscope', repo: 'u/r' };
expect(manager._externalGroupKey(hf)).toBe('huggingface:u/r');
expect(manager._externalGroupKey(ms)).toBe('modelscope:u/r');
expect(manager._externalGroupKey(hf)).not.toBe(manager._externalGroupKey(ms));
});
});