feat(recipes): add reconnect remediation paths for missing recipe LoRAs

- Snapshot pre-rematch entry state (reconnectSnapshot) so rematched
  entries can be undone via the existing restore flow
- Bulk missing-LoRA downloads mark unresolvable failures hash-invalid,
  flipping those entries from download to reconnect candidacy
- Recipe modal always offers a reconnect action next to download for
  missing LoRA entries
- Rematch runs collect an opt-in relaxed-matching choice (also reconnect
  missing models by file name) via a pre-run options dialog on the
  global, bulk and single-recipe entries
- L4 (filename-level) matches are listed in a results dialog with
  per-entry undo
This commit is contained in:
Will Miao
2026-09-09 06:59:54 +08:00
parent e747946f7a
commit 1b5cbbbaa0
33 changed files with 2103 additions and 76 deletions
@@ -73,6 +73,22 @@ vi.mock('../../../static/js/components/shared/NsfwLevelSelector.js', () => ({
getNsfwLevelSelector: vi.fn(),
}));
// The real RematchModalManager runs against the mocked modalManager; confirm
// is invoked explicitly, mirroring the user clicking Rematch in the dialog.
async function confirmRematchOptions() {
const { rematchModalManager } = await import(
'../../../static/js/managers/RematchModalManager.js'
);
return rematchModalManager.confirmOptions();
}
async function cancelRematchOptions() {
const { rematchModalManager } = await import(
'../../../static/js/managers/RematchModalManager.js'
);
rematchModalManager.cancelOptions();
}
describe('BulkManager.rematchSelectedRecipes', () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -114,12 +130,16 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(rematchBulkModelsMock).toHaveBeenCalledWith([
'/recipes/a.webp',
'/recipes/b.webp',
'/recipes/c.webp',
]);
expect(rematchBulkModelsMock).toHaveBeenCalledWith(
[
'/recipes/a.webp',
'/recipes/b.webp',
'/recipes/c.webp',
],
{ relaxed: false }
);
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchComplete',
{ rematched: 4, skipped: 1, total: 3, entries: 4, recipes: 2, failures: 0 },
@@ -155,6 +175,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchCompleteErrors',
@@ -182,6 +203,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchAllFailed',
@@ -215,6 +237,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchUnmatched',
@@ -243,6 +266,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchSkipped',
@@ -268,6 +292,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchFailed',
@@ -284,6 +309,7 @@ describe('BulkManager.rematchSelectedRecipes', () => {
rematchBulkModelsMock.mockRejectedValue(new Error('network down'));
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.rematchFailed',
@@ -319,4 +345,100 @@ describe('BulkManager.rematchSelectedRecipes', () => {
);
expect(rematchBulkModelsMock).not.toHaveBeenCalled();
});
it('does not start the rematch until the options dialog is confirmed', async () => {
const bulk = await createBulkManager();
stateStub.selectedModels.add('/recipes/a.webp');
await bulk.rematchSelectedRecipes();
expect(rematchBulkModelsMock).not.toHaveBeenCalled();
rematchBulkModelsMock.mockResolvedValue({
success: true,
total: 1,
rematched: 1,
skipped: 0,
errors: 0,
matched_recipes: 1,
matched_entries: 1,
recipes: [],
});
await confirmRematchOptions();
expect(rematchBulkModelsMock).toHaveBeenCalledWith(['/recipes/a.webp'], { relaxed: false });
});
it('sends relaxed: true when the relaxed checkbox is checked', async () => {
const bulk = await createBulkManager();
stateStub.selectedModels.add('/recipes/a.webp');
document.body.innerHTML = '<input type="checkbox" id="rematchOptionsRelaxed">';
rematchBulkModelsMock.mockResolvedValue({
success: true,
total: 1,
rematched: 0,
skipped: 1,
errors: 0,
recipes: [],
});
await bulk.rematchSelectedRecipes();
// The dialog resets the checkbox to unchecked on open; the user opts in.
document.getElementById('rematchOptionsRelaxed').checked = true;
await confirmRematchOptions();
expect(rematchBulkModelsMock).toHaveBeenCalledWith(['/recipes/a.webp'], { relaxed: true });
document.body.innerHTML = '';
});
it('starts nothing when the options dialog is cancelled', async () => {
const bulk = await createBulkManager();
stateStub.selectedModels.add('/recipes/a.webp');
await bulk.rematchSelectedRecipes();
await cancelRematchOptions();
expect(rematchBulkModelsMock).not.toHaveBeenCalled();
expect(showToastMock).not.toHaveBeenCalledWith(
'toast.recipes.rematchComplete',
expect.anything(),
expect.anything()
);
});
it('shows the L4 results modal when the bulk result carries l4_matches', async () => {
const bulk = await createBulkManager();
stateStub.selectedModels.add('/recipes/a.webp');
const l4Matches = [
{ recipe_id: 'a', type: 'lora', entry: 'old.safetensors', file_name: 'new.safetensors', lora_index: 0 },
];
rematchBulkModelsMock.mockResolvedValue({
success: true,
total: 1,
rematched: 1,
skipped: 0,
errors: 0,
matched_recipes: 1,
matched_entries: 1,
recipes: [],
l4_matches: l4Matches,
});
const { rematchModalManager } = await import(
'../../../static/js/managers/RematchModalManager.js'
);
const showResultsSpy = vi
.spyOn(rematchModalManager, 'showResultsModal')
.mockImplementation(() => {});
await bulk.rematchSelectedRecipes();
await confirmRematchOptions();
expect(showResultsSpy).toHaveBeenCalledWith(l4Matches);
showResultsSpy.mockRestore();
});
});
@@ -0,0 +1,192 @@
import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest';
const MODULE = '../../../static/js/managers/BulkMissingLoraDownloadManager.js';
const showToastMock = vi.fn();
const updateProgressMock = vi.fn();
const updateSingleItemMock = vi.fn();
const mockApiClient = {
downloadModel: vi.fn(),
cancelDownload: vi.fn(),
fetchModelRoots: vi.fn(() => Promise.resolve({ roots: ['/models/loras'] })),
};
const loadingManagerStub = {
showDownloadProgress: vi.fn(() => updateProgressMock),
setStatus: vi.fn(),
showCancelButton: vi.fn(),
hide: vi.fn(),
};
vi.mock('../../../static/js/utils/uiHelpers.js', () => ({
showToast: showToastMock,
// Keep the real predicate: these tests assert on its classification.
isUnresolvableDownloadError: (message) =>
!!message && /(not found|no longer available|deleted|removed|404|410|gone)/.test(String(message).toLowerCase()),
}));
vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({
translate: vi.fn((_, __, fallback) => fallback ?? ''),
}));
vi.mock('../../../static/js/api/modelApiFactory.js', () => ({
getModelApiClient: vi.fn(() => mockApiClient),
}));
vi.mock('../../../static/js/api/apiConfig.js', () => ({
MODEL_TYPES: { LORA: 'loras', CHECKPOINT: 'checkpoints', EMBEDDING: 'embeddings' },
}));
vi.mock('../../../static/js/api/recipeApi.js', () => ({
extractRecipeId: (filePath) => {
if (!filePath) return null;
const basename = filePath.split('/').pop().split('\\').pop();
const dotIndex = basename.lastIndexOf('.');
return dotIndex > 0 ? basename.substring(0, dotIndex) : basename;
},
}));
vi.mock('../../../static/js/state/index.js', () => ({
state: {
loadingManager: loadingManagerStub,
virtualScroller: { updateSingleItem: updateSingleItemMock },
global: { settings: {} },
},
}));
vi.mock('../../../static/js/managers/ModalManager.js', () => ({
modalManager: { showModal: vi.fn(), closeModal: vi.fn() },
}));
/** Mirrors the FakeWebSocket pattern from downloadManager.batchSummary.test.js. */
class FakeWebSocket {
static instances = [];
constructor(url) {
this.url = url;
this.onopen = null;
this.onmessage = null;
this.onerror = null;
this.close = vi.fn();
FakeWebSocket.instances.push(this);
queueMicrotask(() => {
if (this.onopen) this.onopen();
});
}
}
const makeRecipe = (filePath, loras) => ({ file_path: filePath, loras });
describe('BulkMissingLoraDownloadManager unresolvable-failure write-back', () => {
let manager;
let fetchMock;
let requests;
beforeEach(async () => {
FakeWebSocket.instances = [];
vi.clearAllMocks();
loadingManagerStub.showDownloadProgress.mockReturnValue(updateProgressMock);
requests = [];
fetchMock = vi.fn((url, options) => {
requests.push({ url, options });
if (url === '/api/lm/recipe/lora/mark-hash-invalid') {
return Promise.resolve({ ok: true, json: () => Promise.resolve({}) });
}
// Recipe detail refresh after the download loop
return Promise.resolve({ ok: true, json: () => Promise.resolve({ id: 'refreshed' }) });
});
vi.stubGlobal('fetch', fetchMock);
vi.stubGlobal('WebSocket', FakeWebSocket);
vi.resetModules();
({ bulkMissingLoraDownloadManager: manager } = await import(MODULE));
manager.pendingLoras = [];
manager.pendingRecipes = [];
manager.pendingMissingByRecipe = null;
});
afterEach(() => {
vi.unstubAllGlobals();
});
const primePending = (recipes) => {
const stats = manager.collectMissingLoras(recipes);
manager.pendingRecipes = recipes;
manager.pendingMissingByRecipe = stats.missingLorasByRecipe;
return stats.uniqueLoras;
};
it('marks every recipe occurrence hash-invalid when the failure is unresolvable', async () => {
const entryA = { hash: 'abc123', file_name: 'a.safetensors', inLibrary: false, modelId: 1, id: 10 };
const entryB = { hash: 'abc123', file_name: 'a.safetensors', inLibrary: false, modelId: 1, id: 10 };
const recipe1 = makeRecipe('/recipes/r1.json', [entryA]);
const recipe2 = makeRecipe('/recipes/r2.json', [{ hash: 'x', file_name: 'keep.safetensors', inLibrary: true }, entryB]);
const uniqueLoras = primePending([recipe1, recipe2]);
mockApiClient.downloadModel.mockResolvedValue({ success: false, error: 'Model not found' });
await manager.executeDownload(uniqueLoras);
const markCalls = requests.filter(r => r.url === '/api/lm/recipe/lora/mark-hash-invalid');
expect(markCalls).toHaveLength(2);
const payloads = markCalls.map(r => JSON.parse(r.options.body));
expect(payloads).toContainEqual({ recipe_id: 'r1', lora_index: 0 });
expect(payloads).toContainEqual({ recipe_id: 'r2', lora_index: 1 });
expect(entryA.hashInvalid).toBe(true);
expect(entryB.hashInvalid).toBe(true);
expect(showToastMock).toHaveBeenCalledWith(
'toast.recipes.unresolvableMarkedForReconnect',
{ count: 2 },
'info',
expect.any(String),
);
});
it('leaves entries untouched when the failure is transient', async () => {
const entry = { hash: 'abc123', file_name: 'a.safetensors', inLibrary: false, modelId: 1, id: 10 };
const recipe = makeRecipe('/recipes/r1.json', [entry]);
const uniqueLoras = primePending([recipe]);
mockApiClient.downloadModel.mockResolvedValue({ success: false, error: 'Connection timed out' });
await manager.executeDownload(uniqueLoras);
expect(requests.some(r => r.url === '/api/lm/recipe/lora/mark-hash-invalid')).toBe(false);
expect(entry.hashInvalid).toBeUndefined();
expect(showToastMock).not.toHaveBeenCalledWith(
'toast.recipes.unresolvableMarkedForReconnect',
expect.anything(),
expect.anything(),
expect.anything(),
);
});
it('marks hash-invalid when the download request itself throws an unresolvable error', async () => {
const entry = { hash: 'abc123', file_name: 'a.safetensors', inLibrary: false, modelId: 1, id: 10 };
const recipe = makeRecipe('/recipes/r1.json', [entry]);
const uniqueLoras = primePending([recipe]);
mockApiClient.downloadModel.mockRejectedValue(new Error('410 Gone'));
await manager.executeDownload(uniqueLoras);
const markCalls = requests.filter(r => r.url === '/api/lm/recipe/lora/mark-hash-invalid');
expect(markCalls).toHaveLength(1);
expect(entry.hashInvalid).toBe(true);
});
it('does not mark entries whose download succeeds', async () => {
const entry = { hash: 'abc123', file_name: 'a.safetensors', inLibrary: false, modelId: 1, id: 10 };
const recipe = makeRecipe('/recipes/r1.json', [entry]);
const uniqueLoras = primePending([recipe]);
mockApiClient.downloadModel.mockResolvedValue({ success: true });
await manager.executeDownload(uniqueLoras);
expect(requests.some(r => r.url === '/api/lm/recipe/lora/mark-hash-invalid')).toBe(false);
});
});
@@ -0,0 +1,226 @@
import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest';
const showToastMock = vi.fn();
const translateMock = vi.fn((key, params, fallback) => (typeof fallback === 'string' ? fallback : key));
const modalManagerMock = {
showModal: vi.fn(),
closeModal: vi.fn(),
};
vi.mock('../../../static/js/managers/ModalManager.js', () => ({
modalManager: modalManagerMock,
}));
vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({
translate: translateMock,
}));
vi.mock('../../../static/js/utils/uiHelpers.js', () => ({
showToast: showToastMock,
}));
async function getManager() {
const { rematchModalManager } = await import(
'../../../static/js/managers/RematchModalManager.js'
);
return rematchModalManager;
}
describe('RematchModalManager options dialog', () => {
beforeEach(() => {
vi.clearAllMocks();
document.body.innerHTML = `
<p id="rematchOptionsMessage"></p>
<input type="checkbox" id="rematchOptionsRelaxed">
`;
});
afterEach(() => {
document.body.innerHTML = '';
});
it('does not invoke the callback until confirmOptions is called', async () => {
const manager = await getManager();
const onConfirm = vi.fn();
manager.showOptionsModal({ recipeCount: 3, onConfirm });
expect(modalManagerMock.showModal).toHaveBeenCalledWith('rematchOptionsModal');
expect(onConfirm).not.toHaveBeenCalled();
// Bulk message mentions the selection size.
expect(document.getElementById('rematchOptionsMessage').textContent).toContain('3');
// The checkbox always starts unchecked.
expect(document.getElementById('rematchOptionsRelaxed').checked).toBe(false);
manager.confirmOptions();
expect(onConfirm).toHaveBeenCalledWith({ relaxed: false });
expect(modalManagerMock.closeModal).toHaveBeenCalledWith('rematchOptionsModal');
});
it('uses the generic message when no recipe count is given', async () => {
const manager = await getManager();
manager.showOptionsModal({ onConfirm: vi.fn() });
expect(translateMock).toHaveBeenCalledWith(
'modals.rematchOptions.messageGlobal',
{},
'All recipes will be scanned against your local model library.'
);
});
it('uses the single-recipe message for scope: single', async () => {
const manager = await getManager();
manager.showOptionsModal({ scope: 'single', onConfirm: vi.fn() });
expect(translateMock).toHaveBeenCalledWith(
'modals.rematchOptions.messageSingle',
{},
'This recipe will be scanned against your local model library.'
);
});
it('passes relaxed: true when the checkbox is checked', async () => {
const manager = await getManager();
const onConfirm = vi.fn();
manager.showOptionsModal({ onConfirm });
document.getElementById('rematchOptionsRelaxed').checked = true;
manager.confirmOptions();
expect(onConfirm).toHaveBeenCalledWith({ relaxed: true });
});
it('resets the checkbox to unchecked each time the dialog opens', async () => {
const manager = await getManager();
const checkbox = document.getElementById('rematchOptionsRelaxed');
checkbox.checked = true;
manager.showOptionsModal({ onConfirm: vi.fn() });
expect(checkbox.checked).toBe(false);
});
it('cancelOptions runs nothing and clears the callback', async () => {
const manager = await getManager();
const onConfirm = vi.fn();
manager.showOptionsModal({ onConfirm });
manager.cancelOptions();
expect(modalManagerMock.closeModal).toHaveBeenCalledWith('rematchOptionsModal');
// A later confirm must not fire the cancelled callback.
manager.confirmOptions();
expect(onConfirm).not.toHaveBeenCalled();
});
});
describe('RematchModalManager results modal', () => {
beforeEach(() => {
vi.clearAllMocks();
document.body.innerHTML = '<ul id="rematchResultsList"></ul>';
global.fetch = vi.fn();
});
afterEach(() => {
document.body.innerHTML = '';
delete global.fetch;
});
it('does nothing for an empty match list', async () => {
const manager = await getManager();
manager.showResultsModal([]);
expect(modalManagerMock.showModal).not.toHaveBeenCalled();
});
it('renders one row per L4 match with entry, file name and recipe', async () => {
const manager = await getManager();
manager.showResultsModal([
{ recipe_id: 'r1', type: 'lora', entry: 'old.safetensors', file_name: 'new.safetensors', lora_index: 2 },
{ recipe_id: 'r2', type: 'checkpoint', entry: 'cp-old', file_name: 'cp-new.safetensors' },
]);
const rows = document.querySelectorAll('#rematchResultsList .rematch-results-row');
expect(rows).toHaveLength(2);
expect(rows[0].textContent).toContain('old.safetensors');
expect(rows[0].textContent).toContain('new.safetensors');
expect(rows[0].textContent).toContain('r1');
expect(rows[1].textContent).toContain('cp-new.safetensors');
expect(modalManagerMock.showModal).toHaveBeenCalledWith('rematchResultsModal');
});
it('undo posts to the lora restore endpoint and disables the row', async () => {
const manager = await getManager();
global.fetch.mockResolvedValue({
ok: true,
json: async () => ({ success: true }),
});
manager.showResultsModal([
{ recipe_id: 'r1', type: 'lora', entry: 'old.safetensors', file_name: 'new.safetensors', lora_index: 2 },
]);
const row = document.querySelector('.rematch-results-row');
const button = row.querySelector('.rematch-results-undo');
button.click();
await vi.waitFor(() => expect(button.disabled).toBe(true));
expect(global.fetch).toHaveBeenCalledWith('/api/lm/recipe/lora/restore', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ recipe_id: 'r1', lora_index: 2 }),
});
expect(row.classList.contains('undone')).toBe(true);
});
it('undo posts to the checkpoint restore endpoint with recipe_id only', async () => {
const manager = await getManager();
global.fetch.mockResolvedValue({
ok: true,
json: async () => ({ success: true }),
});
manager.showResultsModal([
{ recipe_id: 'r2', type: 'checkpoint', entry: 'cp-old', file_name: 'cp-new.safetensors' },
]);
const button = document.querySelector('.rematch-results-undo');
button.click();
await vi.waitFor(() => expect(button.disabled).toBe(true));
expect(global.fetch).toHaveBeenCalledWith('/api/lm/recipe/checkpoint/restore', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ recipe_id: 'r2' }),
});
});
it('keeps the row actionable and toasts when undo fails', async () => {
const manager = await getManager();
global.fetch.mockResolvedValue({
ok: true,
json: async () => ({ success: false, error: 'no snapshot' }),
});
manager.showResultsModal([
{ recipe_id: 'r1', type: 'lora', entry: 'old.safetensors', file_name: 'new.safetensors', lora_index: 0 },
]);
const row = document.querySelector('.rematch-results-row');
const button = row.querySelector('.rematch-results-undo');
button.click();
await vi.waitFor(() => expect(showToastMock).toHaveBeenCalled());
expect(button.disabled).toBe(false);
expect(row.classList.contains('undone')).toBe(false);
expect(showToastMock).toHaveBeenCalledWith(
'modals.rematchResults.undoFailed',
{ message: 'no snapshot' },
'error'
);
});
});