feat(scanner): scan a single root and keep unreachable entries (#1108)

Refreshing had no way to say "scan only this drive": a user with three external
drives had to spin all of them up for every refresh, and switching a drive off
made the next refresh treat its whole library as deleted (rows pruned from the
memory cache and the SQLite cache, preview_url stripped on the next scroll).

Backend (py/services/model_scanner.py, py/config.py):
* ReconcileScope(roots, folder) + _reconcile_cache(scope=...): files inside the
  scope reconcile normally, everything outside is neither re-read nor removed.
  The folder half is plumbing for the sidebar entry in the next change.
* Path-level pruning guard: cached entries under a path this walk could not
  read are kept and reported instead of removed. Sources: a configured root that
  is not reachable (drive switched off while LM runs), a directory os.walk
  failed to enter (permissions / I/O error / Windows junction to an offline
  drive), and a known first-level symlink whose target is gone
  (Config.iter_path_mappings()).
* The recorded folder list is unioned instead of replaced whenever the scan did
  not verify every root, so a scoped scan cannot empty the sidebar.
* _reconcile_cache returns a summary (added / removed / repaired /
  scanned_roots / skipped_roots / unavailable_paths / kept_unreachable),
  exposed as ModelScanner.last_reconcile_summary, returned by
  BaseModelService.scan_models() and broadcast in the completed WS payload.
* _root_display_labels(): set-aware labels ("G: loras", "usb/loras") grown
  leftwards with real parent segments until unique, shared by the walk-progress
  line and the roots API.
* GET /scan accepts repeated `roots` (400 for unknown roots, 400 combined with
  full_rebuild=true); GET /roots gains root_details (label / reachable / cached
  count) while `roots` stays a plain path list for existing callers.
* serve_preview: a 404 no longer clears the cached preview_url when the file's
  own directory is unreachable - browsing the grid with a drive off used to
  strip preview references from the persistent cache.

Frontend:
* Refresh ▾ gains a "Scan one folder" section listing the page's roots with
  their cached counts; offline roots stay clickable and explain themselves; rows
  are wired by delegation (new static/js/components/controls/ScanScopeMenu.js).
* A scoped scan reports "Scanned <root>: N new, M removed"; a scan that kept
  entries reports "<N> models kept: <paths> not reachable".
* registerAPI() now injects the two cross-page passthroughs (fetchModelRoots and
  an argument-forwarding refreshModels) so a page facade cannot drop them: the
  first version rendered an empty menu and would have run a full refresh.
* createToastElement whitelists toast types, so a wrong `type` argument degrades
  to the info style instead of rendering an unstyled box.

Verified in a sandbox instance with three roots: a scoped scan walks only the
requested root (progress roots=0/1, 240 files); a full refresh with one root
offline reports kept_unreachable=60 and leaves all 420 models cached; /roots
reports the offline root with its cached count. 3699 passed, 7 skipped;
frontend 1488 passed (148 files); vue widgets 96 passed.
This commit is contained in:
Will Miao
2026-10-07 15:01:26 +08:00
parent 2dcaf6a30e
commit 470d85cca6
34 changed files with 1981 additions and 106 deletions
+59 -2
View File
@@ -511,8 +511,9 @@ export class BaseModelApiClient {
}
}
async refreshModels(fullRebuild = false) {
async refreshModels(fullRebuild = false, { roots = null } = {}) {
const abortController = new AbortController();
const scopeRoots = Array.isArray(roots) ? roots.filter(Boolean) : [];
const displayName = this.apiConfig.config.displayName;
const singularName = this.apiConfig.config.singularName;
const actionText = translate(
@@ -600,6 +601,9 @@ export class BaseModelApiClient {
const url = new URL(this.apiConfig.endpoints.scan, window.location.origin);
url.searchParams.append('full_rebuild', fullRebuild);
for (const root of scopeRoots) {
url.searchParams.append('roots', root);
}
const response = await fetch(url, { signal: abortController.signal });
@@ -615,7 +619,7 @@ export class BaseModelApiClient {
resetAndReload(true);
showToast('toast.api.refreshComplete', { action: actionText }, 'success');
this._showRefreshSummary(data, actionText, scopeRoots);
} catch (error) {
if (error.name === 'AbortError') {
showToast('toast.api.operationCancelled', {}, 'info');
@@ -632,6 +636,59 @@ export class BaseModelApiClient {
}
}
/**
* Report what a finished scan did.
*
* A scoped scan names the folder it scanned and how many models changed; a
* scan that could not read part of its scope says so instead of silently
* looking like "nothing found" (the entries are kept, not deleted).
* @param {Object} summary - Scan response payload
* @param {string} actionText - Localized "Refresh" / "Full rebuild"
* @param {Array<string>} scopeRoots - Roots the scan was restricted to
*/
_showRefreshSummary(summary, actionText, scopeRoots = []) {
const payload = summary || {};
const scannedRoots = Array.isArray(payload.scanned_roots) ? payload.scanned_roots : [];
const scopeLabel = scannedRoots.length ? scannedRoots.join(', ') : '';
if (scopeRoots.length && scopeLabel) {
showToast(
'toast.api.refreshCompleteScoped',
{
scope: scopeLabel,
added: Number(payload.added || 0),
removed: Number(payload.removed || 0),
},
'success'
);
} else {
showToast('toast.api.refreshComplete', { action: actionText }, 'success');
}
const keptCount = Number(payload.kept_unreachable || 0);
if (keptCount > 0) {
const unavailable = Array.isArray(payload.unavailable_paths)
? payload.unavailable_paths
: [];
const skipped = Array.isArray(payload.skipped_roots) ? payload.skipped_roots : [];
// Skipped roots already carry a short label; the remaining
// unreadable folders only have a path.
const names = (
skipped.length
? skipped.map(entry => entry?.label || entry?.path)
: unavailable.map(entry => entry?.path)
).filter(Boolean);
showToast(
'toast.api.refreshKeptUnreachable',
{
count: keptCount.toLocaleString(),
paths: names.slice(0, 3).join(', ') || '—',
},
'info'
);
}
}
/**
* Connect to the shared fetch-progress WebSocket for scan progress updates.
* Returns null when the connection cannot be established (silent fallback).
@@ -33,9 +33,6 @@ export class CheckpointsControls extends PageControls {
return await resetAndReload(updateFolders);
},
refreshModels: async (fullRebuild = false) => {
return await getModelApiClient().refreshModels(fullRebuild);
},
// Add fetch from Civitai functionality for checkpoints
fetchFromCivitai: async () => {
@@ -30,9 +30,6 @@ export class EmbeddingsControls extends PageControls {
return await resetAndReload(updateFolders);
},
refreshModels: async (fullRebuild = false) => {
return await getModelApiClient().refreshModels(fullRebuild);
},
// Add fetch from Civitai functionality for embeddings
fetchFromCivitai: async () => {
@@ -37,9 +37,6 @@ export class LorasControls extends PageControls {
return await resetAndReload(updateFolders);
},
refreshModels: async (fullRebuild = false) => {
return await getModelApiClient().refreshModels(fullRebuild);
},
// LoRA-specific API functions
fetchFromCivitai: async () => {
@@ -31,9 +31,6 @@ export class OtherControls extends PageControls {
return await resetAndReload(updateFolders);
},
refreshModels: async (fullRebuild = false) => {
return await getModelApiClient().refreshModels(fullRebuild);
},
// Add fetch from Civitai functionality for other models
fetchFromCivitai: async () => {
+93 -4
View File
@@ -5,7 +5,9 @@ import { showToast, openCivitaiByMetadata, isTypingContext } from '../../utils/u
import { eventManager } from '../../utils/EventManager.js';
import { performModelUpdateCheck } from '../../utils/updateCheckHelpers.js';
import { sidebarManager } from '../SidebarManager.js';
import { getModelApiClient } from '../../api/modelApiFactory.js';
import { initSortDropdown, applySortToSelect, randomizeSortValue } from './SortDropdown.js';
import { renderScanScopeMenu, resolveScanScopeTarget } from './ScanScopeMenu.js';
/**
* PageControls class - Unified control management for model pages
@@ -82,7 +84,18 @@ export class PageControls {
* @param {Object} api - API methods for the page
*/
registerAPI(api) {
this.api = api;
this.api = {
...api,
// These two are identical for every page and must stay in sync with
// the scan API: the refresh scope menu reads the root list, and a
// scoped refresh has to pass `roots` through. They are defined here
// rather than in each page facade so a page cannot silently drop
// them (which is exactly how the scope menu ended up empty).
fetchModelRoots: async () => getModelApiClient().fetchModelRoots(),
// Forward the exact arguments: an unscoped refresh must keep calling
// the client with a single argument.
refreshModels: async (...args) => getModelApiClient().refreshModels(...args),
};
console.log(`API methods registered for ${this.pageType} page`);
// Initialize sidebar manager after API is registered
@@ -233,7 +246,10 @@ export class PageControls {
});
// Toggle current dropdown
dropdownGroup.classList.toggle('active');
const isOpen = dropdownGroup.classList.toggle('active');
if (isOpen) {
this.loadScanScopeMenu();
}
});
});
@@ -248,6 +264,21 @@ export class PageControls {
});
}
// Per-root scan rows are rendered dynamically, so they are wired by
// delegation instead of a direct querySelector per item.
const scopeMenu = document.getElementById('refreshScopeMenu');
if (scopeMenu) {
scopeMenu.addEventListener('click', (e) => {
const item = e.target.closest('[data-action="scan-root"]');
if (!item) {
return;
}
e.stopPropagation();
this.handleScanRootSelection(item);
document.querySelector('.dropdown-group.active')?.classList.remove('active');
});
}
const checkUpdatesOption = document.getElementById('checkUpdatesMenuItem');
if (checkUpdatesOption) {
checkUpdatesOption.addEventListener('click', async (e) => {
@@ -266,6 +297,55 @@ export class PageControls {
});
}
/**
* Render the per-root scan entries of the refresh dropdown.
*
* Roots come from the backend (label, model count, reachability) so the menu
* always names a root exactly like the scan progress line does. The list is
* cached until the next scan finishes.
*/
async loadScanScopeMenu() {
const menu = document.getElementById('refreshScopeMenu');
if (!menu || typeof this.api?.fetchModelRoots !== 'function') {
return;
}
if (this._scanScopeDetails) {
this.renderScanScopeMenu(this._scanScopeDetails);
return;
}
if (this._scanScopeLoading) {
return;
}
this._scanScopeLoading = true;
try {
const payload = await this.api.fetchModelRoots();
const details = Array.isArray(payload?.root_details) ? payload.root_details : [];
this._scanScopeDetails = details;
renderScanScopeMenu(menu, details);
} catch (error) {
console.error('Error loading model roots:', error);
menu.innerHTML = '';
} finally {
this._scanScopeLoading = false;
}
}
handleScanRootSelection(item) {
const { rootPath, label, offline } = resolveScanScopeTarget(item);
if (!rootPath) {
return;
}
if (offline) {
// showToast(key, params, type): the sentence must NOT be passed here,
// it used to land in the `type` slot and produced an unstyled toast.
showToast('toast.api.scanRootUnreachable', { scope: label }, 'info');
return;
}
this.refreshModels(false, { roots: [rootPath] });
}
async handleCheckModelUpdates(menuItem) {
if (this._updateCheckInProgress) {
return;
@@ -482,14 +562,23 @@ export class PageControls {
* Refresh models list
* @param {boolean} fullRebuild - Whether to perform a full rebuild
*/
async refreshModels(fullRebuild = false) {
async refreshModels(fullRebuild = false, { roots = null } = {}) {
if (!this.api) {
console.error('API methods not registered');
return;
}
// Root labels / reachability may have changed with the scan, and the
// model counts definitely did.
this._scanScopeDetails = null;
const scopedRoots = Array.isArray(roots) ? roots.filter(Boolean) : [];
try {
await this.api.refreshModels(fullRebuild);
if (scopedRoots.length) {
await this.api.refreshModels(fullRebuild, { roots: scopedRoots });
} else {
await this.api.refreshModels(fullRebuild);
}
// Refresh sidebar after rebuild
if (this.sidebarManager) {
@@ -0,0 +1,72 @@
// ScanScopeMenu.js - per-root entries of the Refresh dropdown (issue #1108)
import { translate } from '../../utils/i18nHelpers.js';
/**
* Render one row per configured model root.
*
* Rows carry the root path in `data-root` and are wired by delegation, because
* the list is rebuilt every time the root set changes. An offline root stays
* clickable (the click explains why nothing can be scanned) and is only
* de-emphasised visually.
*
* @param {HTMLElement|null} menu - Container element (#refreshScopeMenu)
* @param {Array<Object>} details - `root_details` from GET /api/lm/{type}/roots
*/
export function renderScanScopeMenu(menu, details = []) {
if (!menu) {
return;
}
menu.innerHTML = '';
details.forEach(detail => {
if (!detail || !detail.path) {
return;
}
const item = document.createElement('div');
item.className = `dropdown-item scan-root-item${detail.reachable ? '' : ' is-offline'}`;
item.dataset.action = 'scan-root';
item.dataset.root = detail.path;
item.title = detail.path;
const icon = document.createElement('i');
icon.className = `fas ${detail.reachable ? 'fa-folder-open' : 'fa-plug-circle-xmark'}`;
item.appendChild(icon);
const label = document.createElement('span');
label.className = 'scan-root-label';
label.textContent = detail.label || detail.path;
item.appendChild(label);
const meta = document.createElement('span');
if (detail.reachable) {
const count = Number(detail.models || 0).toLocaleString();
meta.className = 'scan-root-count';
meta.textContent = translate(
'loras.controls.refresh.rootModels',
{ count },
`${count} models`
);
} else {
meta.className = 'scan-root-offline';
meta.textContent = translate('loras.controls.refresh.rootOffline', {}, 'Offline');
}
item.appendChild(meta);
menu.appendChild(item);
});
}
/**
* Read what a clicked row refers to.
* @param {HTMLElement} item - Row rendered by renderScanScopeMenu
* @returns {{rootPath: string, label: string, offline: boolean}}
*/
export function resolveScanScopeTarget(item) {
const rootPath = item?.dataset?.root || '';
const label = item?.querySelector?.('.scan-root-label')?.textContent || rootPath;
return {
rootPath,
label,
offline: Boolean(item?.classList?.contains('is-offline')),
};
}
+6 -1
View File
@@ -139,9 +139,14 @@ export async function copyToClipboard(text, successMessage = null) {
* @param {string} type - Toast type (info/success/warning/error)
* @returns {HTMLElement} The toast element (not yet attached to the DOM)
*/
// Only these carry styling (icon + accent border). An unknown type is coerced
// to `info` instead of ending up as a bare `.toast` box — a wrong argument
// order at a call site used to leak the message itself into the class list.
const TOAST_TYPES = new Set(['info', 'success', 'warning', 'error', 'copy']);
function createToastElement(message, type) {
const toast = document.createElement('div');
toast.className = `toast toast-${type}`;
toast.className = `toast toast-${TOAST_TYPES.has(type) ? type : 'info'}`;
toast.textContent = message;
return toast;
}