From 3468686e5f020d1e752a8fb777ba6c31055f3688 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Thu, 8 Oct 2026 07:47:54 +0800 Subject: [PATCH] feat(metadata): skip models on offline roots during bulk fetch (#1108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bulk metadata refresh picked candidates purely from cache state, so the entries the scoped scan deliberately keeps for unreachable drives were fetched anyway — burning CivitAI rate limit on sidecar/preview writes that cannot land. - Filter the fetch queue by root reachability (live check via describe_model_roots), covering both startup-offline and mid-session drive-off; probing failures degrade to no filtering - Report offline_skipped / offline_roots in the WS terminal payloads, the HTTP result and the completion message - Show a "Drive Offline" stat card in the fetch summary modal when any model was skipped this way --- locales/de.json | 1 + locales/en.json | 1 + locales/es.json | 1 + locales/fr.json | 1 + locales/he.json | 1 + locales/ja.json | 1 + locales/ko.json | 1 + locales/ru.json | 1 + locales/zh-CN.json | 1 + locales/zh-TW.json | 1 + .../bulk_metadata_refresh_use_case.py | 89 +++++++++++- .../components/metadata-refresh-result.css | 4 + static/js/api/baseModelApi.js | 11 +- .../test_bulk_metadata_refresh_use_case.py | 135 ++++++++++++++++++ 14 files changed, 243 insertions(+), 6 deletions(-) diff --git a/locales/de.json b/locales/de.json index 6bc5b3b5..a9f58673 100644 --- a/locales/de.json +++ b/locales/de.json @@ -2130,6 +2130,7 @@ "statSuccess": "Erfolgreich", "statFailed": "Fehlgeschlagen", "statSkipped": "Übersprungen", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "Gesamt geprüft", "statDuration": "Dauer", "successMessage": "Alle {count} {type}s erfolgreich aktualisiert!", diff --git a/locales/en.json b/locales/en.json index 998622fb..040a5f89 100644 --- a/locales/en.json +++ b/locales/en.json @@ -2130,6 +2130,7 @@ "statSuccess": "Success", "statFailed": "Failed", "statSkipped": "Skipped", + "statOffline": "Drive Offline", "statTotal": "Total Scanned", "statDuration": "Duration", "successMessage": "All {count} {type}s updated successfully!", diff --git a/locales/es.json b/locales/es.json index 58914a4a..c945eeab 100644 --- a/locales/es.json +++ b/locales/es.json @@ -2130,6 +2130,7 @@ "statSuccess": "Éxito", "statFailed": "Fallido", "statSkipped": "Omitido", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "Total escaneado", "statDuration": "Duración", "successMessage": "¡Todos los {count} {type}s actualizados correctamente!", diff --git a/locales/fr.json b/locales/fr.json index 9e905ba0..086e43d8 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -2130,6 +2130,7 @@ "statSuccess": "Réussi", "statFailed": "Échoué", "statSkipped": "Ignoré", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "Total scanné", "statDuration": "Durée", "successMessage": "Tous les {count} {type}s mis à jour avec succès !", diff --git a/locales/he.json b/locales/he.json index 94a0369a..eab34874 100644 --- a/locales/he.json +++ b/locales/he.json @@ -2130,6 +2130,7 @@ "statSuccess": "הצלחה", "statFailed": "נכשל", "statSkipped": "דולג", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "סה\"כ נסרק", "statDuration": "משך", "successMessage": "כל {count} {type}s עודכנו בהצלחה!", diff --git a/locales/ja.json b/locales/ja.json index 85062b69..1bd1efbf 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -2130,6 +2130,7 @@ "statSuccess": "成功", "statFailed": "失敗", "statSkipped": "スキップ", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "スキャン合計", "statDuration": "所要時間", "successMessage": "すべての{count}件の{type}を正常に更新しました", diff --git a/locales/ko.json b/locales/ko.json index 7f0d99d1..a6fdd6da 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -2130,6 +2130,7 @@ "statSuccess": "성공", "statFailed": "실패", "statSkipped": "건너뜀", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "총 스캔", "statDuration": "소요 시간", "successMessage": "모든 {count}개 {type}이(가) 성공적으로 업데이트되었습니다", diff --git a/locales/ru.json b/locales/ru.json index c9738f4f..73dbe8a6 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -2130,6 +2130,7 @@ "statSuccess": "Успешно", "statFailed": "Ошибка", "statSkipped": "Пропущено", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "Всего проверено", "statDuration": "Длительность", "successMessage": "Все {count} {type}s успешно обновлены", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index db20f658..bb44f0c7 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -2130,6 +2130,7 @@ "statSuccess": "成功", "statFailed": "失败", "statSkipped": "已跳过", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "总计扫描", "statDuration": "耗时", "successMessage": "全部 {count} 个 {type} 更新成功！", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 628a68ab..fd256914 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -2130,6 +2130,7 @@ "statSuccess": "成功", "statFailed": "失敗", "statSkipped": "已跳過", + "statOffline": "[TODO: Translate] Drive Offline", "statTotal": "總計掃描", "statDuration": "耗時", "successMessage": "全部 {count} 個 {type} 更新成功！", diff --git a/py/services/use_cases/bulk_metadata_refresh_use_case.py b/py/services/use_cases/bulk_metadata_refresh_use_case.py index 05346dd8..6223c7e6 100644 --- a/py/services/use_cases/bulk_metadata_refresh_use_case.py +++ b/py/services/use_cases/bulk_metadata_refresh_use_case.py @@ -7,6 +7,7 @@ import time from typing import Any, Dict, List, Optional, Protocol, Sequence from ..metadata_sync_service import MetadataSyncService +from ..model_scanner import _normalized_root_prefix, _path_matches_prefix from ..model_sources import has_external_source from ...utils.metadata_manager import MetadataManager @@ -46,7 +47,7 @@ class BulkMetadataRefreshUseCase: enable_metadata_archive_db = self._settings.get("enable_metadata_archive_db", False) skip_paths = self._settings.get("metadata_refresh_skip_paths", []) - to_process: Sequence[Dict[str, Any]] = [ + eligible: Sequence[Dict[str, Any]] = [ model for model in cache.raw_data if not model.get("skip_metadata_refresh", False) @@ -68,6 +69,26 @@ class BulkMetadataRefreshUseCase: ) ] + # Models on a drive that cannot be read right now stay in the cache + # (scoped scan keeps them), but fetching their metadata would only + # spend CivitAI rate limit on sidecar/preview writes that cannot land. + offline_roots = self._offline_root_details() + if offline_roots: + to_process = [ + model + for model in eligible + if not self._is_under_offline_root(model.get("file_path", ""), offline_roots) + ] + else: + to_process = list(eligible) + offline_skipped = len(eligible) - len(to_process) + if offline_skipped: + self._logger.info( + "Bulk metadata refresh: %d model(s) skipped, drive offline: %s", + offline_skipped, + ", ".join(root["label"] for root in offline_roots), + ) + total_to_process = len(to_process) initial_skipped = total_models - total_to_process # models excluded from fetch queue processed = 0 @@ -90,14 +111,21 @@ class BulkMetadataRefreshUseCase: "success": success, "failure_count": len(failures), "skipped_count": skipped_count, + "offline_skipped": offline_skipped, "handled": handled_count, "elapsed_seconds": int(time.monotonic() - start_time), } # Only include full failure details in terminal emits (completed, # cancelled, rate_limited) to avoid serializing the list on every # per-model progress update. - if failures and status in ("completed", "cancelled", "rate_limited"): - payload["failures"] = failures + if status in ("completed", "cancelled", "rate_limited"): + if failures: + payload["failures"] = failures + if offline_roots: + payload["offline_roots"] = [ + {"path": root["path"], "label": root["label"]} + for root in offline_roots + ] payload.update(extra) await progress_callback.on_progress(payload) @@ -110,7 +138,7 @@ class BulkMetadataRefreshUseCase: if self._service.scanner.is_cancelled(): self._logger.info("Bulk metadata refresh cancelled by user") await emit("cancelled", processed=processed, success=success) - return {"success": False, "message": "Operation cancelled", "processed": processed, "updated": success, "total": total_models, "failures": failures, "failure_count": len(failures), "skipped_count": skipped_count, "elapsed_seconds": int(time.monotonic() - start_time)} + return {"success": False, "message": "Operation cancelled", "processed": processed, "updated": success, "total": total_models, "failures": failures, "failure_count": len(failures), "skipped_count": skipped_count, "offline_skipped": offline_skipped, "elapsed_seconds": int(time.monotonic() - start_time)} try: original_name = model.get("model_name") @@ -202,6 +230,7 @@ class BulkMetadataRefreshUseCase: "failures": failures, "failure_count": len(failures), "skipped_count": skipped_count, + "offline_skipped": offline_skipped, "elapsed_seconds": int(time.monotonic() - start_time), } @@ -237,8 +266,58 @@ class BulkMetadataRefreshUseCase: "Successfully updated " f"{success} of {processed} processed {self._service.model_type}s (total: {total_models})" ) + if offline_skipped: + message += ( + f"; {offline_skipped} skipped, drive offline (" + + ", ".join(root["label"] for root in offline_roots) + + ")" + ) - return {"success": True, "message": message, "processed": processed, "updated": success, "total": total_models, "failures": failures, "failure_count": len(failures), "skipped_count": skipped_count, "elapsed_seconds": int(time.monotonic() - start_time)} + return {"success": True, "message": message, "processed": processed, "updated": success, "total": total_models, "failures": failures, "failure_count": len(failures), "skipped_count": skipped_count, "offline_skipped": offline_skipped, "elapsed_seconds": int(time.monotonic() - start_time)} + + def _offline_root_details(self) -> List[Dict[str, str]]: + """Configured roots whose directory cannot be read right now. + + Uses the scanner's root description (live ``os.path.exists`` per root), + so both a drive switched off at startup and one unplugged mid-session + are covered. Root probing must never break a fetch: any failure yields + an empty list, which disables the offline filter. + """ + describe = getattr(self._service.scanner, "describe_model_roots", None) + if describe is None: + return [] + try: + details = describe() + except Exception as exc: # pragma: no cover - defensive logging + self._logger.debug("Failed to describe model roots: %s", exc) + return [] + + offline: List[Dict[str, str]] = [] + for detail in details: + if not isinstance(detail, dict) or detail.get("reachable", True): + continue + path = detail.get("path") or "" + if not path: + continue + offline.append( + { + "path": path, + "label": detail.get("label") or path, + "prefix": _normalized_root_prefix(path), + } + ) + return offline + + @staticmethod + def _is_under_offline_root( + file_path: str, offline_roots: Sequence[Dict[str, str]] + ) -> bool: + """Match a cached business path against offline root prefixes.""" + if not file_path: + return False + return any( + _path_matches_prefix(file_path, root["prefix"]) for root in offline_roots + ) @staticmethod def _is_in_skip_path(folder: str, skip_paths: List[str]) -> bool: diff --git a/static/css/components/metadata-refresh-result.css b/static/css/components/metadata-refresh-result.css index 6bfcf6ad..b33bf85d 100644 --- a/static/css/components/metadata-refresh-result.css +++ b/static/css/components/metadata-refresh-result.css @@ -55,6 +55,10 @@ border-left-color: var(--color-warning); } +.stat-card-offline { + border-left-color: var(--text-secondary); +} + .stat-card-total { border-left-color: var(--lora-border); } diff --git a/static/js/api/baseModelApi.js b/static/js/api/baseModelApi.js index f7439cb0..2b7ea64f 100644 --- a/static/js/api/baseModelApi.js +++ b/static/js/api/baseModelApi.js @@ -898,7 +898,9 @@ export class BaseModelApiClient { } _showFailureDetailsModal(data) { - const { failures = [], success, processed, total, failure_count, skipped_count, elapsed_seconds } = data; + const { failures = [], success, processed, total, failure_count, skipped_count, elapsed_seconds, offline_skipped = 0, offline_roots = [] } = data; + + const offlineRootNames = offline_roots.map((root) => root.label || root.path || String(root)).join(', '); // Build failure list HTML const failureRows = failures.map((f, i) => @@ -935,6 +937,13 @@ export class BaseModelApiClient { ${skipped_count} + ${offline_skipped > 0 ? ` +
+
+ ${translate('modals.metadataFetchSummary.statOffline', {}, 'Drive Offline')} + ${offline_skipped} +
+
` : ''}
${translate('modals.metadataFetchSummary.statTotal', {}, 'Total Scanned')} diff --git a/tests/services/use_cases/test_bulk_metadata_refresh_use_case.py b/tests/services/use_cases/test_bulk_metadata_refresh_use_case.py index b0feae76..889bc886 100644 --- a/tests/services/use_cases/test_bulk_metadata_refresh_use_case.py +++ b/tests/services/use_cases/test_bulk_metadata_refresh_use_case.py @@ -402,3 +402,138 @@ async def test_model_without_hash_skipped(use_case, mock_service, mock_metadata_ assert result["processed"] == 1 assert result["updated"] == 0 + + +def _offline_scanner_mock(scanner): + """Point the scanner mock at one offline and one online root.""" + scanner.describe_model_roots = MagicMock( + return_value=[ + {"path": "/mnt/offline_loras", "label": "offline_loras", "reachable": False, "available": False, "models": 1}, + {"path": "/models", "label": "models", "reachable": True, "available": True, "models": 1}, + ] + ) + + +@pytest.mark.asyncio +@patch.object(metadata_manager.MetadataManager, "hydrate_model_data") +async def test_fetch_skips_models_on_offline_root(mock_hydrate, use_case, mock_service, mock_metadata_sync): + """Models under an unreachable root are excluded from the fetch queue.""" + mock_hydrate.return_value = None + _offline_scanner_mock(mock_service.scanner) + + offline_model = { + "file_path": "/mnt/offline_loras/model.safetensors", + "sha256": "hash_offline", + "model_name": "Offline Model", + "civitai": {}, + "from_civitai": False, + "civitai_deleted": False, + } + online_model = { + "file_path": "/models/model.safetensors", + "sha256": "hash_online", + "model_name": "Online Model", + "civitai": {}, + "from_civitai": False, + "civitai_deleted": False, + } + + cache = SimpleNamespace(raw_data=[offline_model, online_model], resort=AsyncMock()) + mock_service.scanner.get_cached_data.return_value = cache + + reporter = MockProgressReporter() + result = await use_case.execute(progress_callback=reporter) + + # Only the online model is fetched + mock_metadata_sync.fetch_and_update_model.assert_called_once() + call_args = mock_metadata_sync.fetch_and_update_model.call_args[1] + assert call_args["file_path"] == "/models/model.safetensors" + + assert result["success"] is True + assert result["processed"] == 1 + assert result["offline_skipped"] == 1 + # The offline model still counts as excluded from the queue + assert result["skipped_count"] == 1 + + completed_calls = [c for c in reporter.progress_calls if c["status"] == "completed"] + assert completed_calls[0]["offline_skipped"] == 1 + assert completed_calls[0]["offline_roots"] == [ + {"path": "/mnt/offline_loras", "label": "offline_loras"} + ] + + +@pytest.mark.asyncio +@patch.object(metadata_manager.MetadataManager, "hydrate_model_data") +async def test_fetch_offline_filter_only_catches_offline_prefix(mock_hydrate, use_case, mock_service, mock_metadata_sync): + """A sibling path that merely shares a prefix string is not excluded.""" + mock_hydrate.return_value = None + _offline_scanner_mock(mock_service.scanner) + + # "/mnt/offline_loras_backup/..." must NOT match the "/mnt/offline_loras/" prefix + sibling_model = { + "file_path": "/mnt/offline_loras_backup/model.safetensors", + "sha256": "hash_sibling", + "model_name": "Sibling Model", + "civitai": {}, + "from_civitai": False, + "civitai_deleted": False, + } + + cache = SimpleNamespace(raw_data=[sibling_model], resort=AsyncMock()) + mock_service.scanner.get_cached_data.return_value = cache + + result = await use_case.execute() + + mock_metadata_sync.fetch_and_update_model.assert_called_once() + assert result["offline_skipped"] == 0 + + +@pytest.mark.asyncio +@patch.object(metadata_manager.MetadataManager, "hydrate_model_data") +async def test_fetch_proceeds_unfiltered_without_root_description(mock_hydrate, use_case, mock_service, mock_metadata_sync): + """Scanners without describe_model_roots keep the previous behaviour.""" + mock_hydrate.return_value = None + del mock_service.scanner.describe_model_roots + + model = { + "file_path": "/mnt/offline_loras/model.safetensors", + "sha256": "hash123", + "model_name": "Test Model", + "civitai": {}, + "from_civitai": False, + "civitai_deleted": False, + } + + cache = SimpleNamespace(raw_data=[model], resort=AsyncMock()) + mock_service.scanner.get_cached_data.return_value = cache + + result = await use_case.execute() + + mock_metadata_sync.fetch_and_update_model.assert_called_once() + assert result["offline_skipped"] == 0 + + +@pytest.mark.asyncio +@patch.object(metadata_manager.MetadataManager, "hydrate_model_data") +async def test_fetch_proceeds_unfiltered_when_root_description_fails(mock_hydrate, use_case, mock_service, mock_metadata_sync): + """A failing describe_model_roots must not break the fetch.""" + mock_hydrate.return_value = None + mock_service.scanner.describe_model_roots = MagicMock(side_effect=RuntimeError("boom")) + + model = { + "file_path": "/models/model.safetensors", + "sha256": "hash123", + "model_name": "Test Model", + "civitai": {}, + "from_civitai": False, + "civitai_deleted": False, + } + + cache = SimpleNamespace(raw_data=[model], resort=AsyncMock()) + mock_service.scanner.get_cached_data.return_value = cache + + result = await use_case.execute() + + mock_metadata_sync.fetch_and_update_model.assert_called_once() + assert result["success"] is True + assert result["offline_skipped"] == 0