fix(example-images): use in-place cache sync and bulk pending-check index for large libraries

tests/services/test_check_pending_models.py

@@ -363,6 +363,148 @@ async def test_check_pending_models_handles_corrupted_progress_file(
     assert result["pending_count"] == 1
 
 
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("tmp_path")
+async def test_check_pending_models_uses_bulk_folder_index_for_large_libraries(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path,
+    settings_manager,
+):
+    """For >1000 candidates the pre-check scans the library root once instead of
+    probing every folder individually."""
+
+    ws_manager = RecordingWebSocketManager()
+    manager = download_module.DownloadManager(ws_manager=ws_manager)
+
+    monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
+
+    # 1500 unprocessed models triggers the bulk lookup path
+    models = [
+        {"sha256": f"{i:064x}", "model_name": f"Model {i}"}
+        for i in range(1500)
+    ]
+
+    # Create folders with files for the first 500 models
+    for i in range(500):
+        model_dir = tmp_path / f"{i:064x}"
+        model_dir.mkdir()
+        (model_dir / "image_0.png").write_text("data")
+
+    _patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
+
+    per_model_checks = 0
+
+    def counting_model_directory_has_files(path: str) -> bool:
+        nonlocal per_model_checks
+        per_model_checks += 1
+        return False
+
+    monkeypatch.setattr(
+        download_module,
+        "_model_directory_has_files",
+        counting_model_directory_has_files,
+    )
+
+    result = await manager.check_pending_models(["lora"])
+
+    assert result["success"] is True
+    assert result["total_models"] == 1500
+    assert result["pending_count"] == 1000
+    assert result["needs_download"] is True
+    # The per-folder check should not be used once we cross the threshold.
+    assert per_model_checks == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("tmp_path")
+async def test_check_pending_models_uses_per_folder_check_for_small_candidate_sets(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path,
+    settings_manager,
+):
+    """For <=1000 candidates the pre-check keeps the accurate per-folder path."""
+
+    ws_manager = RecordingWebSocketManager()
+    manager = download_module.DownloadManager(ws_manager=ws_manager)
+
+    monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
+
+    models = [
+        {"sha256": f"{i:064x}", "model_name": f"Model {i}"}
+        for i in range(500)
+    ]
+
+    # Create folders with files for the first 200 models
+    for i in range(200):
+        model_dir = tmp_path / f"{i:064x}"
+        model_dir.mkdir()
+        (model_dir / "image_0.png").write_text("data")
+
+    _patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
+
+    per_model_checks = 0
+    original_has_files = download_module._model_directory_has_files
+
+    def counting_model_directory_has_files(path: str) -> bool:
+        nonlocal per_model_checks
+        per_model_checks += 1
+        return original_has_files(path)
+
+    monkeypatch.setattr(
+        download_module,
+        "_model_directory_has_files",
+        counting_model_directory_has_files,
+    )
+
+    result = await manager.check_pending_models(["lora"])
+
+    assert result["success"] is True
+    assert result["total_models"] == 500
+    assert result["pending_count"] == 300
+    assert result["needs_download"] is True
+    # Per-folder path should run once per candidate.
+    assert per_model_checks == 500
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("tmp_path")
+async def test_check_pending_models_bulk_index_includes_legacy_folders(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path,
+    settings_manager,
+):
+    """In multi-library mode the bulk index also scans the legacy root so models
+    whose folders have not been consolidated yet are not reported pending."""
+
+    ws_manager = RecordingWebSocketManager()
+    manager = download_module.DownloadManager(ws_manager=ws_manager)
+
+    monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
+    monkeypatch.setitem(settings_manager.settings, "libraries", {"default": {}, "extra": {}})
+    monkeypatch.setitem(settings_manager.settings, "active_library", "extra")
+
+    # 1500 unprocessed models triggers the bulk lookup path
+    models = [
+        {"sha256": f"{i:064x}", "model_name": f"Model {i}"}
+        for i in range(1500)
+    ]
+
+    # Folders live at the LEGACY root/<hash> path (not yet consolidated)
+    for i in range(500):
+        model_dir = tmp_path / f"{i:064x}"
+        model_dir.mkdir()
+        (model_dir / "image_0.png").write_text("data")
+
+    _patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
+
+    result = await manager.check_pending_models(["lora"])
+
+    assert result["success"] is True
+    assert result["total_models"] == 1500
+    assert result["pending_count"] == 1000
+    assert result["needs_download"] is True
+
+
 @pytest.fixture
 def settings_manager():
     return get_settings_manager()

tests/services/test_example_images_download_manager_async.py

@@ -26,6 +26,7 @@ class StubScanner:
 
     def __init__(self, models: list[dict]) -> None:
         self._cache = SimpleNamespace(raw_data=models)
+        self.sync_calls: list[tuple[str, dict]] = []
 
     async def get_cached_data(self):
         return self._cache
@@ -38,6 +39,14 @@ class StubScanner:
                 break
         return True
 
+    async def sync_cache_from_metadata(self, file_path: str, metadata: dict) -> bool:
+        self.sync_calls.append((file_path, metadata))
+        for index, model in enumerate(self._cache.raw_data):
+            if model.get("file_path") == metadata.get("file_path"):
+                self._cache.raw_data[index] = metadata
+                break
+        return True
+
 
 def _patch_scanner(monkeypatch: pytest.MonkeyPatch, scanner: StubScanner) -> None:
     async def _get_lora_scanner(cls):
@@ -588,6 +597,9 @@ async def test_not_found_example_images_are_cleaned(
     assert missing_url in downloader.calls
     assert manager._progress["failed_models"] == {model_hash}
     assert model_hash in manager._progress["processed_models"]
+    assert scanner.sync_calls
+    assert len(scanner.sync_calls) == 1
+    assert scanner.sync_calls[0][0] == str(model_path)
 
     remaining_images = model_metadata["civitai"]["images"]
     assert remaining_images == [

tests/utils/test_example_images_metadata.py

@@ -15,6 +15,7 @@ class StubScanner:
     def __init__(self, cache_items: List[Dict[str, Any]]) -> None:
         self.cache = SimpleNamespace(raw_data=cache_items)
         self.updates: List[Tuple[str, str, Dict[str, Any]]] = []
+        self.sync_updates: List[Tuple[str, Dict[str, Any]]] = []
 
     async def get_cached_data(self):
         return self.cache
@@ -23,6 +24,10 @@ class StubScanner:
         self.updates.append((old_path, new_path, metadata))
         return True
 
+    async def sync_cache_from_metadata(self, file_path: str, metadata: Dict[str, Any]) -> bool:
+        self.sync_updates.append((file_path, metadata))
+        return True
+
 
 @pytest.fixture(autouse=True)
 def patch_metadata_manager(monkeypatch: pytest.MonkeyPatch):
@@ -83,7 +88,7 @@ async def test_update_metadata_after_import_enriches_entries(monkeypatch: pytest
     assert custom[0]["type"] == "image"
 
     assert Path(patch_metadata_manager[0][0]) == model_file
-    assert scanner.updates
+    assert scanner.sync_updates
 
 
 @pytest.mark.asyncio
@@ -151,8 +156,8 @@ async def test_update_metadata_after_import_preserves_existing_metadata(
     assert saved_payload["civitai"]["trainedWords"] == ["foo"]
     assert {entry["id"] for entry in saved_payload["civitai"]["customImages"]} == {"existing-id", "new-id"}
 
-    assert scanner.updates
-    updated_metadata = scanner.updates[-1][2]
+    assert scanner.sync_updates
+    updated_metadata = scanner.sync_updates[-1][1]
     assert updated_metadata["civitai"]["images"] == existing_payload["civitai"]["images"]
     assert {entry["id"] for entry in updated_metadata["civitai"]["customImages"]} == {"existing-id", "new-id"}