diff --git a/py/services/persistent_recipe_cache.py b/py/services/persistent_recipe_cache.py index f035bd25..93979c5c 100644 --- a/py/services/persistent_recipe_cache.py +++ b/py/services/persistent_recipe_cache.py @@ -333,6 +333,44 @@ class PersistentRecipeCache: except Exception as exc: logger.debug("Failed to persist image_id_map: %s", exc) + def get_metadata_value(self, key: str) -> Optional[str]: + """Return a value from cache_metadata, or None if missing.""" + if not self.is_enabled() or not self._schema_initialized: + return None + + try: + with self._db_lock: + conn = self._connect(readonly=True) + try: + row = conn.execute( + "SELECT value FROM cache_metadata WHERE key = ?", + (key,), + ).fetchone() + return row["value"] if row else None + finally: + conn.close() + except Exception: + return None + + def set_metadata_value(self, key: str, value: str) -> None: + """Store a value in cache_metadata without rewriting the full cache.""" + if not self.is_enabled() or not self._schema_initialized: + return + + try: + with self._db_lock: + conn = self._connect() + try: + conn.execute( + "INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)", + (key, value), + ) + conn.commit() + finally: + conn.close() + except Exception as exc: + logger.debug("Failed to persist cache metadata %s: %s", key, exc) + def get_indexed_recipe_ids(self) -> Set[str]: """Return all recipe IDs in the cache. diff --git a/py/services/recipe_scanner.py b/py/services/recipe_scanner.py index 57cbf576..53e520f7 100644 --- a/py/services/recipe_scanner.py +++ b/py/services/recipe_scanner.py @@ -1547,7 +1547,7 @@ class RecipeScanner: self._cache.raw_data = recipes self._update_folder_metadata(self._cache) self._sort_cache_sync() - # Backfill source_path from JSON files if missing (schema migration) + # Backfill source_path from JSON files if missing (one-shot schema migration) if self._backfill_source_path_if_needed(recipes, json_paths): self._cache.image_id_map = self._build_image_id_map() self._persistent_cache.save_cache( @@ -1574,7 +1574,7 @@ class RecipeScanner: self._cache.raw_data = recipes self._update_folder_metadata(self._cache) self._sort_cache_sync() - # Backfill source_path from JSON files if missing (schema migration) + # Backfill source_path from JSON files if missing (one-shot schema migration) self._backfill_source_path_if_needed(recipes, json_paths) self._cache.image_id_map = self._build_image_id_map() # Persist updated cache @@ -1711,6 +1711,9 @@ class RecipeScanner: return recipes, changed, json_paths + # Metadata key recording that the one-shot source_path backfill has run. + _SOURCE_PATH_BACKFILL_MARKER = "source_path_backfilled" + def _backfill_source_path_if_needed( self, recipes: List[Dict[str, Any]], @@ -1718,8 +1721,21 @@ class RecipeScanner: ) -> bool: """Backfill source_path from recipe JSON files if missing from cache. + This is a one-shot schema migration: once it has run, a completion + marker is stored in the persistent cache metadata and later startups + skip it entirely. Recipes without a source_path in their JSON file + would otherwise be re-read and re-parsed on every startup. New or + changed recipe files still get source_path from the normal parse path + during reconciliation. + Returns True if any recipes were updated (caller should persist cache). """ + cache = self._persistent_cache + if ( + cache is not None + and cache.get_metadata_value(self._SOURCE_PATH_BACKFILL_MARKER) == "1" + ): + return False updated = False for recipe in recipes: if recipe.get("source_path"): @@ -1737,6 +1753,8 @@ class RecipeScanner: updated = True except Exception: pass + if cache is not None: + cache.set_metadata_value(self._SOURCE_PATH_BACKFILL_MARKER, "1") return updated def _full_directory_scan_sync( diff --git a/tests/services/test_recipe_scanner_source_path_backfill.py b/tests/services/test_recipe_scanner_source_path_backfill.py new file mode 100644 index 00000000..aaa70862 --- /dev/null +++ b/tests/services/test_recipe_scanner_source_path_backfill.py @@ -0,0 +1,104 @@ +"""Tests for the one-shot source_path backfill in RecipeScanner.""" + +import json +from pathlib import Path + +import pytest + +from py.services.persistent_recipe_cache import PersistentRecipeCache +from py.services.recipe_scanner import RecipeScanner + + +@pytest.fixture +def scanner_with_cache(tmp_path: Path): + """RecipeScanner instance backed by a real persistent cache in tmp_path.""" + cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite")) + scanner = RecipeScanner.__new__(RecipeScanner) + scanner._persistent_cache = cache + return scanner, cache + + +def _write_recipe_json(path: Path, recipe_id: str, source_path: str | None = None) -> None: + data = {"id": recipe_id} + if source_path is not None: + data["source_path"] = source_path + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_backfill_processes_recipes_when_marker_absent(scanner_with_cache, tmp_path: Path): + """Without the completion marker, missing source_path values are backfilled.""" + scanner, cache = scanner_with_cache + + json_with_source = tmp_path / "r1.recipe.json" + _write_recipe_json(json_with_source, "r1", source_path="https://civitai.com/images/1") + json_without_source = tmp_path / "r2.recipe.json" + _write_recipe_json(json_without_source, "r2") + + recipes = [ + {"id": "r1", "source_path": ""}, + {"id": "r2", "source_path": ""}, + {"id": "r3", "source_path": "https://civitai.com/images/3"}, + ] + json_paths = {"r1": str(json_with_source), "r2": str(json_without_source)} + + updated = scanner._backfill_source_path_if_needed(recipes, json_paths) + + assert updated is True + assert recipes[0]["source_path"] == "https://civitai.com/images/1" + # JSON legitimately has no source_path: stays empty, no error + assert recipes[1]["source_path"] == "" + assert recipes[2]["source_path"] == "https://civitai.com/images/3" + # The run records the completion marker + assert ( + cache.get_metadata_value(RecipeScanner._SOURCE_PATH_BACKFILL_MARKER) == "1" + ) + + +def test_backfill_is_skipped_once_marker_is_set(scanner_with_cache, tmp_path: Path, monkeypatch): + """The second initialization must not re-read recipe JSON files.""" + scanner, cache = scanner_with_cache + + json_path = tmp_path / "r1.recipe.json" + _write_recipe_json(json_path, "r1", source_path="https://civitai.com/images/1") + + recipes = [{"id": "r1", "source_path": ""}] + json_paths = {"r1": str(json_path)} + + # First run: backfills and records the marker + assert scanner._backfill_source_path_if_needed(recipes, json_paths) is True + assert recipes[0]["source_path"] == "https://civitai.com/images/1" + + # Second run: simulate a fresh startup where the cache still lacks + # source_path. Under the old behavior the file would be re-read and + # re-parsed; now the marker must suppress any file access. + recipes = [{"id": "r1", "source_path": ""}] + + import os + + real_exists = os.path.exists + + def _failing_exists(path): + if str(path).endswith(".recipe.json"): + raise AssertionError("backfill touched the filesystem despite marker") + return real_exists(path) + + monkeypatch.setattr(os.path, "exists", _failing_exists) + + updated = scanner._backfill_source_path_if_needed(recipes, json_paths) + + assert updated is False + assert recipes[0]["source_path"] == "" + + +def test_backfill_marker_does_not_suppress_reconcile_parsed_source_path(scanner_with_cache): + """The marker only gates the backfill; parsed recipes keep their source_path.""" + scanner, cache = scanner_with_cache + cache.set_metadata_value(RecipeScanner._SOURCE_PATH_BACKFILL_MARKER, "1") + + # A recipe that arrived from the normal parse path with a source_path is + # left untouched by the (skipped) backfill. + recipes = [{"id": "r1", "source_path": "https://civitai.com/images/9"}] + updated = scanner._backfill_source_path_if_needed(recipes, {}) + + assert updated is False + assert recipes[0]["source_path"] == "https://civitai.com/images/9" diff --git a/tests/test_persistent_recipe_cache.py b/tests/test_persistent_recipe_cache.py index 9502c8f8..655022b7 100644 --- a/tests/test_persistent_recipe_cache.py +++ b/tests/test_persistent_recipe_cache.py @@ -552,6 +552,27 @@ class TestPersistentRecipeCache: assert loaded is not None assert loaded.image_id_map == {"222": "new-only"} + def test_metadata_value_roundtrip(self, temp_db_path): + """set_metadata_value/get_metadata_value store and replace values.""" + cache = PersistentRecipeCache(db_path=temp_db_path) + + assert cache.get_metadata_value("source_path_backfilled") is None + + cache.set_metadata_value("source_path_backfilled", "1") + assert cache.get_metadata_value("source_path_backfilled") == "1" + + cache.set_metadata_value("source_path_backfilled", "2") + assert cache.get_metadata_value("source_path_backfilled") == "2" + + def test_metadata_value_survives_save_cache(self, temp_db_path, sample_recipes): + """A full save_cache must not drop unrelated cache_metadata entries.""" + cache = PersistentRecipeCache(db_path=temp_db_path) + + cache.set_metadata_value("source_path_backfilled", "1") + cache.save_cache(sample_recipes) + + assert cache.get_metadata_value("source_path_backfilled") == "1" + class TestHasWorkflowColumn: """has_workflow column persistence (plan 3.1)."""