fix(recipes): make source_path backfill a one-shot migration

py/services/persistent_recipe_cache.py

@@ -333,6 +333,44 @@ class PersistentRecipeCache:
         except Exception as exc:
             logger.debug("Failed to persist image_id_map: %s", exc)
 
+    def get_metadata_value(self, key: str) -> Optional[str]:
+        """Return a value from cache_metadata, or None if missing."""
+        if not self.is_enabled() or not self._schema_initialized:
+            return None
+
+        try:
+            with self._db_lock:
+                conn = self._connect(readonly=True)
+                try:
+                    row = conn.execute(
+                        "SELECT value FROM cache_metadata WHERE key = ?",
+                        (key,),
+                    ).fetchone()
+                    return row["value"] if row else None
+                finally:
+                    conn.close()
+        except Exception:
+            return None
+
+    def set_metadata_value(self, key: str, value: str) -> None:
+        """Store a value in cache_metadata without rewriting the full cache."""
+        if not self.is_enabled() or not self._schema_initialized:
+            return
+
+        try:
+            with self._db_lock:
+                conn = self._connect()
+                try:
+                    conn.execute(
+                        "INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
+                        (key, value),
+                    )
+                    conn.commit()
+                finally:
+                    conn.close()
+        except Exception as exc:
+            logger.debug("Failed to persist cache metadata %s: %s", key, exc)
+
     def get_indexed_recipe_ids(self) -> Set[str]:
         """Return all recipe IDs in the cache.
 

py/services/recipe_scanner.py

@@ -1547,7 +1547,7 @@ class RecipeScanner:
                     self._cache.raw_data = recipes
                     self._update_folder_metadata(self._cache)
                     self._sort_cache_sync()
-                    # Backfill source_path from JSON files if missing (schema migration)
+                    # Backfill source_path from JSON files if missing (one-shot schema migration)
                     if self._backfill_source_path_if_needed(recipes, json_paths):
                         self._cache.image_id_map = self._build_image_id_map()
                         self._persistent_cache.save_cache(
@@ -1574,7 +1574,7 @@ class RecipeScanner:
                     self._cache.raw_data = recipes
                     self._update_folder_metadata(self._cache)
                     self._sort_cache_sync()
-                    # Backfill source_path from JSON files if missing (schema migration)
+                    # Backfill source_path from JSON files if missing (one-shot schema migration)
                     self._backfill_source_path_if_needed(recipes, json_paths)
                     self._cache.image_id_map = self._build_image_id_map()
                     # Persist updated cache
@@ -1711,6 +1711,9 @@ class RecipeScanner:
 
         return recipes, changed, json_paths
 
+    # Metadata key recording that the one-shot source_path backfill has run.
+    _SOURCE_PATH_BACKFILL_MARKER = "source_path_backfilled"
+
     def _backfill_source_path_if_needed(
         self,
         recipes: List[Dict[str, Any]],
@@ -1718,8 +1721,21 @@ class RecipeScanner:
     ) -> bool:
         """Backfill source_path from recipe JSON files if missing from cache.
 
+        This is a one-shot schema migration: once it has run, a completion
+        marker is stored in the persistent cache metadata and later startups
+        skip it entirely. Recipes without a source_path in their JSON file
+        would otherwise be re-read and re-parsed on every startup. New or
+        changed recipe files still get source_path from the normal parse path
+        during reconciliation.
+
         Returns True if any recipes were updated (caller should persist cache).
         """
+        cache = self._persistent_cache
+        if (
+            cache is not None
+            and cache.get_metadata_value(self._SOURCE_PATH_BACKFILL_MARKER) == "1"
+        ):
+            return False
         updated = False
         for recipe in recipes:
             if recipe.get("source_path"):
@@ -1737,6 +1753,8 @@ class RecipeScanner:
                     updated = True
             except Exception:
                 pass
+        if cache is not None:
+            cache.set_metadata_value(self._SOURCE_PATH_BACKFILL_MARKER, "1")
         return updated
 
     def _full_directory_scan_sync(

tests/services/test_recipe_scanner_source_path_backfill.py

@@ -0,0 +1,104 @@
+"""Tests for the one-shot source_path backfill in RecipeScanner."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from py.services.persistent_recipe_cache import PersistentRecipeCache
+from py.services.recipe_scanner import RecipeScanner
+
+
+@pytest.fixture
+def scanner_with_cache(tmp_path: Path):
+    """RecipeScanner instance backed by a real persistent cache in tmp_path."""
+    cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
+    scanner = RecipeScanner.__new__(RecipeScanner)
+    scanner._persistent_cache = cache
+    return scanner, cache
+
+
+def _write_recipe_json(path: Path, recipe_id: str, source_path: str | None = None) -> None:
+    data = {"id": recipe_id}
+    if source_path is not None:
+        data["source_path"] = source_path
+    path.write_text(json.dumps(data), encoding="utf-8")
+
+
+def test_backfill_processes_recipes_when_marker_absent(scanner_with_cache, tmp_path: Path):
+    """Without the completion marker, missing source_path values are backfilled."""
+    scanner, cache = scanner_with_cache
+
+    json_with_source = tmp_path / "r1.recipe.json"
+    _write_recipe_json(json_with_source, "r1", source_path="https://civitai.com/images/1")
+    json_without_source = tmp_path / "r2.recipe.json"
+    _write_recipe_json(json_without_source, "r2")
+
+    recipes = [
+        {"id": "r1", "source_path": ""},
+        {"id": "r2", "source_path": ""},
+        {"id": "r3", "source_path": "https://civitai.com/images/3"},
+    ]
+    json_paths = {"r1": str(json_with_source), "r2": str(json_without_source)}
+
+    updated = scanner._backfill_source_path_if_needed(recipes, json_paths)
+
+    assert updated is True
+    assert recipes[0]["source_path"] == "https://civitai.com/images/1"
+    # JSON legitimately has no source_path: stays empty, no error
+    assert recipes[1]["source_path"] == ""
+    assert recipes[2]["source_path"] == "https://civitai.com/images/3"
+    # The run records the completion marker
+    assert (
+        cache.get_metadata_value(RecipeScanner._SOURCE_PATH_BACKFILL_MARKER) == "1"
+    )
+
+
+def test_backfill_is_skipped_once_marker_is_set(scanner_with_cache, tmp_path: Path, monkeypatch):
+    """The second initialization must not re-read recipe JSON files."""
+    scanner, cache = scanner_with_cache
+
+    json_path = tmp_path / "r1.recipe.json"
+    _write_recipe_json(json_path, "r1", source_path="https://civitai.com/images/1")
+
+    recipes = [{"id": "r1", "source_path": ""}]
+    json_paths = {"r1": str(json_path)}
+
+    # First run: backfills and records the marker
+    assert scanner._backfill_source_path_if_needed(recipes, json_paths) is True
+    assert recipes[0]["source_path"] == "https://civitai.com/images/1"
+
+    # Second run: simulate a fresh startup where the cache still lacks
+    # source_path. Under the old behavior the file would be re-read and
+    # re-parsed; now the marker must suppress any file access.
+    recipes = [{"id": "r1", "source_path": ""}]
+
+    import os
+
+    real_exists = os.path.exists
+
+    def _failing_exists(path):
+        if str(path).endswith(".recipe.json"):
+            raise AssertionError("backfill touched the filesystem despite marker")
+        return real_exists(path)
+
+    monkeypatch.setattr(os.path, "exists", _failing_exists)
+
+    updated = scanner._backfill_source_path_if_needed(recipes, json_paths)
+
+    assert updated is False
+    assert recipes[0]["source_path"] == ""
+
+
+def test_backfill_marker_does_not_suppress_reconcile_parsed_source_path(scanner_with_cache):
+    """The marker only gates the backfill; parsed recipes keep their source_path."""
+    scanner, cache = scanner_with_cache
+    cache.set_metadata_value(RecipeScanner._SOURCE_PATH_BACKFILL_MARKER, "1")
+
+    # A recipe that arrived from the normal parse path with a source_path is
+    # left untouched by the (skipped) backfill.
+    recipes = [{"id": "r1", "source_path": "https://civitai.com/images/9"}]
+    updated = scanner._backfill_source_path_if_needed(recipes, {})
+
+    assert updated is False
+    assert recipes[0]["source_path"] == "https://civitai.com/images/9"

tests/test_persistent_recipe_cache.py

@@ -552,6 +552,27 @@ class TestPersistentRecipeCache:
         assert loaded is not None
         assert loaded.image_id_map == {"222": "new-only"}
 
+    def test_metadata_value_roundtrip(self, temp_db_path):
+        """set_metadata_value/get_metadata_value store and replace values."""
+        cache = PersistentRecipeCache(db_path=temp_db_path)
+
+        assert cache.get_metadata_value("source_path_backfilled") is None
+
+        cache.set_metadata_value("source_path_backfilled", "1")
+        assert cache.get_metadata_value("source_path_backfilled") == "1"
+
+        cache.set_metadata_value("source_path_backfilled", "2")
+        assert cache.get_metadata_value("source_path_backfilled") == "2"
+
+    def test_metadata_value_survives_save_cache(self, temp_db_path, sample_recipes):
+        """A full save_cache must not drop unrelated cache_metadata entries."""
+        cache = PersistentRecipeCache(db_path=temp_db_path)
+
+        cache.set_metadata_value("source_path_backfilled", "1")
+        cache.save_cache(sample_recipes)
+
+        assert cache.get_metadata_value("source_path_backfilled") == "1"
+
 
 class TestHasWorkflowColumn:
     """has_workflow column persistence (plan 3.1)."""