From 74f889f1603bf30c417b3a6d2bf9d137214b6ff0 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Tue, 25 Aug 2026 17:38:31 +0800 Subject: [PATCH] fix(recipes): validate FTS index from stored metadata instead of scanning --- py/services/recipe_fts_index.py | 174 +++++++++++++++++++++- tests/test_recipe_fts_index_validation.py | 85 +++++++++++ 2 files changed, 252 insertions(+), 7 deletions(-) diff --git a/py/services/recipe_fts_index.py b/py/services/recipe_fts_index.py index 0d30ab32..2a74af7a 100644 --- a/py/services/recipe_fts_index.py +++ b/py/services/recipe_fts_index.py @@ -7,13 +7,14 @@ enabling sub-100ms search times even with 20k+ recipes. from __future__ import annotations import asyncio +import hashlib import logging import os import re import sqlite3 import threading import time -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration @@ -165,6 +166,7 @@ class RecipeFTSIndex: batch_size = 500 total = len(recipes) inserted = 0 + indexed_ids: Set[str] = set() for i in range(0, total, batch_size): batch = recipes[i:i + batch_size] @@ -179,6 +181,7 @@ class RecipeFTSIndex: row = self._prepare_fts_row(recipe) rows.append(row) inserted += 1 + indexed_ids.add(recipe_id) if rows: # Insert into FTS table @@ -213,7 +216,11 @@ class RecipeFTSIndex: ) conn.execute( "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", - ('recipe_count', str(inserted)) + (self._COUNT_METADATA_KEY, str(inserted)) + ) + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(indexed_ids)) ) conn.commit() @@ -288,6 +295,12 @@ class RecipeFTSIndex: with self._lock: conn = self._connect() try: + # Check existence via the rowid mapping (fast PK lookup) + existed = conn.execute( + "SELECT 1 FROM recipe_rowid WHERE recipe_id = ?", + (recipe_id,) + ).fetchone() is not None + # Remove existing entry if present self._remove_recipe_locked(conn, recipe_id) @@ -312,6 +325,10 @@ class RecipeFTSIndex: (recipe_id, result[0]) ) + # Keep validation metadata in sync (only a new id changes it) + if not existed: + self._update_mutation_metadata_locked(conn, recipe_id, delta=1) + conn.commit() return True finally: @@ -339,7 +356,13 @@ class RecipeFTSIndex: with self._lock: conn = self._connect() try: + existed = conn.execute( + "SELECT 1 FROM recipe_rowid WHERE recipe_id = ?", + (recipe_id,) + ).fetchone() is not None self._remove_recipe_locked(conn, recipe_id) + if existed: + self._update_mutation_metadata_locked(conn, recipe_id, delta=-1) conn.commit() return True finally: @@ -371,6 +394,15 @@ class RecipeFTSIndex: try: conn.execute("DELETE FROM recipe_fts") conn.execute("DELETE FROM recipe_rowid") + # Reset validation metadata to the empty index state + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._COUNT_METADATA_KEY, '0') + ) + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(set())) + ) conn.commit() self._ready.clear() return True @@ -427,10 +459,12 @@ class RecipeFTSIndex: """Check if the FTS index matches the expected recipes. This method validates whether the existing FTS index can be reused - without a full rebuild. It checks: - 1. The index has been initialized - 2. The count matches - 3. The recipe IDs match + without a full rebuild. It compares the expected count and recipe ID + fingerprint against metadata recorded when the index was (re)built, + so it does not scan the FTS content table. Indexes built by older + versions lack this metadata; for those the validation falls back to + a one-time scan of the content table and records the metadata so + subsequent startups are cheap. Args: recipe_count: Expected number of recipes. @@ -446,7 +480,28 @@ class RecipeFTSIndex: return False try: + metadata = self._read_validation_metadata() + if metadata is not None: + stored_count, stored_fingerprint = metadata + if stored_count != recipe_count: + logger.debug( + "FTS index count mismatch: indexed=%d, expected=%d", + stored_count, recipe_count + ) + return False + + if stored_fingerprint != self._compute_ids_fingerprint(recipe_ids): + logger.debug("FTS index recipe ID fingerprint mismatch") + return False + + return True + + # Legacy fallback: no stored metadata, scan the content table once + # and persist the metadata so later validations are cheap. indexed_count = self.get_indexed_count() + indexed_ids = self.get_indexed_recipe_ids() + self._store_validation_metadata(indexed_count, indexed_ids) + if indexed_count != recipe_count: logger.debug( "FTS index count mismatch: indexed=%d, expected=%d", @@ -454,7 +509,6 @@ class RecipeFTSIndex: ) return False - indexed_ids = self.get_indexed_recipe_ids() if indexed_ids != recipe_ids: missing = recipe_ids - indexed_ids extra = indexed_ids - recipe_ids @@ -471,6 +525,112 @@ class RecipeFTSIndex: # Internal helpers + _FINGERPRINT_METADATA_KEY = 'recipe_ids_fingerprint' + _COUNT_METADATA_KEY = 'recipe_count' + + @staticmethod + def _fingerprint_recipe_id(recipe_id: str) -> int: + """Return a stable 64-bit fingerprint contribution for a recipe ID.""" + digest = hashlib.sha256(recipe_id.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") + + @classmethod + def _compute_ids_fingerprint(cls, recipe_ids: Set[str]) -> str: + """Order-independent fingerprint of a recipe ID set (XOR of per-id hashes).""" + fingerprint = 0 + for recipe_id in recipe_ids: + fingerprint ^= cls._fingerprint_recipe_id(str(recipe_id)) + return f"{fingerprint:016x}" + + def _read_validation_metadata(self) -> Optional[Tuple[int, str]]: + """Return stored (recipe count, ID fingerprint), or None if absent.""" + try: + with self._lock: + conn = self._connect(readonly=True) + try: + rows = conn.execute( + "SELECT key, value FROM fts_metadata WHERE key IN (?, ?)", + (self._COUNT_METADATA_KEY, self._FINGERPRINT_METADATA_KEY) + ).fetchall() + values = {row[0]: row[1] for row in rows} + fingerprint = values.get(self._FINGERPRINT_METADATA_KEY) + if fingerprint is None: + return None + try: + count = int(values.get(self._COUNT_METADATA_KEY) or 0) + except (TypeError, ValueError): + return None + return count, fingerprint + finally: + conn.close() + except FileNotFoundError: + return None + except Exception as exc: + logger.debug("Failed to read FTS validation metadata: %s", exc) + return None + + def _store_validation_metadata(self, recipe_count: int, recipe_ids: Set[str]) -> None: + """Persist recipe count and ID fingerprint for cheap future validation.""" + try: + with self._lock: + conn = self._connect() + try: + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._COUNT_METADATA_KEY, str(recipe_count)) + ) + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(recipe_ids)) + ) + conn.commit() + finally: + conn.close() + except Exception as exc: + logger.debug("Failed to store FTS validation metadata: %s", exc) + + def _update_mutation_metadata_locked( + self, + conn: sqlite3.Connection, + recipe_id: str, + delta: int, + ) -> None: + """Incrementally maintain validation metadata after add/remove. + + Caller must hold the lock. The fingerprint is only updated when it + already exists; without it, validation falls back to a one-time scan + that records fresh metadata. + """ + fingerprint_row = conn.execute( + "SELECT value FROM fts_metadata WHERE key = ?", + (self._FINGERPRINT_METADATA_KEY,) + ).fetchone() + if fingerprint_row and fingerprint_row[0]: + try: + fingerprint = int(fingerprint_row[0], 16) + except ValueError: + fingerprint = None + if fingerprint is not None: + fingerprint ^= self._fingerprint_recipe_id(recipe_id) + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._FINGERPRINT_METADATA_KEY, f"{fingerprint & 0xFFFFFFFFFFFFFFFF:016x}") + ) + + count_row = conn.execute( + "SELECT value FROM fts_metadata WHERE key = ?", + (self._COUNT_METADATA_KEY,) + ).fetchone() + if count_row: + try: + count = max(0, int(count_row[0] or 0) + delta) + except (TypeError, ValueError): + return + conn.execute( + "INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)", + (self._COUNT_METADATA_KEY, str(count)) + ) + def _connect(self, readonly: bool = False) -> sqlite3.Connection: """Create a database connection.""" uri = False diff --git a/tests/test_recipe_fts_index_validation.py b/tests/test_recipe_fts_index_validation.py index 9bffe073..6cc724a0 100644 --- a/tests/test_recipe_fts_index_validation.py +++ b/tests/test_recipe_fts_index_validation.py @@ -181,3 +181,88 @@ class TestFTSIndexValidation: # Search should still work results = fts.search("anime") assert "recipe-001" in results + + @staticmethod + def _forbid_content_scan(monkeypatch): + """Fail the test if validation falls back to scanning the FTS content table.""" + + def _raise(*args, **kwargs): + raise AssertionError("validate_index scanned the FTS content table") + + monkeypatch.setattr(RecipeFTSIndex, "get_indexed_count", _raise) + monkeypatch.setattr(RecipeFTSIndex, "get_indexed_recipe_ids", _raise) + + def test_validate_index_with_metadata_does_not_scan( + self, temp_db_path, sample_recipes, monkeypatch + ): + """Validation must rely on stored metadata, not content-table scans.""" + fts = RecipeFTSIndex(db_path=temp_db_path) + fts.build_index(sample_recipes) + + self._forbid_content_scan(monkeypatch) + + result = fts.validate_index(3, {"recipe-001", "recipe-002", "recipe-003"}) + assert result is True + + # Mismatches are also detected from metadata alone + result = fts.validate_index(4, {"recipe-001", "recipe-002", "recipe-003"}) + assert result is False + result = fts.validate_index(3, {"recipe-001", "recipe-002", "recipe-999"}) + assert result is False + + def test_validate_index_legacy_fallback_writes_metadata( + self, temp_db_path, sample_recipes, monkeypatch + ): + """Indexes built by older versions validate via a one-time scan, then metadata.""" + import sqlite3 + + fts = RecipeFTSIndex(db_path=temp_db_path) + fts.build_index(sample_recipes) + + # Simulate a legacy index: drop the fingerprint metadata + conn = sqlite3.connect(temp_db_path) + try: + conn.execute( + "DELETE FROM fts_metadata WHERE key = ?", + (RecipeFTSIndex._FINGERPRINT_METADATA_KEY,) + ) + conn.commit() + finally: + conn.close() + + # Fallback validation scans once and succeeds + result = fts.validate_index(3, {"recipe-001", "recipe-002", "recipe-003"}) + assert result is True + + # Metadata was written, so the next validation needs no scan + self._forbid_content_scan(monkeypatch) + result = fts.validate_index(3, {"recipe-001", "recipe-002", "recipe-003"}) + assert result is True + + def test_validate_index_metadata_tracks_incremental_mutations( + self, temp_db_path, sample_recipes, monkeypatch + ): + """add_recipe/remove_recipe keep the validation metadata in sync.""" + fts = RecipeFTSIndex(db_path=temp_db_path) + fts.build_index(sample_recipes[:2]) + + fts.add_recipe(sample_recipes[2]) + fts.remove_recipe("recipe-001") + + self._forbid_content_scan(monkeypatch) + + result = fts.validate_index(2, {"recipe-002", "recipe-003"}) + assert result is True + + def test_validate_index_after_clear_uses_metadata( + self, temp_db_path, sample_recipes, monkeypatch + ): + """clear() resets the validation metadata to the empty index state.""" + fts = RecipeFTSIndex(db_path=temp_db_path) + fts.build_index(sample_recipes) + fts.clear() + + self._forbid_content_scan(monkeypatch) + + assert fts.validate_index(0, set()) is True + assert fts.validate_index(3, {"recipe-001", "recipe-002", "recipe-003"}) is False