Files
ComfyUI-Lora-Manager/py/services/persistent_recipe_cache.py
T
Will Miao e14a084f0d fix(cache): make shared cache state survive a second instance
Installing a second LoRA Manager instance (standalone or a second
ComfyUI install) that shares the settings directory puts two processes
on the same cache databases. Three things made that unsafe.

- The updater preserved cache/ and model_cache/ but not a legacy
  recipe_cache/ directory, so a portable install predating the cache/
  move lost its recipe database on a git-based update. Add it to
  _PRESERVE_DIRS and to .gitignore.
- Cache connections used the sqlite3 default 5s timeout, which a
  scanning instance can exceed, turning a concurrent write into
  "database is locked". Route every shared cache connection through
  connect_cache_db(), which raises the timeout to 30s and sets
  busy_timeout + synchronous=NORMAL to match the existing WAL mode.
  App-private databases (download queue, update history) are unchanged.
- A full-table cache replace is a read-modify-write that SQLite cannot
  make atomic across processes, so two instances could interleave and
  one snapshot could overwrite the other. Guard the recipe and model
  save_cache paths with a cross-process advisory lock (flock on POSIX,
  msvcrt on Windows). Locking is best-effort: if it is unavailable the
  call proceeds and the SQLite busy timeout is the fallback.

The lock file is a hidden sibling of the database and is deliberately
never unlinked, so a second process cannot lock a fresh inode.
2026-09-17 23:59:22 +08:00

656 lines
24 KiB
Python

# pyright: reportImportCycles=false
# Lazy (function-local) imports still count as static edges in basedpyright's
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
# import cycles. Breaking them would require an architectural refactor.
"""SQLite-based persistent cache for recipe metadata.
This module provides fast recipe cache persistence using SQLite, enabling
quick startup by loading from cache instead of walking directories and
parsing JSON files.
"""
from __future__ import annotations
import json
import logging
import os
import sqlite3
import threading
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple
from ..utils.cache_db import connect_cache_db
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
from ..utils.file_lock import exclusive_lock
logger = logging.getLogger(__name__)
@dataclass
class PersistedRecipeData:
"""Lightweight structure returned by the persistent recipe cache."""
raw_data: List[Dict[str, Any]]
file_stats: Dict[str, Tuple[float, int]] # json_path -> (mtime, size)
image_id_map: Dict[str, str] = field(default_factory=dict)
"""Precomputed mapping of civitai image_id → recipe_id."""
class PersistentRecipeCache:
"""Persist recipe metadata in SQLite for fast startup."""
_DEFAULT_FILENAME = "recipe_cache.sqlite"
_RECIPE_COLUMNS: Tuple[str, ...] = (
"recipe_id",
"file_path",
"json_path",
"title",
"folder",
"source_path",
"base_model",
"fingerprint",
"created_date",
"modified",
"file_mtime",
"file_size",
"favorite",
"preview_nsfw_level",
"loras_json",
"checkpoint_json",
"gen_params_json",
"tags_json",
"has_workflow",
"import_info_json",
)
_instances: Dict[str, "PersistentRecipeCache"] = {}
_instance_lock = threading.Lock()
def __init__(self, library_name: str = "default", db_path: Optional[str] = None) -> None:
self._library_name = library_name or "default"
self._db_path = db_path or self._resolve_default_path(self._library_name)
self._db_lock = threading.Lock()
self._schema_initialized = False
directory = os.path.dirname(self._db_path)
try:
if directory:
os.makedirs(directory, exist_ok=True)
except Exception as exc:
logger.warning("Could not create recipe cache directory %s: %s", directory, exc)
if self.is_enabled():
self._initialize_schema()
@classmethod
def get_default(cls, library_name: Optional[str] = None) -> "PersistentRecipeCache":
name = library_name or "default"
with cls._instance_lock:
if name not in cls._instances:
cls._instances[name] = cls(name)
return cls._instances[name]
@classmethod
def clear_instances(cls) -> None:
"""Clear all cached instances (useful for library switching)."""
with cls._instance_lock:
cls._instances.clear()
def is_enabled(self) -> bool:
return os.environ.get("LORA_MANAGER_DISABLE_PERSISTENT_CACHE", "0") != "1"
def get_database_path(self) -> str:
"""Expose the resolved SQLite database path."""
return self._db_path
def load_cache(self) -> Optional[PersistedRecipeData]:
"""Load all cached recipes from SQLite.
Returns:
PersistedRecipeData with raw_data and file_stats if cache exists,
None if cache is empty or unavailable.
"""
if not self.is_enabled():
return None
if not self._schema_initialized:
self._initialize_schema()
if not self._schema_initialized:
return None
try:
with self._db_lock:
conn = self._connect(readonly=True)
try:
# Load all recipes
columns_sql = ", ".join(self._RECIPE_COLUMNS)
rows = conn.execute(f"SELECT {columns_sql} FROM recipes").fetchall()
if not rows:
return None
# Restore precomputed image_id_map if available
image_id_map: Dict[str, str] = {}
try:
meta_row = conn.execute(
"SELECT value FROM cache_metadata WHERE key = ?",
("image_id_map",),
).fetchone()
if meta_row:
parsed = json.loads(meta_row["value"])
if isinstance(parsed, dict):
image_id_map = parsed
except Exception:
pass # missing or corrupt — rebuilt on next cache refresh
finally:
conn.close()
except FileNotFoundError:
return None
except Exception as exc:
logger.warning("Failed to load persisted recipe cache: %s", exc)
return None
raw_data: List[Dict[str, Any]] = []
file_stats: Dict[str, Tuple[float, int]] = {}
for row in rows:
recipe = self._row_to_recipe(row)
raw_data.append(recipe)
json_path = row["json_path"]
if json_path:
file_stats[json_path] = (
row["file_mtime"] or 0.0,
row["file_size"] or 0,
)
return PersistedRecipeData(
raw_data=raw_data,
file_stats=file_stats,
image_id_map=image_id_map,
)
def save_cache(
self,
recipes: List[Dict[str, Any]],
json_paths: Optional[Dict[str, str]] = None,
image_id_map: Optional[Dict[str, str]] = None,
skip_if_empty: bool = False,
) -> bool:
"""Save all recipes to SQLite cache.
Args:
recipes: List of recipe dictionaries to persist.
json_paths: Optional mapping of recipe_id -> json_path for file stats.
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
skip_if_empty: When True, refuse to replace a non-empty cache with an
empty one. This is the storage-level backstop against a scan that
silently loses every recipe (unavailable drive / mis-resolved
recipes directory): overwriting both deletes the user's data and
destroys their only record of it. Intentional full clears (manual
rebuild) must pass ``skip_if_empty=False``.
Returns:
``True`` when the write happened, ``False`` when it was skipped.
"""
if not self.is_enabled():
return False
if not self._schema_initialized:
self._initialize_schema()
if not self._schema_initialized:
return False
try:
with self._db_lock:
# Cross-process serialization: another LoRA Manager instance may
# share this settings directory, and a full-table replace is a
# read-modify-write that SQLite alone cannot make atomic.
with exclusive_lock(self._db_path):
conn = self._connect()
try:
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("BEGIN")
if skip_if_empty and not recipes:
existing = conn.execute(
"SELECT COUNT(*) FROM recipes"
).fetchone()
if existing and existing[0]:
conn.rollback()
logger.warning(
"Refusing to persist an empty recipe cache: the "
"stored cache still holds %d recipe(s). The scan "
"found nothing, which usually means the recipes "
"path was unavailable or resolved elsewhere; "
"keeping the stored cache so the data stays "
"recoverable.",
existing[0],
)
return False
# Clear existing data
conn.execute("DELETE FROM recipes")
# Prepare and insert all rows
recipe_rows = []
for recipe in recipes:
recipe_id = str(recipe.get("id", ""))
if not recipe_id:
continue
json_path = ""
if json_paths:
json_path = json_paths.get(recipe_id, "")
row = self._prepare_recipe_row(recipe, json_path)
recipe_rows.append(row)
if recipe_rows:
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
columns = ", ".join(self._RECIPE_COLUMNS)
conn.executemany(
f"INSERT INTO recipes ({columns}) VALUES ({placeholders})",
recipe_rows,
)
# Persist image_id_map for O(1) lookups on cache load
conn.execute(
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
("image_id_map", json.dumps(image_id_map or {})),
)
conn.commit()
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
return True
finally:
conn.close()
except Exception as exc:
logger.warning("Failed to persist recipe cache: %s", exc)
return False
def get_file_stats(self) -> Dict[str, Tuple[float, int]]:
"""Return stored file stats for all cached recipes.
Returns:
Dictionary mapping json_path -> (mtime, size).
"""
if not self.is_enabled() or not self._schema_initialized:
return {}
try:
with self._db_lock:
conn = self._connect(readonly=True)
try:
rows = conn.execute(
"SELECT json_path, file_mtime, file_size FROM recipes WHERE json_path IS NOT NULL"
).fetchall()
return {
row["json_path"]: (row["file_mtime"] or 0.0, row["file_size"] or 0)
for row in rows
if row["json_path"]
}
finally:
conn.close()
except Exception:
return {}
def update_recipe(self, recipe: Dict[str, Any], json_path: Optional[str] = None) -> None:
"""Update or insert a single recipe in the cache.
Args:
recipe: The recipe dictionary to persist.
json_path: Optional path to the recipe JSON file.
"""
if not self.is_enabled() or not self._schema_initialized:
return
recipe_id = str(recipe.get("id", ""))
if not recipe_id:
return
try:
with self._db_lock:
conn = self._connect()
try:
row = self._prepare_recipe_row(recipe, json_path or "")
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
columns = ", ".join(self._RECIPE_COLUMNS)
conn.execute(
f"INSERT OR REPLACE INTO recipes ({columns}) VALUES ({placeholders})",
row,
)
conn.commit()
finally:
conn.close()
except Exception as exc:
logger.debug("Failed to update recipe %s in cache: %s", recipe_id, exc)
def remove_recipe(self, recipe_id: str) -> None:
"""Remove a recipe from the cache by ID.
Args:
recipe_id: The ID of the recipe to remove.
"""
if not self.is_enabled() or not self._schema_initialized:
return
if not recipe_id:
return
try:
with self._db_lock:
conn = self._connect()
try:
conn.execute("DELETE FROM recipes WHERE recipe_id = ?", (str(recipe_id),))
conn.commit()
finally:
conn.close()
except Exception as exc:
logger.debug("Failed to remove recipe %s from cache: %s", recipe_id, exc)
def save_image_id_map(self, image_id_map: Dict[str, str]) -> None:
"""Persist the image_id_map to cache_metadata without rewriting the full cache.
This is called after ``add_recipe`` / ``remove_recipe`` mutations so
the persistent copy does not go stale between full ``save_cache`` calls.
"""
if not self.is_enabled() or not self._schema_initialized:
return
try:
with self._db_lock:
conn = self._connect()
try:
conn.execute(
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
("image_id_map", json.dumps(image_id_map)),
)
conn.commit()
finally:
conn.close()
except Exception as exc:
logger.debug("Failed to persist image_id_map: %s", exc)
def get_metadata_value(self, key: str) -> Optional[str]:
"""Return a value from cache_metadata, or None if missing."""
if not self.is_enabled() or not self._schema_initialized:
return None
try:
with self._db_lock:
conn = self._connect(readonly=True)
try:
row = conn.execute(
"SELECT value FROM cache_metadata WHERE key = ?",
(key,),
).fetchone()
return row["value"] if row else None
finally:
conn.close()
except Exception:
return None
def set_metadata_value(self, key: str, value: str) -> None:
"""Store a value in cache_metadata without rewriting the full cache."""
if not self.is_enabled() or not self._schema_initialized:
return
try:
with self._db_lock:
conn = self._connect()
try:
conn.execute(
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
(key, value),
)
conn.commit()
finally:
conn.close()
except Exception as exc:
logger.debug("Failed to persist cache metadata %s: %s", key, exc)
def get_indexed_recipe_ids(self) -> Set[str]:
"""Return all recipe IDs in the cache.
Returns:
Set of recipe ID strings.
"""
if not self.is_enabled() or not self._schema_initialized:
return set()
try:
with self._db_lock:
conn = self._connect(readonly=True)
try:
rows = conn.execute("SELECT recipe_id FROM recipes").fetchall()
return {row["recipe_id"] for row in rows if row["recipe_id"]}
finally:
conn.close()
except Exception:
return set()
def get_recipe_count(self) -> int:
"""Return the number of recipes in the cache."""
if not self.is_enabled() or not self._schema_initialized:
return 0
try:
with self._db_lock:
conn = self._connect(readonly=True)
try:
result = conn.execute("SELECT COUNT(*) FROM recipes").fetchone()
return result[0] if result else 0
finally:
conn.close()
except Exception:
return 0
# Internal helpers
def _resolve_default_path(self, library_name: str) -> str:
env_override = os.environ.get("LORA_MANAGER_RECIPE_CACHE_DB")
return resolve_cache_path_with_migration(
CacheType.RECIPE,
library_name=library_name,
env_override=env_override,
)
def _initialize_schema(self) -> None:
with self._db_lock:
if self._schema_initialized:
return
try:
with self._connect() as conn:
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys = ON")
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS recipes (
recipe_id TEXT PRIMARY KEY,
file_path TEXT,
json_path TEXT,
title TEXT,
folder TEXT,
source_path TEXT,
base_model TEXT,
fingerprint TEXT,
created_date REAL,
modified REAL,
file_mtime REAL,
file_size INTEGER,
favorite INTEGER DEFAULT 0,
preview_nsfw_level INTEGER DEFAULT 0,
loras_json TEXT,
checkpoint_json TEXT,
gen_params_json TEXT,
tags_json TEXT,
has_workflow INTEGER DEFAULT 0,
import_info_json TEXT
);
CREATE INDEX IF NOT EXISTS idx_recipes_json_path ON recipes(json_path);
CREATE INDEX IF NOT EXISTS idx_recipes_fingerprint ON recipes(fingerprint);
CREATE TABLE IF NOT EXISTS cache_metadata (
key TEXT PRIMARY KEY,
value TEXT
);
"""
)
# Migration: add source_path column to existing databases
try:
conn.execute(
"ALTER TABLE recipes ADD COLUMN source_path TEXT"
)
except Exception:
pass # column already exists
# Migration: add has_workflow column to existing databases
try:
conn.execute(
"ALTER TABLE recipes ADD COLUMN has_workflow INTEGER DEFAULT 0"
)
except Exception:
pass # column already exists
# Migration: add import_info_json column to existing databases
try:
conn.execute(
"ALTER TABLE recipes ADD COLUMN import_info_json TEXT"
)
except Exception:
pass # column already exists
conn.commit()
self._schema_initialized = True
except Exception as exc:
logger.warning("Failed to initialize persistent recipe cache schema: %s", exc)
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
if readonly and not os.path.exists(self._db_path):
raise FileNotFoundError(self._db_path)
return connect_cache_db(
self._db_path,
readonly=readonly,
detect_types=sqlite3.PARSE_DECLTYPES,
row_factory=sqlite3.Row,
)
def _prepare_recipe_row(self, recipe: Dict[str, Any], json_path: str) -> Tuple[Any, ...]:
"""Convert a recipe dict to a row tuple for SQLite insertion."""
loras = recipe.get("loras")
loras_json = json.dumps(loras) if loras else None
checkpoint = recipe.get("checkpoint")
checkpoint_json = json.dumps(checkpoint) if checkpoint else None
gen_params = recipe.get("gen_params")
gen_params_json = json.dumps(gen_params) if gen_params else None
tags = recipe.get("tags")
tags_json = json.dumps(tags) if tags else None
import_info = recipe.get("import_info")
import_info_json = json.dumps(import_info) if import_info else None
# Get file stats if json_path exists
file_mtime = 0.0
file_size = 0
if json_path and os.path.exists(json_path):
try:
stat = os.stat(json_path)
file_mtime = stat.st_mtime
file_size = stat.st_size
except OSError:
pass
return (
str(recipe.get("id", "")),
recipe.get("file_path"),
json_path,
recipe.get("title"),
recipe.get("folder"),
recipe.get("source_path"),
recipe.get("base_model"),
recipe.get("fingerprint"),
float(recipe.get("created_date") or 0.0),
float(recipe.get("modified") or 0.0),
file_mtime,
file_size,
1 if recipe.get("favorite") else 0,
int(recipe.get("preview_nsfw_level") or 0),
loras_json,
checkpoint_json,
gen_params_json,
tags_json,
1 if recipe.get("has_workflow") else 0,
import_info_json,
)
def _row_to_recipe(self, row: sqlite3.Row) -> Dict[str, Any]:
"""Convert a SQLite row to a recipe dictionary."""
loras = []
if row["loras_json"]:
try:
loras = json.loads(row["loras_json"])
except json.JSONDecodeError:
pass
checkpoint = None
if row["checkpoint_json"]:
try:
checkpoint = json.loads(row["checkpoint_json"])
except json.JSONDecodeError:
pass
gen_params = {}
if row["gen_params_json"]:
try:
gen_params = json.loads(row["gen_params_json"])
except json.JSONDecodeError:
pass
tags = []
if row["tags_json"]:
try:
tags = json.loads(row["tags_json"])
except json.JSONDecodeError:
pass
import_info = None
if row["import_info_json"]:
try:
import_info = json.loads(row["import_info_json"])
except json.JSONDecodeError:
pass
recipe = {
"id": row["recipe_id"],
"file_path": row["file_path"] or "",
"title": row["title"] or "",
"folder": row["folder"] or "",
"source_path": row["source_path"] or "",
"base_model": row["base_model"] or "",
"fingerprint": row["fingerprint"] or "",
"created_date": row["created_date"] or 0.0,
"modified": row["modified"] or 0.0,
"favorite": bool(row["favorite"]),
"preview_nsfw_level": row["preview_nsfw_level"] or 0,
"has_workflow": bool(row["has_workflow"]),
"loras": loras,
"gen_params": gen_params,
}
if tags:
recipe["tags"] = tags
if checkpoint:
recipe["checkpoint"] = checkpoint
if import_info:
recipe["import_info"] = import_info
return recipe
def get_persistent_recipe_cache() -> PersistentRecipeCache:
"""Get the default persistent recipe cache instance for the active library."""
from .settings_manager import get_settings_manager
library_name = get_settings_manager().get_active_library_name()
return PersistentRecipeCache.get_default(library_name)