mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-06 22:10:14 -03:00
feat(cache): opportunistic cache sync on metadata read with in-place update
- Add PersistentModelCache.update_single_model() for lightweight targeted SQL update (single row + incremental tag/hash deltas, no full table scan) - Add ModelScanner.sync_cache_from_metadata() with compare-first logic: skips entirely when cache is already in sync; when stale, updates the entry in-place (O(1) instead of O(n) remove+append), incrementally adjusts tag counts/hash index/version index, and resorts only when sort-relevant fields changed - Wire sync_cache_from_metadata() into BaseModelService.get_model_metadata() via fire-and-forget asyncio.create_task — disk I/O is already paid for - Include identity re-validation guard against concurrent cache replacement - Add 16 tests covering _cache_entries_differ, sync_cache_from_metadata (no-change, in-place, fallback, conditional resort), and update_single_model (insert, tag delta, hash delta)
This commit is contained in:
@@ -1092,6 +1092,11 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
Listing/search endpoints return lightweight cache entries; this method performs
|
Listing/search endpoints return lightweight cache entries; this method performs
|
||||||
a lazy read of the on-disk metadata snapshot when callers need full detail.
|
a lazy read of the on-disk metadata snapshot when callers need full detail.
|
||||||
|
|
||||||
|
As a beneficial side effect, the in-memory and persistent caches are
|
||||||
|
opportunistically synchronised with the on-disk metadata — this keeps the
|
||||||
|
caches fresh even when a ``.metadata.json`` file was edited outside of the
|
||||||
|
normal save path (e.g. manually or by an external script).
|
||||||
"""
|
"""
|
||||||
metadata, should_skip = await MetadataManager.load_metadata(
|
metadata, should_skip = await MetadataManager.load_metadata(
|
||||||
file_path, self.metadata_class
|
file_path, self.metadata_class
|
||||||
@@ -1109,6 +1114,19 @@ class BaseModelService(ABC):
|
|||||||
MetadataManager.save_metadata(file_path, metadata)
|
MetadataManager.save_metadata(file_path, metadata)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Opportunistically sync the in-memory + persistent caches.
|
||||||
|
# The .metadata.json disk read is already paid for; the sync only
|
||||||
|
# performs work when the cache is actually stale, and uses targeted,
|
||||||
|
# in-place operations to minimise overhead even with large model sets.
|
||||||
|
#
|
||||||
|
# Fire-and-forget by design: the task is intentionally untracked.
|
||||||
|
# sync_cache_from_metadata handles its own errors internally.
|
||||||
|
asyncio.create_task(
|
||||||
|
self.scanner.sync_cache_from_metadata(
|
||||||
|
file_path, metadata.to_dict()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return self.filter_civitai_data(metadata.to_dict().get("civitai", {}))
|
return self.filter_civitai_data(metadata.to_dict().get("civitai", {}))
|
||||||
|
|
||||||
async def get_model_description(self, file_path: str) -> Optional[str]:
|
async def get_model_description(self, file_path: str) -> Optional[str]:
|
||||||
|
|||||||
@@ -1566,6 +1566,218 @@ class ModelScanner:
|
|||||||
|
|
||||||
return cache_entry if metadata else True
|
return cache_entry if metadata else True
|
||||||
|
|
||||||
|
async def sync_cache_from_metadata(
|
||||||
|
self, file_path: str, metadata_dict: Dict[str, Any]
|
||||||
|
) -> bool:
|
||||||
|
"""Opportunistically sync in-memory and persistent caches from metadata.
|
||||||
|
|
||||||
|
Builds a prospective cache entry from *metadata_dict* (deserialized
|
||||||
|
``.metadata.json`` content) and compares it against the current cache
|
||||||
|
entry. When the two are already identical this method returns
|
||||||
|
``False`` without touching anything — avoiding the overhead of
|
||||||
|
``update_single_model_cache``, which always removes and re-inserts
|
||||||
|
the entry, triggers a full resort, and persists via the heavyweight
|
||||||
|
``save_cache()``.
|
||||||
|
|
||||||
|
When differences are detected the update is applied **in-place** with
|
||||||
|
targeted operations:
|
||||||
|
|
||||||
|
* The existing ``raw_data`` entry is modified rather than removed and
|
||||||
|
re-appended (O(1) instead of O(n)).
|
||||||
|
* Tag counts and the hash index are updated incrementally.
|
||||||
|
* The version index is rebuilt only for the affected entry.
|
||||||
|
* ``resort()`` is called **only** when a sort-relevant field changed
|
||||||
|
(``model_name`` / ``file_name`` for name-sort, ``modified`` for
|
||||||
|
date-sort, ``size`` for size-sort).
|
||||||
|
* The persistent (SQLite) cache receives a targeted single-row update
|
||||||
|
via :meth:`PersistentModelCache.update_single_model` rather than a
|
||||||
|
full-table ``save_cache()``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``True`` if any cache update was performed, ``False`` if the
|
||||||
|
caches were already in sync.
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
This is a **best-effort** operation. Failures are logged but
|
||||||
|
never propagated — callers should fire-and-forget via
|
||||||
|
:func:`asyncio.create_task`.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await self._sync_cache_from_metadata_impl(
|
||||||
|
file_path, metadata_dict
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"sync_cache_from_metadata failed for %s",
|
||||||
|
file_path,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def _sync_cache_from_metadata_impl(
|
||||||
|
self, file_path: str, metadata_dict: Dict[str, Any]
|
||||||
|
) -> bool:
|
||||||
|
cache = await self.get_cached_data()
|
||||||
|
|
||||||
|
# Locate the existing cache entry -----------------------------------
|
||||||
|
existing_idx: Optional[int] = None
|
||||||
|
existing_entry: Optional[Dict[str, Any]] = None
|
||||||
|
for i, item in enumerate(cache.raw_data):
|
||||||
|
if item.get("file_path") == file_path:
|
||||||
|
existing_entry = item
|
||||||
|
existing_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
# Build the desired entry from metadata ------------------------------
|
||||||
|
folder_value = (
|
||||||
|
existing_entry.get("folder", "")
|
||||||
|
if existing_entry
|
||||||
|
else self._calculate_folder(file_path)
|
||||||
|
)
|
||||||
|
desired_entry = self._build_cache_entry(
|
||||||
|
metadata_dict,
|
||||||
|
folder=folder_value,
|
||||||
|
file_path_override=file_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ensure sha256 is populated (defensive — metadata should have it)
|
||||||
|
if (
|
||||||
|
not desired_entry.get("sha256")
|
||||||
|
and file_path
|
||||||
|
and os.path.exists(file_path)
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
sha256 = await calculate_sha256(file_path)
|
||||||
|
if sha256:
|
||||||
|
desired_entry["sha256"] = sha256.lower()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Not in cache at all — delegate to the full update path ------------
|
||||||
|
if existing_entry is None:
|
||||||
|
result = await self.update_single_model_cache(
|
||||||
|
file_path, file_path, metadata_dict
|
||||||
|
)
|
||||||
|
return bool(result)
|
||||||
|
|
||||||
|
# Compare — skip everything if already in sync -----------------------
|
||||||
|
if not self._cache_entries_differ(existing_entry, desired_entry):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Re-validate: the cache may have been replaced concurrently
|
||||||
|
# (e.g. by _apply_scan_result). Use identity check, not equality,
|
||||||
|
# so we detect when the raw_data list was swapped out from under us.
|
||||||
|
if self._cache is None or not any(
|
||||||
|
item is existing_entry for item in self._cache.raw_data
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# ---- Differences detected: apply targeted, in-place updates --------
|
||||||
|
|
||||||
|
# Snapshot old values for delta computations
|
||||||
|
old_tags = list(existing_entry.get("tags") or [])
|
||||||
|
old_sha256: str = existing_entry.get("sha256", "") or ""
|
||||||
|
old_model_name: str = existing_entry.get("model_name", "") or ""
|
||||||
|
old_file_name: str = existing_entry.get("file_name", "") or ""
|
||||||
|
old_modified: float = float(existing_entry.get("modified", 0.0) or 0.0)
|
||||||
|
old_size: int = int(existing_entry.get("size", 0) or 0)
|
||||||
|
old_civitai = existing_entry.get("civitai")
|
||||||
|
|
||||||
|
# ---- In-place update of the cache entry ----
|
||||||
|
existing_entry.clear()
|
||||||
|
existing_entry.update(desired_entry)
|
||||||
|
|
||||||
|
# ---- Incremental tag count update ----
|
||||||
|
new_tags: set = set(desired_entry.get("tags") or [])
|
||||||
|
old_tag_set: set = set(old_tags)
|
||||||
|
for tag in old_tag_set - new_tags:
|
||||||
|
current = self._tags_count.get(tag, 0)
|
||||||
|
if current <= 1:
|
||||||
|
self._tags_count.pop(tag, None)
|
||||||
|
else:
|
||||||
|
self._tags_count[tag] = current - 1
|
||||||
|
for tag in new_tags - old_tag_set:
|
||||||
|
self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
|
||||||
|
|
||||||
|
# ---- Incremental hash index update ----
|
||||||
|
new_sha = (desired_entry.get("sha256", "") or "").lower()
|
||||||
|
old_sha = (old_sha256 or "").lower()
|
||||||
|
if new_sha != old_sha:
|
||||||
|
if old_sha:
|
||||||
|
self._hash_index.remove_by_path(file_path)
|
||||||
|
if new_sha:
|
||||||
|
self._hash_index.add_entry(new_sha, file_path)
|
||||||
|
|
||||||
|
# ---- Incremental version index update ----
|
||||||
|
new_civitai = desired_entry.get("civitai")
|
||||||
|
if old_civitai != new_civitai:
|
||||||
|
temp_old = {
|
||||||
|
"file_path": file_path,
|
||||||
|
"file_name": old_file_name,
|
||||||
|
"civitai": old_civitai,
|
||||||
|
}
|
||||||
|
cache.remove_from_version_index(temp_old)
|
||||||
|
cache.add_to_version_index(existing_entry)
|
||||||
|
|
||||||
|
# ---- Conditional resort (only when sort-key fields changed) ----
|
||||||
|
need_resort = False
|
||||||
|
_last = cache._last_sort
|
||||||
|
sort_key: Optional[str] = _last[0] if _last != (None, None) else None
|
||||||
|
if sort_key == "name":
|
||||||
|
if (
|
||||||
|
old_model_name != desired_entry.get("model_name", "")
|
||||||
|
or old_file_name != desired_entry.get("file_name", "")
|
||||||
|
):
|
||||||
|
need_resort = True
|
||||||
|
elif sort_key == "date":
|
||||||
|
if old_modified != float(desired_entry.get("modified", 0.0) or 0.0):
|
||||||
|
need_resort = True
|
||||||
|
elif sort_key == "size":
|
||||||
|
if old_size != int(desired_entry.get("size", 0) or 0):
|
||||||
|
need_resort = True
|
||||||
|
|
||||||
|
if need_resort:
|
||||||
|
await cache.resort()
|
||||||
|
|
||||||
|
# ---- Targeted SQL update (single row, not full save_cache) ----
|
||||||
|
persistent = getattr(self, "_persistent_cache", None)
|
||||||
|
if persistent is not None:
|
||||||
|
old_item_for_sql: Dict[str, Any] = {
|
||||||
|
"file_path": file_path,
|
||||||
|
"tags": old_tags,
|
||||||
|
"sha256": old_sha256,
|
||||||
|
}
|
||||||
|
await asyncio.get_event_loop().run_in_executor(
|
||||||
|
None,
|
||||||
|
persistent.update_single_model,
|
||||||
|
self.model_type,
|
||||||
|
desired_entry,
|
||||||
|
old_item_for_sql,
|
||||||
|
)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _cache_entries_differ(a: Dict[str, Any], b: Dict[str, Any]) -> bool:
|
||||||
|
"""Return ``True`` when two cache-entry dicts differ in any field.
|
||||||
|
|
||||||
|
Tag lists are compared order-insensitively; all other keys use
|
||||||
|
standard equality.
|
||||||
|
"""
|
||||||
|
a_tags = sorted(a.get("tags") or [])
|
||||||
|
b_tags = sorted(b.get("tags") or [])
|
||||||
|
if a_tags != b_tags:
|
||||||
|
return True
|
||||||
|
|
||||||
|
all_keys = set(a.keys()) | set(b.keys())
|
||||||
|
for key in all_keys:
|
||||||
|
if key == "tags":
|
||||||
|
continue
|
||||||
|
if a.get(key) != b.get(key):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def has_hash(self, sha256: str) -> bool:
|
def has_hash(self, sha256: str) -> bool:
|
||||||
"""Check if a model with given hash exists"""
|
"""Check if a model with given hash exists"""
|
||||||
return self._hash_index.has_hash(sha256.lower())
|
return self._hash_index.has_hash(sha256.lower())
|
||||||
|
|||||||
@@ -587,6 +587,95 @@ class PersistentModelCache:
|
|||||||
placeholders = ", ".join(["?"] * len(self._MODEL_COLUMNS))
|
placeholders = ", ".join(["?"] * len(self._MODEL_COLUMNS))
|
||||||
return f"INSERT INTO models ({columns}) VALUES ({placeholders})"
|
return f"INSERT INTO models ({columns}) VALUES ({placeholders})"
|
||||||
|
|
||||||
|
def update_single_model(
|
||||||
|
self,
|
||||||
|
model_type: str,
|
||||||
|
new_item: Dict,
|
||||||
|
old_item: Optional[Dict] = None,
|
||||||
|
) -> None:
|
||||||
|
"""Update a single model row in the persistent cache.
|
||||||
|
|
||||||
|
A lightweight alternative to :meth:`save_cache` that performs a targeted
|
||||||
|
DELETE + INSERT for the model row and computes incremental tag / hash-index
|
||||||
|
deltas from *old_item*. When *old_item* is omitted the previous tags and
|
||||||
|
hash are not cleaned up (callers should only omit it for brand-new entries).
|
||||||
|
|
||||||
|
All operations run inside a single transaction so readers see a consistent
|
||||||
|
view.
|
||||||
|
"""
|
||||||
|
if not self.is_enabled():
|
||||||
|
return
|
||||||
|
if not self._schema_initialized:
|
||||||
|
self._initialize_schema()
|
||||||
|
if not self._schema_initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
file_path: Optional[str] = new_item.get("file_path")
|
||||||
|
if not file_path:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute("PRAGMA foreign_keys = ON")
|
||||||
|
conn.execute("BEGIN")
|
||||||
|
|
||||||
|
# --- model row (DELETE + INSERT = upsert) ---
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||||
|
(model_type, file_path),
|
||||||
|
)
|
||||||
|
row = self._prepare_model_row(model_type, new_item)
|
||||||
|
conn.execute(self._insert_model_sql(), row)
|
||||||
|
|
||||||
|
# --- tags ---
|
||||||
|
new_tags: set = set(new_item.get("tags") or [])
|
||||||
|
old_tags: set = set(old_item.get("tags") or []) if old_item else set()
|
||||||
|
tags_to_delete = old_tags - new_tags
|
||||||
|
tags_to_insert = new_tags - old_tags
|
||||||
|
|
||||||
|
if tags_to_delete:
|
||||||
|
conn.executemany(
|
||||||
|
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||||
|
[(model_type, file_path, t) for t in tags_to_delete],
|
||||||
|
)
|
||||||
|
if tags_to_insert:
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||||
|
[(model_type, file_path, t) for t in tags_to_insert],
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- hash_index ---
|
||||||
|
new_sha: Optional[str] = (new_item.get("sha256") or "").lower() or None
|
||||||
|
old_sha: Optional[str] = (
|
||||||
|
(old_item.get("sha256") or "").lower() or None
|
||||||
|
) if old_item else None
|
||||||
|
if new_sha != old_sha:
|
||||||
|
if old_sha:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||||
|
(model_type, old_sha, file_path),
|
||||||
|
)
|
||||||
|
if new_sha:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||||
|
(model_type, new_sha, file_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.execute("COMMIT")
|
||||||
|
except Exception:
|
||||||
|
conn.execute("ROLLBACK")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to update single model in persistent cache (%s): %s",
|
||||||
|
file_path,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
def _load_tags(self, conn: sqlite3.Connection, model_type: str) -> Dict[str, List[str]]:
|
def _load_tags(self, conn: sqlite3.Connection, model_type: str) -> Dict[str, List[str]]:
|
||||||
tag_rows = conn.execute(
|
tag_rows = conn.execute(
|
||||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||||
|
|||||||
@@ -667,3 +667,310 @@ async def test_log_duplicate_filename_summary_silent_when_no_duplicates(tmp_path
|
|||||||
# No warning should be logged when there are no duplicates
|
# No warning should be logged when there are no duplicates
|
||||||
for record in caplog.records:
|
for record in caplog.records:
|
||||||
assert "Duplicate filename conflict detected" not in record.message
|
assert "Duplicate filename conflict detected" not in record.message
|
||||||
|
|
||||||
|
|
||||||
|
# ── _cache_entries_differ ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"a_tags, b_tags, expect_differ",
|
||||||
|
[
|
||||||
|
(["alpha", "beta"], ["beta", "alpha"], False), # order-insensitive
|
||||||
|
(["alpha"], ["alpha", "beta"], True), # count differs
|
||||||
|
([], ["alpha"], True),
|
||||||
|
(None, [], False), # None ≈ []
|
||||||
|
(["alpha"], None, True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cache_entries_differ_tags(a_tags, b_tags, expect_differ):
|
||||||
|
base = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
|
||||||
|
entry_a = {**base, "tags": a_tags}
|
||||||
|
entry_b = {**base, "tags": b_tags}
|
||||||
|
assert ModelScanner._cache_entries_differ(entry_a, entry_b) == expect_differ
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_entries_differ_identical():
|
||||||
|
entry = {
|
||||||
|
"file_path": "/m/a.safetensors", "model_name": "A", "size": 1,
|
||||||
|
"tags": ["x"], "civitai": {"id": 1}, "notes": "hi",
|
||||||
|
}
|
||||||
|
assert ModelScanner._cache_entries_differ(entry, dict(entry)) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_entries_differ_field_changed():
|
||||||
|
a = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
|
||||||
|
b = {**a, "model_name": "B"}
|
||||||
|
assert ModelScanner._cache_entries_differ(a, b) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_entries_differ_extra_key():
|
||||||
|
a = {"file_path": "/m/a.safetensors", "model_name": "A"}
|
||||||
|
b = {**a, "extra_field": "value"}
|
||||||
|
assert ModelScanner._cache_entries_differ(a, b) is True
|
||||||
|
|
||||||
|
|
||||||
|
# ── sync_cache_from_metadata ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_cache_entry(**overrides) -> dict:
|
||||||
|
entry = {
|
||||||
|
"file_path": "/m/a.safetensors",
|
||||||
|
"model_name": "TestModel",
|
||||||
|
"file_name": "a",
|
||||||
|
"folder": "",
|
||||||
|
"size": 100,
|
||||||
|
"modified": 10.0,
|
||||||
|
"sha256": "abc123",
|
||||||
|
"base_model": "SD1.5",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "old note",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"metadata_source": None,
|
||||||
|
"exclude": False,
|
||||||
|
"db_checked": False,
|
||||||
|
"last_checked_at": 0.0,
|
||||||
|
"tags": ["alpha"],
|
||||||
|
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||||
|
"civitai_deleted": False,
|
||||||
|
"skip_metadata_refresh": False,
|
||||||
|
"hf_url": "",
|
||||||
|
"license_flags": 113,
|
||||||
|
"hash_status": "completed",
|
||||||
|
}
|
||||||
|
entry.update(overrides)
|
||||||
|
return entry
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sync_cache_no_change(tmp_path: Path):
|
||||||
|
"""When metadata matches the cache entry, return False and mutate nothing."""
|
||||||
|
scanner = DummyScanner(tmp_path)
|
||||||
|
entry = _make_cache_entry()
|
||||||
|
scanner._cache = ModelCache(
|
||||||
|
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||||
|
)
|
||||||
|
await scanner._cache.resort()
|
||||||
|
scanner._tags_count = {"alpha": 1}
|
||||||
|
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||||
|
|
||||||
|
# metadata_dict that would produce the identical cache entry
|
||||||
|
metadata_dict = {
|
||||||
|
"file_path": "/m/a.safetensors",
|
||||||
|
"model_name": "TestModel",
|
||||||
|
"file_name": "a",
|
||||||
|
"folder": "",
|
||||||
|
"size": 100,
|
||||||
|
"modified": 10.0,
|
||||||
|
"sha256": "abc123",
|
||||||
|
"base_model": "SD1.5",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "old note",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"tags": ["alpha"],
|
||||||
|
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||||
|
"hf_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
changed = await scanner.sync_cache_from_metadata(
|
||||||
|
"/m/a.safetensors", metadata_dict
|
||||||
|
)
|
||||||
|
assert changed is False
|
||||||
|
# Verify cache was NOT mutated
|
||||||
|
cached = await scanner.get_cached_data()
|
||||||
|
assert cached.raw_data[0]["notes"] == "old note"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sync_cache_in_place_update(tmp_path: Path):
|
||||||
|
"""When metadata differs, update the cache entry in-place."""
|
||||||
|
scanner = DummyScanner(tmp_path)
|
||||||
|
entry = _make_cache_entry(notes="old note", tags=["alpha"], model_name="OldName")
|
||||||
|
scanner._cache = ModelCache(
|
||||||
|
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||||
|
)
|
||||||
|
await scanner._cache.resort()
|
||||||
|
scanner._tags_count = {"alpha": 1}
|
||||||
|
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||||
|
|
||||||
|
# Capture the exact dict object in raw_data before sync
|
||||||
|
original_entry_ref = scanner._cache.raw_data[0]
|
||||||
|
|
||||||
|
metadata_dict = {
|
||||||
|
"file_path": "/m/a.safetensors",
|
||||||
|
"model_name": "NewName",
|
||||||
|
"file_name": "a",
|
||||||
|
"folder": "",
|
||||||
|
"size": 100,
|
||||||
|
"modified": 10.0,
|
||||||
|
"sha256": "abc123",
|
||||||
|
"base_model": "SD1.5",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "new note",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"tags": ["beta", "gamma"],
|
||||||
|
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||||
|
"hf_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
changed = await scanner.sync_cache_from_metadata(
|
||||||
|
"/m/a.safetensors", metadata_dict
|
||||||
|
)
|
||||||
|
assert changed is True
|
||||||
|
|
||||||
|
cached = await scanner.get_cached_data()
|
||||||
|
updated = cached.raw_data[0]
|
||||||
|
# In-place: the same dict object persisted in raw_data
|
||||||
|
assert updated is original_entry_ref
|
||||||
|
assert updated["notes"] == "new note"
|
||||||
|
assert updated["model_name"] == "NewName"
|
||||||
|
assert sorted(updated["tags"]) == ["beta", "gamma"]
|
||||||
|
# Tag counts updated incrementally
|
||||||
|
assert scanner._tags_count.get("alpha", 0) == 0
|
||||||
|
assert scanner._tags_count.get("beta", 0) == 1
|
||||||
|
assert scanner._tags_count.get("gamma", 0) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sync_cache_not_in_cache_delegates(tmp_path: Path):
|
||||||
|
"""When the file_path is not in the cache at all, fall back to full update."""
|
||||||
|
scanner = DummyScanner(tmp_path)
|
||||||
|
scanner._cache = ModelCache(raw_data=[], folders=[], name_display_mode="model_name")
|
||||||
|
await scanner._cache.resort()
|
||||||
|
|
||||||
|
metadata_dict = {
|
||||||
|
"file_path": "/m/b.safetensors",
|
||||||
|
"model_name": "BrandNew",
|
||||||
|
"file_name": "b",
|
||||||
|
"folder": "",
|
||||||
|
"size": 200,
|
||||||
|
"modified": 20.0,
|
||||||
|
"sha256": "def456",
|
||||||
|
"base_model": "SDXL",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"tags": [],
|
||||||
|
"civitai": {},
|
||||||
|
"hf_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
changed = await scanner.sync_cache_from_metadata(
|
||||||
|
"/m/b.safetensors", metadata_dict
|
||||||
|
)
|
||||||
|
assert changed is True
|
||||||
|
cached = await scanner.get_cached_data()
|
||||||
|
assert len(cached.raw_data) == 1
|
||||||
|
assert cached.raw_data[0]["model_name"] == "BrandNew"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sync_cache_conditional_resort_skipped(tmp_path: Path, monkeypatch):
|
||||||
|
"""When only non-sort-key fields change, resort() is NOT called."""
|
||||||
|
scanner = DummyScanner(tmp_path)
|
||||||
|
entry = _make_cache_entry(notes="old note", model_name="SameName")
|
||||||
|
scanner._cache = ModelCache(
|
||||||
|
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||||
|
)
|
||||||
|
await scanner._cache.resort()
|
||||||
|
scanner._cache._last_sort = ("name", "asc") # name sort is active
|
||||||
|
scanner._tags_count = {"alpha": 1}
|
||||||
|
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||||
|
|
||||||
|
# Track resort calls
|
||||||
|
resort_called = False
|
||||||
|
original_resort = scanner._cache.resort
|
||||||
|
|
||||||
|
async def tracking_resort():
|
||||||
|
nonlocal resort_called
|
||||||
|
resort_called = True
|
||||||
|
await original_resort()
|
||||||
|
|
||||||
|
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
|
||||||
|
|
||||||
|
metadata_dict = {
|
||||||
|
"file_path": "/m/a.safetensors",
|
||||||
|
"model_name": "SameName", # unchanged — no resort needed
|
||||||
|
"file_name": "a",
|
||||||
|
"folder": "",
|
||||||
|
"size": 100,
|
||||||
|
"modified": 10.0,
|
||||||
|
"sha256": "abc123",
|
||||||
|
"base_model": "SD1.5",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "updated note", # changed, but not sort-relevant
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"tags": ["alpha"],
|
||||||
|
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||||
|
"hf_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
changed = await scanner.sync_cache_from_metadata(
|
||||||
|
"/m/a.safetensors", metadata_dict
|
||||||
|
)
|
||||||
|
assert changed is True
|
||||||
|
assert resort_called is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sync_cache_conditional_resort_triggered(tmp_path: Path, monkeypatch):
|
||||||
|
"""When the sort-key field changes, resort() IS called."""
|
||||||
|
scanner = DummyScanner(tmp_path)
|
||||||
|
entry = _make_cache_entry(model_name="OldName")
|
||||||
|
scanner._cache = ModelCache(
|
||||||
|
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||||
|
)
|
||||||
|
await scanner._cache.resort()
|
||||||
|
scanner._cache._last_sort = ("name", "asc")
|
||||||
|
scanner._tags_count = {"alpha": 1}
|
||||||
|
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||||
|
|
||||||
|
resort_calls = 0
|
||||||
|
original_resort = scanner._cache.resort
|
||||||
|
|
||||||
|
async def tracking_resort():
|
||||||
|
nonlocal resort_calls
|
||||||
|
resort_calls += 1
|
||||||
|
await original_resort()
|
||||||
|
|
||||||
|
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
|
||||||
|
|
||||||
|
metadata_dict = {
|
||||||
|
"file_path": "/m/a.safetensors",
|
||||||
|
"model_name": "NewName", # changed — should trigger resort
|
||||||
|
"file_name": "a",
|
||||||
|
"folder": "",
|
||||||
|
"size": 100,
|
||||||
|
"modified": 10.0,
|
||||||
|
"sha256": "abc123",
|
||||||
|
"base_model": "SD1.5",
|
||||||
|
"preview_url": "",
|
||||||
|
"preview_nsfw_level": 0,
|
||||||
|
"from_civitai": True,
|
||||||
|
"favorite": False,
|
||||||
|
"notes": "old note",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"tags": ["alpha"],
|
||||||
|
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||||
|
"hf_url": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
changed = await scanner.sync_cache_from_metadata(
|
||||||
|
"/m/a.safetensors", metadata_dict
|
||||||
|
)
|
||||||
|
assert changed is True
|
||||||
|
assert resort_calls == 1
|
||||||
|
|||||||
@@ -225,3 +225,119 @@ def test_incremental_updates_only_touch_changed_rows(tmp_path: Path, monkeypatch
|
|||||||
assert second['metadata_source'] == 'archive_db'
|
assert second['metadata_source'] == 'archive_db'
|
||||||
assert second['civitai_deleted'] is True
|
assert second['civitai_deleted'] is True
|
||||||
assert second['civitai']['creator']['username'] == 'builder_v2'
|
assert second['civitai']['creator']['username'] == 'builder_v2'
|
||||||
|
|
||||||
|
|
||||||
|
# ── update_single_model ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_single_model_insert(tmp_path: Path, monkeypatch):
|
||||||
|
"""Insert a brand-new model row via update_single_model."""
|
||||||
|
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||||
|
db_path = tmp_path / 'cache.sqlite'
|
||||||
|
store = PersistentModelCache(db_path=str(db_path))
|
||||||
|
|
||||||
|
file_path = (tmp_path / 'x.safetensors').as_posix()
|
||||||
|
new_item = {
|
||||||
|
'file_path': file_path,
|
||||||
|
'file_name': 'x',
|
||||||
|
'model_name': 'Model X',
|
||||||
|
'folder': '',
|
||||||
|
'size': 42,
|
||||||
|
'modified': 1.0,
|
||||||
|
'sha256': 'sha-x',
|
||||||
|
'base_model': 'SDXL',
|
||||||
|
'preview_url': '',
|
||||||
|
'preview_nsfw_level': 0,
|
||||||
|
'from_civitai': True,
|
||||||
|
'favorite': True,
|
||||||
|
'notes': 'test note',
|
||||||
|
'usage_tips': '{}',
|
||||||
|
'metadata_source': None,
|
||||||
|
'exclude': False,
|
||||||
|
'db_checked': False,
|
||||||
|
'last_checked_at': 0.0,
|
||||||
|
'tags': ['test', 'new'],
|
||||||
|
'civitai': None,
|
||||||
|
'civitai_deleted': False,
|
||||||
|
'skip_metadata_refresh': False,
|
||||||
|
'license_flags': DEFAULT_LICENSE_FLAGS,
|
||||||
|
'hash_status': 'completed',
|
||||||
|
'hf_url': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
store.update_single_model('dummy', new_item)
|
||||||
|
|
||||||
|
persisted = store.load_cache('dummy')
|
||||||
|
assert persisted is not None
|
||||||
|
items = {item['file_path']: item for item in persisted.raw_data}
|
||||||
|
assert file_path in items
|
||||||
|
assert items[file_path]['model_name'] == 'Model X'
|
||||||
|
assert items[file_path]['favorite'] is True
|
||||||
|
assert sorted(items[file_path]['tags']) == ['new', 'test']
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_single_model_update_tags(tmp_path: Path, monkeypatch):
|
||||||
|
"""Tags are updated incrementally: old tags removed, new tags added."""
|
||||||
|
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||||
|
db_path = tmp_path / 'cache.sqlite'
|
||||||
|
store = PersistentModelCache(db_path=str(db_path))
|
||||||
|
|
||||||
|
file_path = (tmp_path / 'y.safetensors').as_posix()
|
||||||
|
base = {
|
||||||
|
'file_path': file_path, 'file_name': 'y', 'model_name': 'Y',
|
||||||
|
'folder': '', 'size': 1, 'modified': 1.0, 'sha256': 'sha-y',
|
||||||
|
'base_model': '', 'preview_url': '', 'preview_nsfw_level': 0,
|
||||||
|
'from_civitai': True, 'favorite': False, 'notes': '', 'usage_tips': '{}',
|
||||||
|
'metadata_source': None, 'exclude': False, 'db_checked': False,
|
||||||
|
'last_checked_at': 0.0, 'civitai': None, 'civitai_deleted': False,
|
||||||
|
'skip_metadata_refresh': False, 'license_flags': DEFAULT_LICENSE_FLAGS,
|
||||||
|
'hash_status': 'completed', 'hf_url': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
# First insert with tags [alpha, beta]
|
||||||
|
store.update_single_model('dummy', {**base, 'tags': ['alpha', 'beta']})
|
||||||
|
|
||||||
|
# Now update: replace with [beta, gamma]
|
||||||
|
old_item = {'file_path': file_path, 'tags': ['alpha', 'beta'], 'sha256': 'sha-y'}
|
||||||
|
new_item = {**base, 'tags': ['beta', 'gamma']}
|
||||||
|
store.update_single_model('dummy', new_item, old_item=old_item)
|
||||||
|
|
||||||
|
persisted = store.load_cache('dummy')
|
||||||
|
assert persisted is not None
|
||||||
|
items = {item['file_path']: item for item in persisted.raw_data}
|
||||||
|
assert sorted(items[file_path]['tags']) == ['beta', 'gamma']
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_single_model_update_hash(tmp_path: Path, monkeypatch):
|
||||||
|
"""When sha256 changes, the hash_index is updated incrementally."""
|
||||||
|
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||||
|
db_path = tmp_path / 'cache.sqlite'
|
||||||
|
store = PersistentModelCache(db_path=str(db_path))
|
||||||
|
|
||||||
|
file_path = (tmp_path / 'z.safetensors').as_posix()
|
||||||
|
base = {
|
||||||
|
'file_path': file_path, 'file_name': 'z', 'model_name': 'Z',
|
||||||
|
'folder': '', 'size': 1, 'modified': 1.0, 'base_model': '',
|
||||||
|
'preview_url': '', 'preview_nsfw_level': 0, 'from_civitai': True,
|
||||||
|
'favorite': False, 'notes': '', 'usage_tips': '{}',
|
||||||
|
'metadata_source': None, 'exclude': False, 'db_checked': False,
|
||||||
|
'last_checked_at': 0.0, 'tags': [], 'civitai': None,
|
||||||
|
'civitai_deleted': False, 'skip_metadata_refresh': False,
|
||||||
|
'license_flags': DEFAULT_LICENSE_FLAGS, 'hash_status': 'completed', 'hf_url': '',
|
||||||
|
}
|
||||||
|
|
||||||
|
store.update_single_model('dummy', {**base, 'sha256': 'old-hash'})
|
||||||
|
|
||||||
|
old_item = {'file_path': file_path, 'tags': [], 'sha256': 'old-hash'}
|
||||||
|
new_item = {**base, 'sha256': 'new-hash'}
|
||||||
|
store.update_single_model('dummy', new_item, old_item=old_item)
|
||||||
|
|
||||||
|
persisted = store.load_cache('dummy')
|
||||||
|
assert persisted is not None
|
||||||
|
# old hash should be gone from hash_index
|
||||||
|
old_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'old-hash']
|
||||||
|
assert len(old_hash_pairs) == 0
|
||||||
|
# new hash should be present
|
||||||
|
new_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'new-hash']
|
||||||
|
assert len(new_hash_pairs) == 1
|
||||||
|
assert new_hash_pairs[0][1] == file_path
|
||||||
|
|||||||
Reference in New Issue
Block a user