mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-06 22:10:14 -03:00
feat(cache): opportunistic cache sync on metadata read with in-place update
- Add PersistentModelCache.update_single_model() for lightweight targeted SQL update (single row + incremental tag/hash deltas, no full table scan) - Add ModelScanner.sync_cache_from_metadata() with compare-first logic: skips entirely when cache is already in sync; when stale, updates the entry in-place (O(1) instead of O(n) remove+append), incrementally adjusts tag counts/hash index/version index, and resorts only when sort-relevant fields changed - Wire sync_cache_from_metadata() into BaseModelService.get_model_metadata() via fire-and-forget asyncio.create_task — disk I/O is already paid for - Include identity re-validation guard against concurrent cache replacement - Add 16 tests covering _cache_entries_differ, sync_cache_from_metadata (no-change, in-place, fallback, conditional resort), and update_single_model (insert, tag delta, hash delta)
This commit is contained in:
@@ -1092,6 +1092,11 @@ class BaseModelService(ABC):
|
||||
|
||||
Listing/search endpoints return lightweight cache entries; this method performs
|
||||
a lazy read of the on-disk metadata snapshot when callers need full detail.
|
||||
|
||||
As a beneficial side effect, the in-memory and persistent caches are
|
||||
opportunistically synchronised with the on-disk metadata — this keeps the
|
||||
caches fresh even when a ``.metadata.json`` file was edited outside of the
|
||||
normal save path (e.g. manually or by an external script).
|
||||
"""
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path, self.metadata_class
|
||||
@@ -1109,6 +1114,19 @@ class BaseModelService(ABC):
|
||||
MetadataManager.save_metadata(file_path, metadata)
|
||||
)
|
||||
|
||||
# Opportunistically sync the in-memory + persistent caches.
|
||||
# The .metadata.json disk read is already paid for; the sync only
|
||||
# performs work when the cache is actually stale, and uses targeted,
|
||||
# in-place operations to minimise overhead even with large model sets.
|
||||
#
|
||||
# Fire-and-forget by design: the task is intentionally untracked.
|
||||
# sync_cache_from_metadata handles its own errors internally.
|
||||
asyncio.create_task(
|
||||
self.scanner.sync_cache_from_metadata(
|
||||
file_path, metadata.to_dict()
|
||||
)
|
||||
)
|
||||
|
||||
return self.filter_civitai_data(metadata.to_dict().get("civitai", {}))
|
||||
|
||||
async def get_model_description(self, file_path: str) -> Optional[str]:
|
||||
|
||||
@@ -1566,6 +1566,218 @@ class ModelScanner:
|
||||
|
||||
return cache_entry if metadata else True
|
||||
|
||||
async def sync_cache_from_metadata(
|
||||
self, file_path: str, metadata_dict: Dict[str, Any]
|
||||
) -> bool:
|
||||
"""Opportunistically sync in-memory and persistent caches from metadata.
|
||||
|
||||
Builds a prospective cache entry from *metadata_dict* (deserialized
|
||||
``.metadata.json`` content) and compares it against the current cache
|
||||
entry. When the two are already identical this method returns
|
||||
``False`` without touching anything — avoiding the overhead of
|
||||
``update_single_model_cache``, which always removes and re-inserts
|
||||
the entry, triggers a full resort, and persists via the heavyweight
|
||||
``save_cache()``.
|
||||
|
||||
When differences are detected the update is applied **in-place** with
|
||||
targeted operations:
|
||||
|
||||
* The existing ``raw_data`` entry is modified rather than removed and
|
||||
re-appended (O(1) instead of O(n)).
|
||||
* Tag counts and the hash index are updated incrementally.
|
||||
* The version index is rebuilt only for the affected entry.
|
||||
* ``resort()`` is called **only** when a sort-relevant field changed
|
||||
(``model_name`` / ``file_name`` for name-sort, ``modified`` for
|
||||
date-sort, ``size`` for size-sort).
|
||||
* The persistent (SQLite) cache receives a targeted single-row update
|
||||
via :meth:`PersistentModelCache.update_single_model` rather than a
|
||||
full-table ``save_cache()``.
|
||||
|
||||
Returns:
|
||||
``True`` if any cache update was performed, ``False`` if the
|
||||
caches were already in sync.
|
||||
|
||||
.. note::
|
||||
|
||||
This is a **best-effort** operation. Failures are logged but
|
||||
never propagated — callers should fire-and-forget via
|
||||
:func:`asyncio.create_task`.
|
||||
"""
|
||||
try:
|
||||
return await self._sync_cache_from_metadata_impl(
|
||||
file_path, metadata_dict
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"sync_cache_from_metadata failed for %s",
|
||||
file_path,
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
async def _sync_cache_from_metadata_impl(
|
||||
self, file_path: str, metadata_dict: Dict[str, Any]
|
||||
) -> bool:
|
||||
cache = await self.get_cached_data()
|
||||
|
||||
# Locate the existing cache entry -----------------------------------
|
||||
existing_idx: Optional[int] = None
|
||||
existing_entry: Optional[Dict[str, Any]] = None
|
||||
for i, item in enumerate(cache.raw_data):
|
||||
if item.get("file_path") == file_path:
|
||||
existing_entry = item
|
||||
existing_idx = i
|
||||
break
|
||||
|
||||
# Build the desired entry from metadata ------------------------------
|
||||
folder_value = (
|
||||
existing_entry.get("folder", "")
|
||||
if existing_entry
|
||||
else self._calculate_folder(file_path)
|
||||
)
|
||||
desired_entry = self._build_cache_entry(
|
||||
metadata_dict,
|
||||
folder=folder_value,
|
||||
file_path_override=file_path,
|
||||
)
|
||||
|
||||
# Ensure sha256 is populated (defensive — metadata should have it)
|
||||
if (
|
||||
not desired_entry.get("sha256")
|
||||
and file_path
|
||||
and os.path.exists(file_path)
|
||||
):
|
||||
try:
|
||||
sha256 = await calculate_sha256(file_path)
|
||||
if sha256:
|
||||
desired_entry["sha256"] = sha256.lower()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Not in cache at all — delegate to the full update path ------------
|
||||
if existing_entry is None:
|
||||
result = await self.update_single_model_cache(
|
||||
file_path, file_path, metadata_dict
|
||||
)
|
||||
return bool(result)
|
||||
|
||||
# Compare — skip everything if already in sync -----------------------
|
||||
if not self._cache_entries_differ(existing_entry, desired_entry):
|
||||
return False
|
||||
|
||||
# Re-validate: the cache may have been replaced concurrently
|
||||
# (e.g. by _apply_scan_result). Use identity check, not equality,
|
||||
# so we detect when the raw_data list was swapped out from under us.
|
||||
if self._cache is None or not any(
|
||||
item is existing_entry for item in self._cache.raw_data
|
||||
):
|
||||
return False
|
||||
|
||||
# ---- Differences detected: apply targeted, in-place updates --------
|
||||
|
||||
# Snapshot old values for delta computations
|
||||
old_tags = list(existing_entry.get("tags") or [])
|
||||
old_sha256: str = existing_entry.get("sha256", "") or ""
|
||||
old_model_name: str = existing_entry.get("model_name", "") or ""
|
||||
old_file_name: str = existing_entry.get("file_name", "") or ""
|
||||
old_modified: float = float(existing_entry.get("modified", 0.0) or 0.0)
|
||||
old_size: int = int(existing_entry.get("size", 0) or 0)
|
||||
old_civitai = existing_entry.get("civitai")
|
||||
|
||||
# ---- In-place update of the cache entry ----
|
||||
existing_entry.clear()
|
||||
existing_entry.update(desired_entry)
|
||||
|
||||
# ---- Incremental tag count update ----
|
||||
new_tags: set = set(desired_entry.get("tags") or [])
|
||||
old_tag_set: set = set(old_tags)
|
||||
for tag in old_tag_set - new_tags:
|
||||
current = self._tags_count.get(tag, 0)
|
||||
if current <= 1:
|
||||
self._tags_count.pop(tag, None)
|
||||
else:
|
||||
self._tags_count[tag] = current - 1
|
||||
for tag in new_tags - old_tag_set:
|
||||
self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
|
||||
|
||||
# ---- Incremental hash index update ----
|
||||
new_sha = (desired_entry.get("sha256", "") or "").lower()
|
||||
old_sha = (old_sha256 or "").lower()
|
||||
if new_sha != old_sha:
|
||||
if old_sha:
|
||||
self._hash_index.remove_by_path(file_path)
|
||||
if new_sha:
|
||||
self._hash_index.add_entry(new_sha, file_path)
|
||||
|
||||
# ---- Incremental version index update ----
|
||||
new_civitai = desired_entry.get("civitai")
|
||||
if old_civitai != new_civitai:
|
||||
temp_old = {
|
||||
"file_path": file_path,
|
||||
"file_name": old_file_name,
|
||||
"civitai": old_civitai,
|
||||
}
|
||||
cache.remove_from_version_index(temp_old)
|
||||
cache.add_to_version_index(existing_entry)
|
||||
|
||||
# ---- Conditional resort (only when sort-key fields changed) ----
|
||||
need_resort = False
|
||||
_last = cache._last_sort
|
||||
sort_key: Optional[str] = _last[0] if _last != (None, None) else None
|
||||
if sort_key == "name":
|
||||
if (
|
||||
old_model_name != desired_entry.get("model_name", "")
|
||||
or old_file_name != desired_entry.get("file_name", "")
|
||||
):
|
||||
need_resort = True
|
||||
elif sort_key == "date":
|
||||
if old_modified != float(desired_entry.get("modified", 0.0) or 0.0):
|
||||
need_resort = True
|
||||
elif sort_key == "size":
|
||||
if old_size != int(desired_entry.get("size", 0) or 0):
|
||||
need_resort = True
|
||||
|
||||
if need_resort:
|
||||
await cache.resort()
|
||||
|
||||
# ---- Targeted SQL update (single row, not full save_cache) ----
|
||||
persistent = getattr(self, "_persistent_cache", None)
|
||||
if persistent is not None:
|
||||
old_item_for_sql: Dict[str, Any] = {
|
||||
"file_path": file_path,
|
||||
"tags": old_tags,
|
||||
"sha256": old_sha256,
|
||||
}
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None,
|
||||
persistent.update_single_model,
|
||||
self.model_type,
|
||||
desired_entry,
|
||||
old_item_for_sql,
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _cache_entries_differ(a: Dict[str, Any], b: Dict[str, Any]) -> bool:
|
||||
"""Return ``True`` when two cache-entry dicts differ in any field.
|
||||
|
||||
Tag lists are compared order-insensitively; all other keys use
|
||||
standard equality.
|
||||
"""
|
||||
a_tags = sorted(a.get("tags") or [])
|
||||
b_tags = sorted(b.get("tags") or [])
|
||||
if a_tags != b_tags:
|
||||
return True
|
||||
|
||||
all_keys = set(a.keys()) | set(b.keys())
|
||||
for key in all_keys:
|
||||
if key == "tags":
|
||||
continue
|
||||
if a.get(key) != b.get(key):
|
||||
return True
|
||||
return False
|
||||
|
||||
def has_hash(self, sha256: str) -> bool:
|
||||
"""Check if a model with given hash exists"""
|
||||
return self._hash_index.has_hash(sha256.lower())
|
||||
|
||||
@@ -587,6 +587,95 @@ class PersistentModelCache:
|
||||
placeholders = ", ".join(["?"] * len(self._MODEL_COLUMNS))
|
||||
return f"INSERT INTO models ({columns}) VALUES ({placeholders})"
|
||||
|
||||
def update_single_model(
|
||||
self,
|
||||
model_type: str,
|
||||
new_item: Dict,
|
||||
old_item: Optional[Dict] = None,
|
||||
) -> None:
|
||||
"""Update a single model row in the persistent cache.
|
||||
|
||||
A lightweight alternative to :meth:`save_cache` that performs a targeted
|
||||
DELETE + INSERT for the model row and computes incremental tag / hash-index
|
||||
deltas from *old_item*. When *old_item* is omitted the previous tags and
|
||||
hash are not cleaned up (callers should only omit it for brand-new entries).
|
||||
|
||||
All operations run inside a single transaction so readers see a consistent
|
||||
view.
|
||||
"""
|
||||
if not self.is_enabled():
|
||||
return
|
||||
if not self._schema_initialized:
|
||||
self._initialize_schema()
|
||||
if not self._schema_initialized:
|
||||
return
|
||||
|
||||
file_path: Optional[str] = new_item.get("file_path")
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
|
||||
# --- model row (DELETE + INSERT = upsert) ---
|
||||
conn.execute(
|
||||
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||
(model_type, file_path),
|
||||
)
|
||||
row = self._prepare_model_row(model_type, new_item)
|
||||
conn.execute(self._insert_model_sql(), row)
|
||||
|
||||
# --- tags ---
|
||||
new_tags: set = set(new_item.get("tags") or [])
|
||||
old_tags: set = set(old_item.get("tags") or []) if old_item else set()
|
||||
tags_to_delete = old_tags - new_tags
|
||||
tags_to_insert = new_tags - old_tags
|
||||
|
||||
if tags_to_delete:
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||
[(model_type, file_path, t) for t in tags_to_delete],
|
||||
)
|
||||
if tags_to_insert:
|
||||
conn.executemany(
|
||||
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||
[(model_type, file_path, t) for t in tags_to_insert],
|
||||
)
|
||||
|
||||
# --- hash_index ---
|
||||
new_sha: Optional[str] = (new_item.get("sha256") or "").lower() or None
|
||||
old_sha: Optional[str] = (
|
||||
(old_item.get("sha256") or "").lower() or None
|
||||
) if old_item else None
|
||||
if new_sha != old_sha:
|
||||
if old_sha:
|
||||
conn.execute(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||
(model_type, old_sha, file_path),
|
||||
)
|
||||
if new_sha:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||
(model_type, new_sha, file_path),
|
||||
)
|
||||
|
||||
conn.execute("COMMIT")
|
||||
except Exception:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to update single model in persistent cache (%s): %s",
|
||||
file_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
def _load_tags(self, conn: sqlite3.Connection, model_type: str) -> Dict[str, List[str]]:
|
||||
tag_rows = conn.execute(
|
||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||
|
||||
@@ -667,3 +667,310 @@ async def test_log_duplicate_filename_summary_silent_when_no_duplicates(tmp_path
|
||||
# No warning should be logged when there are no duplicates
|
||||
for record in caplog.records:
|
||||
assert "Duplicate filename conflict detected" not in record.message
|
||||
|
||||
|
||||
# ── _cache_entries_differ ────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_tags, b_tags, expect_differ",
|
||||
[
|
||||
(["alpha", "beta"], ["beta", "alpha"], False), # order-insensitive
|
||||
(["alpha"], ["alpha", "beta"], True), # count differs
|
||||
([], ["alpha"], True),
|
||||
(None, [], False), # None ≈ []
|
||||
(["alpha"], None, True),
|
||||
],
|
||||
)
|
||||
def test_cache_entries_differ_tags(a_tags, b_tags, expect_differ):
|
||||
base = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
|
||||
entry_a = {**base, "tags": a_tags}
|
||||
entry_b = {**base, "tags": b_tags}
|
||||
assert ModelScanner._cache_entries_differ(entry_a, entry_b) == expect_differ
|
||||
|
||||
|
||||
def test_cache_entries_differ_identical():
|
||||
entry = {
|
||||
"file_path": "/m/a.safetensors", "model_name": "A", "size": 1,
|
||||
"tags": ["x"], "civitai": {"id": 1}, "notes": "hi",
|
||||
}
|
||||
assert ModelScanner._cache_entries_differ(entry, dict(entry)) is False
|
||||
|
||||
|
||||
def test_cache_entries_differ_field_changed():
|
||||
a = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
|
||||
b = {**a, "model_name": "B"}
|
||||
assert ModelScanner._cache_entries_differ(a, b) is True
|
||||
|
||||
|
||||
def test_cache_entries_differ_extra_key():
|
||||
a = {"file_path": "/m/a.safetensors", "model_name": "A"}
|
||||
b = {**a, "extra_field": "value"}
|
||||
assert ModelScanner._cache_entries_differ(a, b) is True
|
||||
|
||||
|
||||
# ── sync_cache_from_metadata ─────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_cache_entry(**overrides) -> dict:
|
||||
entry = {
|
||||
"file_path": "/m/a.safetensors",
|
||||
"model_name": "TestModel",
|
||||
"file_name": "a",
|
||||
"folder": "",
|
||||
"size": 100,
|
||||
"modified": 10.0,
|
||||
"sha256": "abc123",
|
||||
"base_model": "SD1.5",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "old note",
|
||||
"usage_tips": "{}",
|
||||
"metadata_source": None,
|
||||
"exclude": False,
|
||||
"db_checked": False,
|
||||
"last_checked_at": 0.0,
|
||||
"tags": ["alpha"],
|
||||
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||
"civitai_deleted": False,
|
||||
"skip_metadata_refresh": False,
|
||||
"hf_url": "",
|
||||
"license_flags": 113,
|
||||
"hash_status": "completed",
|
||||
}
|
||||
entry.update(overrides)
|
||||
return entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_cache_no_change(tmp_path: Path):
|
||||
"""When metadata matches the cache entry, return False and mutate nothing."""
|
||||
scanner = DummyScanner(tmp_path)
|
||||
entry = _make_cache_entry()
|
||||
scanner._cache = ModelCache(
|
||||
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||
)
|
||||
await scanner._cache.resort()
|
||||
scanner._tags_count = {"alpha": 1}
|
||||
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||
|
||||
# metadata_dict that would produce the identical cache entry
|
||||
metadata_dict = {
|
||||
"file_path": "/m/a.safetensors",
|
||||
"model_name": "TestModel",
|
||||
"file_name": "a",
|
||||
"folder": "",
|
||||
"size": 100,
|
||||
"modified": 10.0,
|
||||
"sha256": "abc123",
|
||||
"base_model": "SD1.5",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "old note",
|
||||
"usage_tips": "{}",
|
||||
"tags": ["alpha"],
|
||||
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||
"hf_url": "",
|
||||
}
|
||||
|
||||
changed = await scanner.sync_cache_from_metadata(
|
||||
"/m/a.safetensors", metadata_dict
|
||||
)
|
||||
assert changed is False
|
||||
# Verify cache was NOT mutated
|
||||
cached = await scanner.get_cached_data()
|
||||
assert cached.raw_data[0]["notes"] == "old note"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_cache_in_place_update(tmp_path: Path):
|
||||
"""When metadata differs, update the cache entry in-place."""
|
||||
scanner = DummyScanner(tmp_path)
|
||||
entry = _make_cache_entry(notes="old note", tags=["alpha"], model_name="OldName")
|
||||
scanner._cache = ModelCache(
|
||||
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||
)
|
||||
await scanner._cache.resort()
|
||||
scanner._tags_count = {"alpha": 1}
|
||||
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||
|
||||
# Capture the exact dict object in raw_data before sync
|
||||
original_entry_ref = scanner._cache.raw_data[0]
|
||||
|
||||
metadata_dict = {
|
||||
"file_path": "/m/a.safetensors",
|
||||
"model_name": "NewName",
|
||||
"file_name": "a",
|
||||
"folder": "",
|
||||
"size": 100,
|
||||
"modified": 10.0,
|
||||
"sha256": "abc123",
|
||||
"base_model": "SD1.5",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "new note",
|
||||
"usage_tips": "{}",
|
||||
"tags": ["beta", "gamma"],
|
||||
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||
"hf_url": "",
|
||||
}
|
||||
|
||||
changed = await scanner.sync_cache_from_metadata(
|
||||
"/m/a.safetensors", metadata_dict
|
||||
)
|
||||
assert changed is True
|
||||
|
||||
cached = await scanner.get_cached_data()
|
||||
updated = cached.raw_data[0]
|
||||
# In-place: the same dict object persisted in raw_data
|
||||
assert updated is original_entry_ref
|
||||
assert updated["notes"] == "new note"
|
||||
assert updated["model_name"] == "NewName"
|
||||
assert sorted(updated["tags"]) == ["beta", "gamma"]
|
||||
# Tag counts updated incrementally
|
||||
assert scanner._tags_count.get("alpha", 0) == 0
|
||||
assert scanner._tags_count.get("beta", 0) == 1
|
||||
assert scanner._tags_count.get("gamma", 0) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_cache_not_in_cache_delegates(tmp_path: Path):
|
||||
"""When the file_path is not in the cache at all, fall back to full update."""
|
||||
scanner = DummyScanner(tmp_path)
|
||||
scanner._cache = ModelCache(raw_data=[], folders=[], name_display_mode="model_name")
|
||||
await scanner._cache.resort()
|
||||
|
||||
metadata_dict = {
|
||||
"file_path": "/m/b.safetensors",
|
||||
"model_name": "BrandNew",
|
||||
"file_name": "b",
|
||||
"folder": "",
|
||||
"size": 200,
|
||||
"modified": 20.0,
|
||||
"sha256": "def456",
|
||||
"base_model": "SDXL",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "",
|
||||
"usage_tips": "{}",
|
||||
"tags": [],
|
||||
"civitai": {},
|
||||
"hf_url": "",
|
||||
}
|
||||
|
||||
changed = await scanner.sync_cache_from_metadata(
|
||||
"/m/b.safetensors", metadata_dict
|
||||
)
|
||||
assert changed is True
|
||||
cached = await scanner.get_cached_data()
|
||||
assert len(cached.raw_data) == 1
|
||||
assert cached.raw_data[0]["model_name"] == "BrandNew"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_cache_conditional_resort_skipped(tmp_path: Path, monkeypatch):
|
||||
"""When only non-sort-key fields change, resort() is NOT called."""
|
||||
scanner = DummyScanner(tmp_path)
|
||||
entry = _make_cache_entry(notes="old note", model_name="SameName")
|
||||
scanner._cache = ModelCache(
|
||||
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||
)
|
||||
await scanner._cache.resort()
|
||||
scanner._cache._last_sort = ("name", "asc") # name sort is active
|
||||
scanner._tags_count = {"alpha": 1}
|
||||
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||
|
||||
# Track resort calls
|
||||
resort_called = False
|
||||
original_resort = scanner._cache.resort
|
||||
|
||||
async def tracking_resort():
|
||||
nonlocal resort_called
|
||||
resort_called = True
|
||||
await original_resort()
|
||||
|
||||
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
|
||||
|
||||
metadata_dict = {
|
||||
"file_path": "/m/a.safetensors",
|
||||
"model_name": "SameName", # unchanged — no resort needed
|
||||
"file_name": "a",
|
||||
"folder": "",
|
||||
"size": 100,
|
||||
"modified": 10.0,
|
||||
"sha256": "abc123",
|
||||
"base_model": "SD1.5",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "updated note", # changed, but not sort-relevant
|
||||
"usage_tips": "{}",
|
||||
"tags": ["alpha"],
|
||||
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||
"hf_url": "",
|
||||
}
|
||||
|
||||
changed = await scanner.sync_cache_from_metadata(
|
||||
"/m/a.safetensors", metadata_dict
|
||||
)
|
||||
assert changed is True
|
||||
assert resort_called is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_cache_conditional_resort_triggered(tmp_path: Path, monkeypatch):
|
||||
"""When the sort-key field changes, resort() IS called."""
|
||||
scanner = DummyScanner(tmp_path)
|
||||
entry = _make_cache_entry(model_name="OldName")
|
||||
scanner._cache = ModelCache(
|
||||
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
|
||||
)
|
||||
await scanner._cache.resort()
|
||||
scanner._cache._last_sort = ("name", "asc")
|
||||
scanner._tags_count = {"alpha": 1}
|
||||
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
|
||||
|
||||
resort_calls = 0
|
||||
original_resort = scanner._cache.resort
|
||||
|
||||
async def tracking_resort():
|
||||
nonlocal resort_calls
|
||||
resort_calls += 1
|
||||
await original_resort()
|
||||
|
||||
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
|
||||
|
||||
metadata_dict = {
|
||||
"file_path": "/m/a.safetensors",
|
||||
"model_name": "NewName", # changed — should trigger resort
|
||||
"file_name": "a",
|
||||
"folder": "",
|
||||
"size": 100,
|
||||
"modified": 10.0,
|
||||
"sha256": "abc123",
|
||||
"base_model": "SD1.5",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"from_civitai": True,
|
||||
"favorite": False,
|
||||
"notes": "old note",
|
||||
"usage_tips": "{}",
|
||||
"tags": ["alpha"],
|
||||
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
|
||||
"hf_url": "",
|
||||
}
|
||||
|
||||
changed = await scanner.sync_cache_from_metadata(
|
||||
"/m/a.safetensors", metadata_dict
|
||||
)
|
||||
assert changed is True
|
||||
assert resort_calls == 1
|
||||
|
||||
@@ -225,3 +225,119 @@ def test_incremental_updates_only_touch_changed_rows(tmp_path: Path, monkeypatch
|
||||
assert second['metadata_source'] == 'archive_db'
|
||||
assert second['civitai_deleted'] is True
|
||||
assert second['civitai']['creator']['username'] == 'builder_v2'
|
||||
|
||||
|
||||
# ── update_single_model ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_update_single_model_insert(tmp_path: Path, monkeypatch):
|
||||
"""Insert a brand-new model row via update_single_model."""
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
db_path = tmp_path / 'cache.sqlite'
|
||||
store = PersistentModelCache(db_path=str(db_path))
|
||||
|
||||
file_path = (tmp_path / 'x.safetensors').as_posix()
|
||||
new_item = {
|
||||
'file_path': file_path,
|
||||
'file_name': 'x',
|
||||
'model_name': 'Model X',
|
||||
'folder': '',
|
||||
'size': 42,
|
||||
'modified': 1.0,
|
||||
'sha256': 'sha-x',
|
||||
'base_model': 'SDXL',
|
||||
'preview_url': '',
|
||||
'preview_nsfw_level': 0,
|
||||
'from_civitai': True,
|
||||
'favorite': True,
|
||||
'notes': 'test note',
|
||||
'usage_tips': '{}',
|
||||
'metadata_source': None,
|
||||
'exclude': False,
|
||||
'db_checked': False,
|
||||
'last_checked_at': 0.0,
|
||||
'tags': ['test', 'new'],
|
||||
'civitai': None,
|
||||
'civitai_deleted': False,
|
||||
'skip_metadata_refresh': False,
|
||||
'license_flags': DEFAULT_LICENSE_FLAGS,
|
||||
'hash_status': 'completed',
|
||||
'hf_url': '',
|
||||
}
|
||||
|
||||
store.update_single_model('dummy', new_item)
|
||||
|
||||
persisted = store.load_cache('dummy')
|
||||
assert persisted is not None
|
||||
items = {item['file_path']: item for item in persisted.raw_data}
|
||||
assert file_path in items
|
||||
assert items[file_path]['model_name'] == 'Model X'
|
||||
assert items[file_path]['favorite'] is True
|
||||
assert sorted(items[file_path]['tags']) == ['new', 'test']
|
||||
|
||||
|
||||
def test_update_single_model_update_tags(tmp_path: Path, monkeypatch):
|
||||
"""Tags are updated incrementally: old tags removed, new tags added."""
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
db_path = tmp_path / 'cache.sqlite'
|
||||
store = PersistentModelCache(db_path=str(db_path))
|
||||
|
||||
file_path = (tmp_path / 'y.safetensors').as_posix()
|
||||
base = {
|
||||
'file_path': file_path, 'file_name': 'y', 'model_name': 'Y',
|
||||
'folder': '', 'size': 1, 'modified': 1.0, 'sha256': 'sha-y',
|
||||
'base_model': '', 'preview_url': '', 'preview_nsfw_level': 0,
|
||||
'from_civitai': True, 'favorite': False, 'notes': '', 'usage_tips': '{}',
|
||||
'metadata_source': None, 'exclude': False, 'db_checked': False,
|
||||
'last_checked_at': 0.0, 'civitai': None, 'civitai_deleted': False,
|
||||
'skip_metadata_refresh': False, 'license_flags': DEFAULT_LICENSE_FLAGS,
|
||||
'hash_status': 'completed', 'hf_url': '',
|
||||
}
|
||||
|
||||
# First insert with tags [alpha, beta]
|
||||
store.update_single_model('dummy', {**base, 'tags': ['alpha', 'beta']})
|
||||
|
||||
# Now update: replace with [beta, gamma]
|
||||
old_item = {'file_path': file_path, 'tags': ['alpha', 'beta'], 'sha256': 'sha-y'}
|
||||
new_item = {**base, 'tags': ['beta', 'gamma']}
|
||||
store.update_single_model('dummy', new_item, old_item=old_item)
|
||||
|
||||
persisted = store.load_cache('dummy')
|
||||
assert persisted is not None
|
||||
items = {item['file_path']: item for item in persisted.raw_data}
|
||||
assert sorted(items[file_path]['tags']) == ['beta', 'gamma']
|
||||
|
||||
|
||||
def test_update_single_model_update_hash(tmp_path: Path, monkeypatch):
|
||||
"""When sha256 changes, the hash_index is updated incrementally."""
|
||||
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
|
||||
db_path = tmp_path / 'cache.sqlite'
|
||||
store = PersistentModelCache(db_path=str(db_path))
|
||||
|
||||
file_path = (tmp_path / 'z.safetensors').as_posix()
|
||||
base = {
|
||||
'file_path': file_path, 'file_name': 'z', 'model_name': 'Z',
|
||||
'folder': '', 'size': 1, 'modified': 1.0, 'base_model': '',
|
||||
'preview_url': '', 'preview_nsfw_level': 0, 'from_civitai': True,
|
||||
'favorite': False, 'notes': '', 'usage_tips': '{}',
|
||||
'metadata_source': None, 'exclude': False, 'db_checked': False,
|
||||
'last_checked_at': 0.0, 'tags': [], 'civitai': None,
|
||||
'civitai_deleted': False, 'skip_metadata_refresh': False,
|
||||
'license_flags': DEFAULT_LICENSE_FLAGS, 'hash_status': 'completed', 'hf_url': '',
|
||||
}
|
||||
|
||||
store.update_single_model('dummy', {**base, 'sha256': 'old-hash'})
|
||||
|
||||
old_item = {'file_path': file_path, 'tags': [], 'sha256': 'old-hash'}
|
||||
new_item = {**base, 'sha256': 'new-hash'}
|
||||
store.update_single_model('dummy', new_item, old_item=old_item)
|
||||
|
||||
persisted = store.load_cache('dummy')
|
||||
assert persisted is not None
|
||||
# old hash should be gone from hash_index
|
||||
old_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'old-hash']
|
||||
assert len(old_hash_pairs) == 0
|
||||
# new hash should be present
|
||||
new_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'new-hash']
|
||||
assert len(new_hash_pairs) == 1
|
||||
assert new_hash_pairs[0][1] == file_path
|
||||
|
||||
Reference in New Issue
Block a user