alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-08-06 14:10:13 -03:00
Code Issues Packages Projects Releases Wiki Activity

feat(cache): opportunistic cache sync on metadata read with in-place update

Browse Source 
- Add PersistentModelCache.update_single_model() for lightweight targeted
  SQL update (single row + incremental tag/hash deltas, no full table scan)
- Add ModelScanner.sync_cache_from_metadata() with compare-first logic:
  skips entirely when cache is already in sync; when stale, updates the
  entry in-place (O(1) instead of O(n) remove+append), incrementally
  adjusts tag counts/hash index/version index, and resorts only when
  sort-relevant fields changed
- Wire sync_cache_from_metadata() into BaseModelService.get_model_metadata()
  via fire-and-forget asyncio.create_task — disk I/O is already paid for
- Include identity re-validation guard against concurrent cache replacement
- Add 16 tests covering _cache_entries_differ, sync_cache_from_metadata
  (no-change, in-place, fallback, conditional resort), and
  update_single_model (insert, tag delta, hash delta)
This commit is contained in:
Will Miao
2026-07-19 08:32:21 +08:00
parent d15a8aa9a2
commit c27e4d1bfc
5 changed files with 742 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

307
tests/services/test_model_scanner.py
View File

@@ -667,3 +667,310 @@ async def test_log_duplicate_filename_summary_silent_when_no_duplicates(tmp_path
# No warning should be logged when there are no duplicates
for record in caplog.records:
assert "Duplicate filename conflict detected" not in record.message
# ── _cache_entries_differ ────────────────────────────────────────────
@pytest.mark.parametrize(
"a_tags, b_tags, expect_differ",
[
(["alpha", "beta"], ["beta", "alpha"], False), # order-insensitive
(["alpha"], ["alpha", "beta"], True), # count differs
([], ["alpha"], True),
(None, [], False), # None ≈ []
(["alpha"], None, True),
],
)
def test_cache_entries_differ_tags(a_tags, b_tags, expect_differ):
base = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
entry_a = {**base, "tags": a_tags}
entry_b = {**base, "tags": b_tags}
assert ModelScanner._cache_entries_differ(entry_a, entry_b) == expect_differ
def test_cache_entries_differ_identical():
entry = {
"file_path": "/m/a.safetensors", "model_name": "A", "size": 1,
"tags": ["x"], "civitai": {"id": 1}, "notes": "hi",
}
assert ModelScanner._cache_entries_differ(entry, dict(entry)) is False
def test_cache_entries_differ_field_changed():
a = {"file_path": "/m/a.safetensors", "model_name": "A", "size": 1}
b = {**a, "model_name": "B"}
assert ModelScanner._cache_entries_differ(a, b) is True
def test_cache_entries_differ_extra_key():
a = {"file_path": "/m/a.safetensors", "model_name": "A"}
b = {**a, "extra_field": "value"}
assert ModelScanner._cache_entries_differ(a, b) is True
# ── sync_cache_from_metadata ─────────────────────────────────────────
def _make_cache_entry(**overrides) -> dict:
entry = {
"file_path": "/m/a.safetensors",
"model_name": "TestModel",
"file_name": "a",
"folder": "",
"size": 100,
"modified": 10.0,
"sha256": "abc123",
"base_model": "SD1.5",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "old note",
"usage_tips": "{}",
"metadata_source": None,
"exclude": False,
"db_checked": False,
"last_checked_at": 0.0,
"tags": ["alpha"],
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
"civitai_deleted": False,
"skip_metadata_refresh": False,
"hf_url": "",
"license_flags": 113,
"hash_status": "completed",
}
entry.update(overrides)
return entry
@pytest.mark.asyncio
async def test_sync_cache_no_change(tmp_path: Path):
"""When metadata matches the cache entry, return False and mutate nothing."""
scanner = DummyScanner(tmp_path)
entry = _make_cache_entry()
scanner._cache = ModelCache(
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
# metadata_dict that would produce the identical cache entry
metadata_dict = {
"file_path": "/m/a.safetensors",
"model_name": "TestModel",
"file_name": "a",
"folder": "",
"size": 100,
"modified": 10.0,
"sha256": "abc123",
"base_model": "SD1.5",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "old note",
"usage_tips": "{}",
"tags": ["alpha"],
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
"hf_url": "",
}
changed = await scanner.sync_cache_from_metadata(
"/m/a.safetensors", metadata_dict
)
assert changed is False
# Verify cache was NOT mutated
cached = await scanner.get_cached_data()
assert cached.raw_data[0]["notes"] == "old note"
@pytest.mark.asyncio
async def test_sync_cache_in_place_update(tmp_path: Path):
"""When metadata differs, update the cache entry in-place."""
scanner = DummyScanner(tmp_path)
entry = _make_cache_entry(notes="old note", tags=["alpha"], model_name="OldName")
scanner._cache = ModelCache(
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
# Capture the exact dict object in raw_data before sync
original_entry_ref = scanner._cache.raw_data[0]
metadata_dict = {
"file_path": "/m/a.safetensors",
"model_name": "NewName",
"file_name": "a",
"folder": "",
"size": 100,
"modified": 10.0,
"sha256": "abc123",
"base_model": "SD1.5",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "new note",
"usage_tips": "{}",
"tags": ["beta", "gamma"],
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
"hf_url": "",
}
changed = await scanner.sync_cache_from_metadata(
"/m/a.safetensors", metadata_dict
)
assert changed is True
cached = await scanner.get_cached_data()
updated = cached.raw_data[0]
# In-place: the same dict object persisted in raw_data
assert updated is original_entry_ref
assert updated["notes"] == "new note"
assert updated["model_name"] == "NewName"
assert sorted(updated["tags"]) == ["beta", "gamma"]
# Tag counts updated incrementally
assert scanner._tags_count.get("alpha", 0) == 0
assert scanner._tags_count.get("beta", 0) == 1
assert scanner._tags_count.get("gamma", 0) == 1
@pytest.mark.asyncio
async def test_sync_cache_not_in_cache_delegates(tmp_path: Path):
"""When the file_path is not in the cache at all, fall back to full update."""
scanner = DummyScanner(tmp_path)
scanner._cache = ModelCache(raw_data=[], folders=[], name_display_mode="model_name")
await scanner._cache.resort()
metadata_dict = {
"file_path": "/m/b.safetensors",
"model_name": "BrandNew",
"file_name": "b",
"folder": "",
"size": 200,
"modified": 20.0,
"sha256": "def456",
"base_model": "SDXL",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "",
"usage_tips": "{}",
"tags": [],
"civitai": {},
"hf_url": "",
}
changed = await scanner.sync_cache_from_metadata(
"/m/b.safetensors", metadata_dict
)
assert changed is True
cached = await scanner.get_cached_data()
assert len(cached.raw_data) == 1
assert cached.raw_data[0]["model_name"] == "BrandNew"
@pytest.mark.asyncio
async def test_sync_cache_conditional_resort_skipped(tmp_path: Path, monkeypatch):
"""When only non-sort-key fields change, resort() is NOT called."""
scanner = DummyScanner(tmp_path)
entry = _make_cache_entry(notes="old note", model_name="SameName")
scanner._cache = ModelCache(
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._cache._last_sort = ("name", "asc") # name sort is active
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
# Track resort calls
resort_called = False
original_resort = scanner._cache.resort
async def tracking_resort():
nonlocal resort_called
resort_called = True
await original_resort()
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
metadata_dict = {
"file_path": "/m/a.safetensors",
"model_name": "SameName", # unchanged — no resort needed
"file_name": "a",
"folder": "",
"size": 100,
"modified": 10.0,
"sha256": "abc123",
"base_model": "SD1.5",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "updated note", # changed, but not sort-relevant
"usage_tips": "{}",
"tags": ["alpha"],
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
"hf_url": "",
}
changed = await scanner.sync_cache_from_metadata(
"/m/a.safetensors", metadata_dict
)
assert changed is True
assert resort_called is False
@pytest.mark.asyncio
async def test_sync_cache_conditional_resort_triggered(tmp_path: Path, monkeypatch):
"""When the sort-key field changes, resort() IS called."""
scanner = DummyScanner(tmp_path)
entry = _make_cache_entry(model_name="OldName")
scanner._cache = ModelCache(
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._cache._last_sort = ("name", "asc")
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
resort_calls = 0
original_resort = scanner._cache.resort
async def tracking_resort():
nonlocal resort_calls
resort_calls += 1
await original_resort()
monkeypatch.setattr(scanner._cache, "resort", tracking_resort)
metadata_dict = {
"file_path": "/m/a.safetensors",
"model_name": "NewName", # changed — should trigger resort
"file_name": "a",
"folder": "",
"size": 100,
"modified": 10.0,
"sha256": "abc123",
"base_model": "SD1.5",
"preview_url": "",
"preview_nsfw_level": 0,
"from_civitai": True,
"favorite": False,
"notes": "old note",
"usage_tips": "{}",
"tags": ["alpha"],
"civitai": {"id": 111, "modelId": 222, "name": "v1"},
"hf_url": "",
}
changed = await scanner.sync_cache_from_metadata(
"/m/a.safetensors", metadata_dict
)
assert changed is True
assert resort_calls == 1

116
tests/services/test_persistent_model_cache.py
View File

@@ -225,3 +225,119 @@ def test_incremental_updates_only_touch_changed_rows(tmp_path: Path, monkeypatch
assert second['metadata_source'] == 'archive_db'
assert second['civitai_deleted'] is True
assert second['civitai']['creator']['username'] == 'builder_v2'
# ── update_single_model ───────────────────────────────────────────────
def test_update_single_model_insert(tmp_path: Path, monkeypatch):
"""Insert a brand-new model row via update_single_model."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'x.safetensors').as_posix()
new_item = {
'file_path': file_path,
'file_name': 'x',
'model_name': 'Model X',
'folder': '',
'size': 42,
'modified': 1.0,
'sha256': 'sha-x',
'base_model': 'SDXL',
'preview_url': '',
'preview_nsfw_level': 0,
'from_civitai': True,
'favorite': True,
'notes': 'test note',
'usage_tips': '{}',
'metadata_source': None,
'exclude': False,
'db_checked': False,
'last_checked_at': 0.0,
'tags': ['test', 'new'],
'civitai': None,
'civitai_deleted': False,
'skip_metadata_refresh': False,
'license_flags': DEFAULT_LICENSE_FLAGS,
'hash_status': 'completed',
'hf_url': '',
}
store.update_single_model('dummy', new_item)
persisted = store.load_cache('dummy')
assert persisted is not None
items = {item['file_path']: item for item in persisted.raw_data}
assert file_path in items
assert items[file_path]['model_name'] == 'Model X'
assert items[file_path]['favorite'] is True
assert sorted(items[file_path]['tags']) == ['new', 'test']
def test_update_single_model_update_tags(tmp_path: Path, monkeypatch):
"""Tags are updated incrementally: old tags removed, new tags added."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'y.safetensors').as_posix()
base = {
'file_path': file_path, 'file_name': 'y', 'model_name': 'Y',
'folder': '', 'size': 1, 'modified': 1.0, 'sha256': 'sha-y',
'base_model': '', 'preview_url': '', 'preview_nsfw_level': 0,
'from_civitai': True, 'favorite': False, 'notes': '', 'usage_tips': '{}',
'metadata_source': None, 'exclude': False, 'db_checked': False,
'last_checked_at': 0.0, 'civitai': None, 'civitai_deleted': False,
'skip_metadata_refresh': False, 'license_flags': DEFAULT_LICENSE_FLAGS,
'hash_status': 'completed', 'hf_url': '',
}
# First insert with tags [alpha, beta]
store.update_single_model('dummy', {**base, 'tags': ['alpha', 'beta']})
# Now update: replace with [beta, gamma]
old_item = {'file_path': file_path, 'tags': ['alpha', 'beta'], 'sha256': 'sha-y'}
new_item = {**base, 'tags': ['beta', 'gamma']}
store.update_single_model('dummy', new_item, old_item=old_item)
persisted = store.load_cache('dummy')
assert persisted is not None
items = {item['file_path']: item for item in persisted.raw_data}
assert sorted(items[file_path]['tags']) == ['beta', 'gamma']
def test_update_single_model_update_hash(tmp_path: Path, monkeypatch):
"""When sha256 changes, the hash_index is updated incrementally."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'z.safetensors').as_posix()
base = {
'file_path': file_path, 'file_name': 'z', 'model_name': 'Z',
'folder': '', 'size': 1, 'modified': 1.0, 'base_model': '',
'preview_url': '', 'preview_nsfw_level': 0, 'from_civitai': True,
'favorite': False, 'notes': '', 'usage_tips': '{}',
'metadata_source': None, 'exclude': False, 'db_checked': False,
'last_checked_at': 0.0, 'tags': [], 'civitai': None,
'civitai_deleted': False, 'skip_metadata_refresh': False,
'license_flags': DEFAULT_LICENSE_FLAGS, 'hash_status': 'completed', 'hf_url': '',
}
store.update_single_model('dummy', {**base, 'sha256': 'old-hash'})
old_item = {'file_path': file_path, 'tags': [], 'sha256': 'old-hash'}
new_item = {**base, 'sha256': 'new-hash'}
store.update_single_model('dummy', new_item, old_item=old_item)
persisted = store.load_cache('dummy')
assert persisted is not None
# old hash should be gone from hash_index
old_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'old-hash']
assert len(old_hash_pairs) == 0
# new hash should be present
new_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'new-hash']
assert len(new_hash_pairs) == 1
assert new_hash_pairs[0][1] == file_path