feat(cache): opportunistic cache sync on metadata read with in-place update

- Add PersistentModelCache.update_single_model() for lightweight targeted
  SQL update (single row + incremental tag/hash deltas, no full table scan)
- Add ModelScanner.sync_cache_from_metadata() with compare-first logic:
  skips entirely when cache is already in sync; when stale, updates the
  entry in-place (O(1) instead of O(n) remove+append), incrementally
  adjusts tag counts/hash index/version index, and resorts only when
  sort-relevant fields changed
- Wire sync_cache_from_metadata() into BaseModelService.get_model_metadata()
  via fire-and-forget asyncio.create_task — disk I/O is already paid for
- Include identity re-validation guard against concurrent cache replacement
- Add 16 tests covering _cache_entries_differ, sync_cache_from_metadata
  (no-change, in-place, fallback, conditional resort), and
  update_single_model (insert, tag delta, hash delta)
This commit is contained in:
Will Miao
2026-07-19 08:32:21 +08:00
parent d15a8aa9a2
commit c27e4d1bfc
5 changed files with 742 additions and 0 deletions

View File

@@ -225,3 +225,119 @@ def test_incremental_updates_only_touch_changed_rows(tmp_path: Path, monkeypatch
assert second['metadata_source'] == 'archive_db'
assert second['civitai_deleted'] is True
assert second['civitai']['creator']['username'] == 'builder_v2'
# ── update_single_model ───────────────────────────────────────────────
def test_update_single_model_insert(tmp_path: Path, monkeypatch):
"""Insert a brand-new model row via update_single_model."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'x.safetensors').as_posix()
new_item = {
'file_path': file_path,
'file_name': 'x',
'model_name': 'Model X',
'folder': '',
'size': 42,
'modified': 1.0,
'sha256': 'sha-x',
'base_model': 'SDXL',
'preview_url': '',
'preview_nsfw_level': 0,
'from_civitai': True,
'favorite': True,
'notes': 'test note',
'usage_tips': '{}',
'metadata_source': None,
'exclude': False,
'db_checked': False,
'last_checked_at': 0.0,
'tags': ['test', 'new'],
'civitai': None,
'civitai_deleted': False,
'skip_metadata_refresh': False,
'license_flags': DEFAULT_LICENSE_FLAGS,
'hash_status': 'completed',
'hf_url': '',
}
store.update_single_model('dummy', new_item)
persisted = store.load_cache('dummy')
assert persisted is not None
items = {item['file_path']: item for item in persisted.raw_data}
assert file_path in items
assert items[file_path]['model_name'] == 'Model X'
assert items[file_path]['favorite'] is True
assert sorted(items[file_path]['tags']) == ['new', 'test']
def test_update_single_model_update_tags(tmp_path: Path, monkeypatch):
"""Tags are updated incrementally: old tags removed, new tags added."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'y.safetensors').as_posix()
base = {
'file_path': file_path, 'file_name': 'y', 'model_name': 'Y',
'folder': '', 'size': 1, 'modified': 1.0, 'sha256': 'sha-y',
'base_model': '', 'preview_url': '', 'preview_nsfw_level': 0,
'from_civitai': True, 'favorite': False, 'notes': '', 'usage_tips': '{}',
'metadata_source': None, 'exclude': False, 'db_checked': False,
'last_checked_at': 0.0, 'civitai': None, 'civitai_deleted': False,
'skip_metadata_refresh': False, 'license_flags': DEFAULT_LICENSE_FLAGS,
'hash_status': 'completed', 'hf_url': '',
}
# First insert with tags [alpha, beta]
store.update_single_model('dummy', {**base, 'tags': ['alpha', 'beta']})
# Now update: replace with [beta, gamma]
old_item = {'file_path': file_path, 'tags': ['alpha', 'beta'], 'sha256': 'sha-y'}
new_item = {**base, 'tags': ['beta', 'gamma']}
store.update_single_model('dummy', new_item, old_item=old_item)
persisted = store.load_cache('dummy')
assert persisted is not None
items = {item['file_path']: item for item in persisted.raw_data}
assert sorted(items[file_path]['tags']) == ['beta', 'gamma']
def test_update_single_model_update_hash(tmp_path: Path, monkeypatch):
"""When sha256 changes, the hash_index is updated incrementally."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = (tmp_path / 'z.safetensors').as_posix()
base = {
'file_path': file_path, 'file_name': 'z', 'model_name': 'Z',
'folder': '', 'size': 1, 'modified': 1.0, 'base_model': '',
'preview_url': '', 'preview_nsfw_level': 0, 'from_civitai': True,
'favorite': False, 'notes': '', 'usage_tips': '{}',
'metadata_source': None, 'exclude': False, 'db_checked': False,
'last_checked_at': 0.0, 'tags': [], 'civitai': None,
'civitai_deleted': False, 'skip_metadata_refresh': False,
'license_flags': DEFAULT_LICENSE_FLAGS, 'hash_status': 'completed', 'hf_url': '',
}
store.update_single_model('dummy', {**base, 'sha256': 'old-hash'})
old_item = {'file_path': file_path, 'tags': [], 'sha256': 'old-hash'}
new_item = {**base, 'sha256': 'new-hash'}
store.update_single_model('dummy', new_item, old_item=old_item)
persisted = store.load_cache('dummy')
assert persisted is not None
# old hash should be gone from hash_index
old_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'old-hash']
assert len(old_hash_pairs) == 0
# new hash should be present
new_hash_pairs = [p for p in persisted.hash_rows if p[0] == 'new-hash']
assert len(new_hash_pairs) == 1
assert new_hash_pairs[0][1] == file_path