fix(scanner): offload persisted-cache hydration from the event loop

This commit is contained in:
Will Miao
2026-08-25 17:38:09 +08:00
parent c83b26b556
commit cdb044cb45
4 changed files with 204 additions and 73 deletions
@@ -1,5 +1,7 @@
from pathlib import Path
import threading
import pytest
from py.services.downloaded_version_history_service import (
@@ -70,6 +72,37 @@ async def test_download_history_bulk_lookup(tmp_path: Path) -> None:
}
@pytest.mark.asyncio
async def test_mark_downloaded_bulk_writes_off_event_loop(
tmp_path: Path, monkeypatch
) -> None:
"""The executemany upsert + commit must not run on the event loop thread."""
db_path = tmp_path / "download-history.sqlite"
service = DownloadedVersionHistoryService(
str(db_path),
settings_manager=DummySettings(),
)
loop_thread = threading.get_ident()
write_threads: list[int] = []
original_write = service._mark_downloaded_bulk_sync
def tracking_write(payload):
write_threads.append(threading.get_ident())
return original_write(payload)
monkeypatch.setattr(service, "_mark_downloaded_bulk_sync", tracking_write)
await service.mark_downloaded_bulk(
"lora",
[{"model_id": 7, "version_id": 701, "file_path": "/m/x.safetensors"}],
source="scan",
)
assert write_threads and write_threads[0] != loop_thread
assert await service.has_been_downloaded("lora", 701) is True
@pytest.mark.asyncio
async def test_per_file_history_tracking(tmp_path: Path) -> None:
"""Per-file records coexist with the version-level row (#1058)."""
+69
View File
@@ -4,6 +4,7 @@ import asyncio
import json
import os
import sqlite3
import threading
import time
from collections.abc import Iterator
from pathlib import Path
@@ -391,6 +392,74 @@ async def test_load_persisted_cache_populates_cache(tmp_path: Path, monkeypatch)
assert ws_stub.payloads[-1]['progress'] == 1
@pytest.mark.asyncio
async def test_load_persisted_cache_rebuilds_off_event_loop(tmp_path: Path, monkeypatch):
"""The SQLite read and per-model rebuild must not run on the event loop."""
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')
db_path = tmp_path / 'cache.sqlite'
store = PersistentModelCache(db_path=str(db_path))
file_path = tmp_path / 'one.txt'
file_path.write_text('one', encoding='utf-8')
normalized = _normalize_path(file_path)
raw_model = {
'file_path': normalized,
'file_name': 'one',
'model_name': 'one',
'folder': '',
'size': 3,
'modified': 123.0,
'sha256': 'hash-one',
'base_model': 'test',
'preview_url': '',
'preview_nsfw_level': 0,
'from_civitai': True,
'favorite': False,
'notes': '',
'usage_tips': '',
'exclude': False,
'db_checked': False,
'last_checked_at': 0.0,
'tags': ['alpha'],
'civitai': {'id': 11, 'modelId': 22, 'name': 'ver', 'trainedWords': ['abc']},
}
store.save_cache('dummy', [raw_model], {'hash-one': [normalized]}, [])
monkeypatch.setattr(model_scanner, 'get_persistent_cache', lambda: store)
scanner = DummyScanner(tmp_path)
ws_stub = RecordingWebSocketManager()
monkeypatch.setattr(model_scanner, 'ws_manager', ws_stub)
loop_thread = threading.get_ident()
worker_threads: List[int] = []
original_load_cache = store.load_cache
def tracking_load_cache(model_type):
worker_threads.append(threading.get_ident())
return original_load_cache(model_type)
monkeypatch.setattr(store, 'load_cache', tracking_load_cache)
original_adjust = scanner.adjust_cached_entry
def tracking_adjust(entry):
worker_threads.append(threading.get_ident())
return original_adjust(entry)
monkeypatch.setattr(scanner, 'adjust_cached_entry', tracking_adjust)
loaded = await scanner._load_persisted_cache('dummy')
assert loaded is True
# Both the SQLite read and the per-entry adjustment ran off the loop
assert len(worker_threads) == 2
assert all(tid != loop_thread for tid in worker_threads)
cache = await scanner.get_cached_data()
assert len(cache.raw_data) == 1
assert cache.raw_data[0]['file_path'] == normalized
@pytest.mark.asyncio
async def test_update_single_model_cache_persists_changes(tmp_path: Path, monkeypatch):
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')