mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-06 14:10:13 -03:00
feat(example-images): add missing-only download path and skip existing files
Split the single-model and bulk context menu actions into 'Download Missing Example Images' (regular endpoint, skips already-processed models) and 'Re-process Example Images' (force endpoint, retries failed models). - start_download accepts model_hashes so a selected subset can be processed with the progress-aware skip logic; explicitly targeted models bypass the failed/processed model-level guards so per-image gaps are filled - pre-download existence check in the processor skips network requests for image files already on disk across all download paths - force download retries previously failed models and clears their failed status on success - add i18n keys for the new menu items across all locales
This commit is contained in:
@@ -2155,4 +2155,31 @@ describe('Interaction-level regression coverage', () => {
|
||||
excludedItem.dispatchEvent(new Event('click', { bubbles: true }));
|
||||
expect(window.pageControls.enterExcludedView).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('routes single-model example downloads to missing-only and force paths', async () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="loraContextMenu" class="context-menu">
|
||||
<div class="context-menu-item" data-action="download-examples"></div>
|
||||
<div class="context-menu-item" data-action="download-examples-force"></div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const { LoraContextMenu } = await import('../../../static/js/components/ContextMenu/LoraContextMenu.js');
|
||||
const contextMenu = new LoraContextMenu();
|
||||
|
||||
const card = document.createElement('div');
|
||||
card.className = 'model-card';
|
||||
card.dataset.filepath = '/models/test.safetensors';
|
||||
card.dataset.sha256 = 'abc123hash';
|
||||
document.body.appendChild(card);
|
||||
|
||||
contextMenu.showMenu(100, 100, card);
|
||||
|
||||
document.querySelector('[data-action="download-examples"]').dispatchEvent(new Event('click', { bubbles: true }));
|
||||
expect(downloadExampleImagesApiMock).toHaveBeenCalledWith(['abc123hash'], null, { force: false });
|
||||
|
||||
contextMenu.showMenu(100, 100, card);
|
||||
document.querySelector('[data-action="download-examples-force"]').dispatchEvent(new Event('click', { bubbles: true }));
|
||||
expect(downloadExampleImagesApiMock).toHaveBeenCalledWith(['abc123hash'], null, { force: true });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -529,7 +529,8 @@ async def test_not_found_example_images_are_cleaned(
|
||||
|
||||
model_dir = images_root / model_hash
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
(model_dir / "image_0.png").write_bytes(b"first")
|
||||
# Pre-existing file collides with the valid image index (1) so the
|
||||
# pre-download existence check must skip it without a network request
|
||||
(model_dir / "image_1.png").write_bytes(b"second")
|
||||
|
||||
async def fake_process_local_examples(*_args, **_kwargs):
|
||||
@@ -608,11 +609,188 @@ async def test_not_found_example_images_are_cleaned(
|
||||
]
|
||||
|
||||
files = sorted(p.name for p in model_dir.iterdir())
|
||||
assert files == ["image_0.png", "image_1.png"]
|
||||
assert (model_dir / "image_0.png").read_bytes() == b"first"
|
||||
assert files == ["image_1.png"]
|
||||
assert (model_dir / "image_1.png").read_bytes() == b"second"
|
||||
|
||||
|
||||
async def test_failed_models_retried_when_explicitly_targeted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
settings_manager,
|
||||
):
|
||||
ws_manager = RecordingWebSocketManager()
|
||||
manager = download_module.DownloadManager(ws_manager=ws_manager)
|
||||
|
||||
images_root = tmp_path / "examples"
|
||||
monkeypatch.setitem(settings_manager.settings, "example_images_path", str(images_root))
|
||||
|
||||
model_hash = "a" * 64
|
||||
model_path = tmp_path / "model.safetensors"
|
||||
model_path.write_text("data", encoding="utf-8")
|
||||
|
||||
model_metadata = {
|
||||
"sha256": model_hash,
|
||||
"model_name": "Failed Example",
|
||||
"file_path": str(model_path),
|
||||
"file_name": "model.safetensors",
|
||||
"civitai": {"images": [{"url": "https://example.com/valid.png"}]},
|
||||
}
|
||||
|
||||
scanner = StubScanner([model_metadata.copy()])
|
||||
_patch_scanner(monkeypatch, scanner)
|
||||
|
||||
# Persist a previous failure so the skip path is exercised
|
||||
images_root.mkdir(parents=True, exist_ok=True)
|
||||
(images_root / ".download_progress.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"failed_models": [model_hash],
|
||||
"processed_models": [],
|
||||
"rate_limited_models": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
async def fake_process_local_examples(*_args, **_kwargs):
|
||||
return False
|
||||
|
||||
async def fake_get_updated_model(model_hash_arg, _scanner):
|
||||
return model_metadata
|
||||
|
||||
class DownloaderStub:
|
||||
def __init__(self):
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def download_to_memory(self, url, *_args, **_kwargs):
|
||||
self.calls.append(url)
|
||||
return True, b"\x89PNG\r\n\x1a\n", {"content-type": "image/png"}
|
||||
|
||||
downloader = DownloaderStub()
|
||||
|
||||
async def fake_get_downloader():
|
||||
return downloader
|
||||
|
||||
monkeypatch.setattr(
|
||||
download_module.ExampleImagesProcessor,
|
||||
"process_local_examples",
|
||||
staticmethod(fake_process_local_examples),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
download_module.MetadataUpdater,
|
||||
"get_updated_model",
|
||||
staticmethod(fake_get_updated_model),
|
||||
)
|
||||
monkeypatch.setattr(download_module, "get_downloader", fake_get_downloader)
|
||||
|
||||
# Without explicit hashes the previously failed model is skipped
|
||||
skipped_manager = download_module.DownloadManager(ws_manager=RecordingWebSocketManager())
|
||||
result = await skipped_manager.start_download({"model_types": ["lora"], "delay": 0})
|
||||
assert result["success"] is True
|
||||
if skipped_manager._download_task is not None:
|
||||
await asyncio.wait_for(skipped_manager._download_task, timeout=1)
|
||||
assert downloader.calls == []
|
||||
|
||||
# With explicit hashes the previously failed model is retried and cleared
|
||||
result = await manager.start_download(
|
||||
{"model_types": ["lora"], "delay": 0, "model_hashes": [model_hash]}
|
||||
)
|
||||
assert result["success"] is True
|
||||
if manager._download_task is not None:
|
||||
await asyncio.wait_for(manager._download_task, timeout=1)
|
||||
assert downloader.calls == ["https://example.com/valid.png"]
|
||||
assert manager._progress["failed_models"] == set()
|
||||
assert model_hash in manager._progress["processed_models"]
|
||||
|
||||
|
||||
async def test_explicit_targets_fill_partial_example_gaps(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
settings_manager,
|
||||
):
|
||||
ws_manager = RecordingWebSocketManager()
|
||||
|
||||
images_root = tmp_path / "examples"
|
||||
monkeypatch.setitem(settings_manager.settings, "example_images_path", str(images_root))
|
||||
|
||||
model_hash = "b" * 64
|
||||
model_path = tmp_path / "model.safetensors"
|
||||
model_path.write_text("data", encoding="utf-8")
|
||||
|
||||
model_metadata = {
|
||||
"sha256": model_hash,
|
||||
"model_name": "Partial Example",
|
||||
"file_path": str(model_path),
|
||||
"file_name": "model.safetensors",
|
||||
"civitai": {
|
||||
"images": [
|
||||
{"url": "https://example.com/first.png"},
|
||||
{"url": "https://example.com/second.png"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
scanner = StubScanner([model_metadata.copy()])
|
||||
_patch_scanner(monkeypatch, scanner)
|
||||
|
||||
# Simulate a partially populated folder: index 0 already downloaded
|
||||
model_dir = images_root / model_hash
|
||||
model_dir.mkdir(parents=True, exist_ok=True)
|
||||
(model_dir / "image_0.png").write_bytes(b"existing")
|
||||
|
||||
async def fake_process_local_examples(*_args, **_kwargs):
|
||||
return False
|
||||
|
||||
async def fake_get_updated_model(model_hash_arg, _scanner):
|
||||
return model_metadata
|
||||
|
||||
class DownloaderStub:
|
||||
def __init__(self):
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def download_to_memory(self, url, *_args, **_kwargs):
|
||||
self.calls.append(url)
|
||||
return True, b"\x89PNG\r\n\x1a\n", {"content-type": "image/png"}
|
||||
|
||||
downloader = DownloaderStub()
|
||||
|
||||
async def fake_get_downloader():
|
||||
return downloader
|
||||
|
||||
monkeypatch.setattr(
|
||||
download_module.ExampleImagesProcessor,
|
||||
"process_local_examples",
|
||||
staticmethod(fake_process_local_examples),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
download_module.MetadataUpdater,
|
||||
"get_updated_model",
|
||||
staticmethod(fake_get_updated_model),
|
||||
)
|
||||
monkeypatch.setattr(download_module, "get_downloader", fake_get_downloader)
|
||||
|
||||
# Untargeted run treats the populated folder as done
|
||||
untargeted = download_module.DownloadManager(ws_manager=RecordingWebSocketManager())
|
||||
result = await untargeted.start_download({"model_types": ["lora"], "delay": 0})
|
||||
assert result["success"] is True
|
||||
if untargeted._download_task is not None:
|
||||
await asyncio.wait_for(untargeted._download_task, timeout=1)
|
||||
assert downloader.calls == []
|
||||
|
||||
# Explicitly targeted run fills only the missing index, skipping the
|
||||
# existing file without a network request
|
||||
targeted = download_module.DownloadManager(ws_manager=ws_manager)
|
||||
result = await targeted.start_download(
|
||||
{"model_types": ["lora"], "delay": 0, "model_hashes": [model_hash]}
|
||||
)
|
||||
assert result["success"] is True
|
||||
if targeted._download_task is not None:
|
||||
await asyncio.wait_for(targeted._download_task, timeout=1)
|
||||
assert downloader.calls == ["https://example.com/second.png"]
|
||||
assert (model_dir / "image_1.png").exists()
|
||||
assert (model_dir / "image_0.png").read_bytes() == b"existing"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def settings_manager():
|
||||
return get_settings_manager()
|
||||
|
||||
@@ -63,7 +63,7 @@ async def test_start_download_bootstraps_progress_and_task(
|
||||
release = asyncio.Event()
|
||||
|
||||
async def fake_download(
|
||||
self, output_dir, optimize, model_types, delay, library_name, force=False
|
||||
self, output_dir, optimize, model_types, delay, library_name, force=False, model_hashes=None
|
||||
):
|
||||
started.set()
|
||||
await release.wait()
|
||||
@@ -93,6 +93,44 @@ async def test_start_download_bootstraps_progress_and_task(
|
||||
assert manager._progress["status"] == "completed"
|
||||
|
||||
|
||||
async def test_start_download_forwards_model_hashes(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
) -> None:
|
||||
settings_manager = get_settings_manager()
|
||||
settings_manager.settings["example_images_path"] = str(tmp_path)
|
||||
settings_manager.settings["libraries"] = {"default": {}}
|
||||
settings_manager.settings["active_library"] = "default"
|
||||
|
||||
manager = download_module.DownloadManager(ws_manager=RecordingWebSocketManager())
|
||||
|
||||
received: Dict[str, Any] = {}
|
||||
|
||||
async def fake_download(
|
||||
self, output_dir, optimize, model_types, delay, library_name, force=False, model_hashes=None
|
||||
):
|
||||
received["model_hashes"] = model_hashes
|
||||
async with self._state_lock:
|
||||
self._is_downloading = False
|
||||
self._download_task = None
|
||||
self._progress["status"] = "completed"
|
||||
|
||||
monkeypatch.setattr(
|
||||
download_module.DownloadManager,
|
||||
"_download_all_example_images",
|
||||
fake_download,
|
||||
)
|
||||
|
||||
result = await manager.start_download(
|
||||
{"model_types": ["lora"], "delay": 0, "model_hashes": ["abc123", "def456"]}
|
||||
)
|
||||
assert result["success"] is True
|
||||
|
||||
task = manager._download_task
|
||||
assert task is not None
|
||||
await asyncio.wait_for(task, timeout=1)
|
||||
assert received["model_hashes"] == ["abc123", "def456"]
|
||||
|
||||
|
||||
async def test_pause_and_resume_flow(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
|
||||
settings_manager = get_settings_manager()
|
||||
settings_manager.settings["example_images_path"] = str(tmp_path)
|
||||
|
||||
@@ -100,6 +100,54 @@ def test_get_file_extension_media_type_hint_low_priority() -> None:
|
||||
assert ext == ".mp4"
|
||||
|
||||
|
||||
def test_example_image_file_exists_checks_plausible_extensions(tmp_path) -> None:
|
||||
proc = processor_module.ExampleImagesProcessor
|
||||
assert proc._example_image_file_exists(str(tmp_path), 0) is False
|
||||
Path(tmp_path, "image_0.webp").write_bytes(b"x")
|
||||
assert proc._example_image_file_exists(str(tmp_path), 0) is True
|
||||
assert proc._example_image_file_exists(str(tmp_path), 1) is False
|
||||
|
||||
|
||||
def test_example_image_file_exists_video_hint_only_checks_video_extensions(tmp_path) -> None:
|
||||
proc = processor_module.ExampleImagesProcessor
|
||||
Path(tmp_path, "image_2.jpg").write_bytes(b"x")
|
||||
# An existing image file must not satisfy a video-hinted lookup
|
||||
assert proc._example_image_file_exists(str(tmp_path), 2, "video") is False
|
||||
Path(tmp_path, "image_2.mp4").write_bytes(b"x")
|
||||
assert proc._example_image_file_exists(str(tmp_path), 2, "video") is True
|
||||
|
||||
|
||||
async def test_download_model_images_with_tracking_skips_existing_files(tmp_path) -> None:
|
||||
proc = processor_module.ExampleImagesProcessor
|
||||
images = [
|
||||
{"url": "https://image.civitai.com/a/b", "type": "image"},
|
||||
{"url": "https://image.civitai.com/c/d", "type": "image"},
|
||||
]
|
||||
Path(tmp_path, "image_0.jpg").write_bytes(b"existing")
|
||||
|
||||
class RecordingDownloader:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def download_to_memory(self, url, use_auth=False, return_headers=False):
|
||||
self.calls.append(url)
|
||||
return True, b"\xff\xd8\xff" + b"data", {}
|
||||
|
||||
downloader = RecordingDownloader()
|
||||
success, is_stale, failed, rate_limited = await proc.download_model_images_with_tracking(
|
||||
"hash", "model", images, str(tmp_path), False, downloader
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert is_stale is False
|
||||
assert failed == []
|
||||
assert rate_limited == []
|
||||
# Only the missing image is requested; the existing one is skipped without a network call
|
||||
assert len(downloader.calls) == 1
|
||||
assert "c/d" in downloader.calls[0]
|
||||
assert Path(tmp_path, "image_1.jpg").exists()
|
||||
|
||||
|
||||
class StubScanner:
|
||||
def __init__(self, models: list[Dict[str, Any]]) -> None:
|
||||
self._cache = SimpleNamespace(raw_data=models)
|
||||
|
||||
Reference in New Issue
Block a user