Compare commits

...
5 Commits
Author SHA1 Message Date
Will Miao 86aa1d8059 fix(ui): position toasts below header to avoid overlapping page controls 2026-08-20 22:08:27 +08:00
Will Miao 74254756ef fix(ui): unify modal backdrop blur across all modals 2026-08-20 21:25:05 +08:00
Will Miao 259e08e47c feat(download): expose per-file downloadedFiles in check-model-exists (#1058)
The version branch of check-model-exists now returns
downloadedFiles: [{fileId, fileName, filePath}] so clients (e.g. the
browser extension) can tell a partially downloaded version apart from a
fully downloaded one. Reuses ModelCivitaiHandler._match_downloaded_files
(D2 rule) against the local cache; unmatchable local files are reported
with fileId: None. No CivitAI API call added.
2026-08-20 20:59:19 +08:00
Will Miao 6647c45731 fix(download): include file identity in queue/history dedup (#1058)
Distinct files of the same model version queued before a backend restart
were silently collapsed by deduplicate(), which grouped rows by
(model_id, model_version_id) only. Extract the file id from file_params
via json_extract and add it to the dedup key; rows without file identity
keep the old per-version behavior (NULL matches NULL).
2026-08-20 20:58:44 +08:00
Will Miao b614a5c447 docs: remove broken star history chart (#1066) 2026-08-20 20:56:22 +08:00
12 changed files with 444 additions and 49 deletions
-5
View File
@@ -346,8 +346,3 @@ WeChat: [Click to view QR code](https://raw.githubusercontent.com/willmiao/Comfy
Join our Discord community for support, discussions, and updates:
[Discord Server](https://discord.gg/vcqNrWVFvM)
---
## Star History
[![Star History Chart](https://api.star-history.com/chart?repos=willmiao/ComfyUI-Lora-Manager&type=date&legend=top-left&sealed_token=Ly6HDvUHnRrM7jWpL67I1aYGnmRiLyxEMcrPz8tNEHpaIAzxVsJVO6trhjWB2gG333EyqRL8ygW0pETjKgRHzoePaQXtaw3xIt6ODjo_5zER74xU51xLVruGOi9kkb16G7Mv8Unq5PkBtQW88tPa53b2jzwZYbDa7GP_k3K8aiA49uk1Vd4yU4uG1q-0)](https://www.star-history.com/?repos=willmiao%2FComfyUI-Lora-Manager&type=date&legend=top-left)
+66
View File
@@ -56,6 +56,7 @@ from ...utils.constants import (
)
from .hf_handlers import HfHandler
from .agent_handlers import AgentHandler
from .model_handlers import ModelCivitaiHandler
from ...utils.civitai_utils import rewrite_preview_url
from ...utils.example_images_paths import (
find_non_compliant_items_in_example_images_root,
@@ -2061,6 +2062,63 @@ class ModelLibraryHandler:
enriched.append(entry)
return enriched
@staticmethod
async def _get_downloaded_files(
scanner: Any, model_version_id: int
) -> list[dict[str, Any]]:
"""Return per-file downloaded state for a version in the library.
This handler has no CivitAI version payload, so the remote file list
is taken from the local entries' cached ``civitai`` metadata (the
full version payload persisted at download time, see
``BaseModelMetadata.from_civitai_info``) and matched with the same
D2 rule used by ``get_civitai_versions`` (#1058). Local entries that
cannot be matched to a known remote file (e.g. missing metadata or
renamed files) are still reported with ``fileId`` set to None.
Returns ``[{fileId, fileName, filePath}]``.
"""
try:
cache = await scanner.get_cached_data()
except Exception: # pragma: no cover - defensive fallback
logger.debug(
"Failed to read cache for downloaded files of version %s",
model_version_id,
exc_info=True,
)
return []
files_getter = getattr(cache, "get_files_by_version_id", None)
local_entries = files_getter(model_version_id) if files_getter else []
if not local_entries:
return []
version_payload: Mapping[str, Any] = {}
for entry in local_entries:
civitai = entry.get("civitai") if isinstance(entry, Mapping) else None
if isinstance(civitai, Mapping) and isinstance(civitai.get("files"), list):
version_payload = civitai
break
downloaded = ModelCivitaiHandler._match_downloaded_files(
version_payload, local_entries
)
# Surface local files that D2 could not map to a known remote file
matched_paths = {item.get("filePath") for item in downloaded}
for entry in local_entries:
if not isinstance(entry, Mapping):
continue
if entry.get("file_path") in matched_paths:
continue
downloaded.append(
{
"fileId": None,
"fileName": entry.get("file_name"),
"filePath": entry.get("file_path"),
}
)
return downloaded
async def check_model_exists(self, request: web.Request) -> web.Response:
try:
model_id_str = request.query.get("modelId")
@@ -2096,9 +2154,11 @@ class ModelLibraryHandler:
exists = False
model_type = None
matched_scanner = None
if await lora_scanner.check_model_version_exists(model_version_id):
exists = True
model_type = "lora"
matched_scanner = lora_scanner
elif (
checkpoint_scanner
and await checkpoint_scanner.check_model_version_exists(
@@ -2107,6 +2167,7 @@ class ModelLibraryHandler:
):
exists = True
model_type = "checkpoint"
matched_scanner = checkpoint_scanner
elif (
embedding_scanner
and await embedding_scanner.check_model_version_exists(
@@ -2115,6 +2176,7 @@ class ModelLibraryHandler:
):
exists = True
model_type = "embedding"
matched_scanner = embedding_scanner
if exists:
return web.json_response(
@@ -2123,6 +2185,9 @@ class ModelLibraryHandler:
"exists": True,
"modelType": model_type,
"hasBeenDownloaded": False,
"downloadedFiles": await self._get_downloaded_files(
matched_scanner, model_version_id
),
}
)
@@ -2144,6 +2209,7 @@ class ModelLibraryHandler:
"exists": False,
"modelType": history_type,
"hasBeenDownloaded": has_been_downloaded,
"downloadedFiles": [],
}
)
+40 -17
View File
@@ -12,6 +12,15 @@ from ..utils.cache_paths import get_cache_base_dir
logger = logging.getLogger(__name__)
# SQL fragment extracting the CivitAI file id from the JSON ``file_params``
# column (#1058). ``json_valid`` guards against NULL and legacy/unparseable
# values, yielding NULL for rows without a file identity; NULL keys group
# together so such rows keep the old version-level dedup behavior.
_FILE_ID_SQL = (
"CASE WHEN json_valid(file_params) "
"THEN json_extract(file_params, '$.id') END"
)
def _resolve_database_path() -> str:
base_dir = get_cache_base_dir(create=True)
@@ -866,33 +875,44 @@ class DownloadQueueService:
async with self._lock:
conn = self._get_conn()
# 1. History: for each (model_id, model_version_id, status) triplet
# keep only the row with the highest id (most recently inserted).
conn.execute("""
# 1. History: for each (model_id, model_version_id, file_id,
# status) group keep only the row with the highest id (most
# recently inserted). file_id comes from file_params (#1058)
# so distinct files of the same version never collapse.
conn.execute(f"""
DELETE FROM download_history
WHERE id NOT IN (
SELECT MAX(id)
FROM download_history
GROUP BY model_id, model_version_id, status
GROUP BY model_id, model_version_id, status,
{_FILE_ID_SQL}
)
""")
result["removed_history"] = conn.execute(
"SELECT changes()"
).fetchone()[0]
# 2. Cross-status dedup: for each (model_id, model_version_id),
# keep only the entry with the highest-priority terminal status.
# 2. Cross-status dedup: for each (model_id, model_version_id,
# file_id), keep only the entry with the highest-priority
# terminal status.
# Priority: completed (3) > failed (2) > canceled (1).
# This prevents the same model version from having both a
# 'failed' and a 'canceled' entry (or a 'completed' alongside
# either) after the bug-created duplicates are removed.
conn.execute("""
# This prevents the same file of a model version from having
# both a 'failed' and a 'canceled' entry (or a 'completed'
# alongside either) after the bug-created duplicates are
# removed. ``IS`` matches NULL file ids against each other so
# rows without file identity keep the old behavior.
conn.execute(f"""
DELETE FROM download_history
WHERE id NOT IN (
SELECT dh.id
FROM download_history dh
FROM (
SELECT id, model_id, model_version_id, status,
{_FILE_ID_SQL} AS file_id
FROM download_history
) dh
INNER JOIN (
SELECT model_id, model_version_id,
{_FILE_ID_SQL} AS file_id,
MAX(CASE status
WHEN 'completed' THEN 3
WHEN 'failed' THEN 2
@@ -900,17 +920,18 @@ class DownloadQueueService:
ELSE 0
END) AS best_prio
FROM download_history
GROUP BY model_id, model_version_id
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
) best
ON dh.model_id = best.model_id
AND dh.model_version_id = best.model_version_id
AND dh.file_id IS best.file_id
AND CASE dh.status
WHEN 'completed' THEN 3
WHEN 'failed' THEN 2
WHEN 'canceled' THEN 1
ELSE 0
END = best.best_prio
GROUP BY dh.model_id, dh.model_version_id
GROUP BY dh.model_id, dh.model_version_id, dh.file_id
HAVING dh.id = MAX(dh.id)
)
""")
@@ -918,15 +939,17 @@ class DownloadQueueService:
"SELECT changes()"
).fetchone()[0]
# 3. Queue: for each (model_id, model_version_id) keep only the
# row with the latest added_at (most recently enqueued).
conn.execute("""
# 3. Queue: for each (model_id, model_version_id, file_id) keep
# only the row with the latest added_at (most recently
# enqueued). file_id comes from file_params (#1058) so
# distinct files of the same version never collapse.
conn.execute(f"""
DELETE FROM download_queue
WHERE rowid NOT IN (
SELECT MAX(rowid)
FROM download_queue
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
GROUP BY model_id, model_version_id
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
)
AND status IN ('queued', 'downloading', 'paused', 'waiting')
""")
+1 -1
View File
@@ -4,7 +4,7 @@
position: fixed;
top: 0;
z-index: var(--z-header);
height: 48px;
height: var(--header-height, 48px);
/* Reduced height */
width: 100%;
box-shadow: var(--shadow-md);
@@ -1,10 +1,3 @@
/* Blurred backdrop for the model modal to match the recipe modal */
#modelModal {
background: rgba(0, 0, 0, 0.5);
backdrop-filter: blur(6px);
-webkit-backdrop-filter: blur(6px);
}
/* Lora Modal Header */
.modal-header {
display: flex;
+3 -1
View File
@@ -6,7 +6,9 @@
left: 0;
width: 100%;
height: calc(100% - var(--header-height, 48px)); /* Adjust height to exclude header */
background: rgba(0, 0, 0, 0.2);
background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
-webkit-backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
z-index: var(--z-modal);
overflow: auto; /* Change from hidden to auto to allow scrolling */
}
@@ -13,7 +13,10 @@
left: 0;
width: 100%;
height: 100%;
/* Darker than --modal-backdrop-bg to stress destructive actions, but keeps the shared blur */
background: rgba(0, 0, 0, 0.8);
backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
-webkit-backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
z-index: var(--z-overlay);
}
-7
View File
@@ -104,13 +104,6 @@
display: none;
}
/* Darker, blurred backdrop keeps the busy page behind the modal from bleeding through */
#recipeModal {
background: rgba(0, 0, 0, 0.5);
backdrop-filter: blur(6px);
-webkit-backdrop-filter: blur(6px);
}
#recipeModal .modal-content {
display: flex;
flex-direction: column;
+11 -11
View File
@@ -5,11 +5,11 @@
border: none;
padding: 8px 16px;
font-size: 0.9em;
transform: translateX(-50%) translateY(20px);
transform: translateY(20px);
}
.toast.toast-copy.show {
transform: translateX(-50%) translateY(0);
transform: translateY(0);
}
/* Toast Notifications */
@@ -19,14 +19,15 @@
right: 20px;
left: auto;
transform: translateX(120%);
min-width: 300px;
box-sizing: border-box;
min-width: 200px;
max-width: 400px;
background: var(--lora-surface);
color: var(--text-color);
padding: 12px 16px;
border-radius: var(--border-radius-sm);
box-shadow: var(--shadow-toast);
z-index: calc(var(--z-overlay) + 10);
z-index: var(--z-toast);
opacity: 0;
transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1),
opacity 0.3s cubic-bezier(0.4, 0, 0.2, 1);
@@ -130,7 +131,6 @@
.toast {
width: calc(100% - 40px);
max-width: none;
right: 20px;
}
}
@@ -166,16 +166,17 @@
opacity: 1;
}
/* Toast Container for stacked notifications */
/* Toast Container for stacked notifications (top-right, flush below the header) */
.toast-container {
position: fixed;
top: 0;
top: var(--header-height, 48px); /* Start right below the fixed header */
right: 0;
z-index: calc(var(--z-overlay) + 10);
z-index: var(--z-toast);
display: flex;
flex-direction: column;
align-items: flex-end;
gap: 10px;
padding: 20px;
padding: 8px 20px 0; /* Small breathing room below the header */
pointer-events: none; /* Allow clicking through the container */
width: 400px;
max-width: 100%;
@@ -215,8 +216,7 @@
/* Responsive adjustments */
@media (max-width: 480px) {
.toast-container {
width: 100%;
padding: 10px;
padding: 0 10px;
}
.toast {
+3
View File
@@ -27,6 +27,9 @@
--shadow-dialog: 0 10px 24px rgba(0, 0, 0, 0.25);
--shadow-inset-top: 0 -2px 8px rgba(0, 0, 0, 0.1);
--modal-backdrop-bg: rgba(0, 0, 0, 0.5);
--modal-backdrop-blur: 6px;
--transition-fast: 150ms ease;
--transition-base: 200ms ease;
--transition-slow: 300ms ease;
+143
View File
@@ -908,6 +908,33 @@ class FakeExistenceScanner:
return []
class FakeVersionFilesCache:
"""Cache stub exposing the multi-valued version->files index (#1058)."""
def __init__(self, entries):
self._entries = entries
def get_files_by_version_id(self, _version_id):
return list(self._entries)
class FakeDownloadedFilesScanner:
"""Scanner stub with one known version and per-file cache entries."""
def __init__(self, version_id, entries):
self._version_id = version_id
self._cache = FakeVersionFilesCache(entries)
async def check_model_version_exists(self, version_id):
return version_id == self._version_id
async def get_model_versions_by_id(self, _model_id):
return []
async def get_cached_data(self):
return self._cache
class FakeMetadataProvider:
async def get_model_versions(self, _model_id):
return {"modelVersions": [], "name": "", "type": "lora"}
@@ -1524,9 +1551,125 @@ async def test_check_model_exists_returns_download_history_when_file_missing():
"exists": False,
"modelType": "checkpoint",
"hasBeenDownloaded": True,
"downloadedFiles": [],
}
@pytest.mark.asyncio
async def test_check_model_exists_returns_downloaded_files():
"""The version branch exposes per-file downloaded state (#1058)."""
civitai_payload = {
"id": 100,
"files": [
{
"id": 1001,
"name": "model-fp16.safetensors",
"hashes": {"SHA256": "A" * 64},
},
{
"id": 1002,
"name": "model-fp32.safetensors",
"hashes": {"SHA256": "B" * 64},
},
],
}
entries = [
{
"file_name": "model-fp16",
"file_path": "/models/loras/model-fp16.safetensors",
"sha256": "a" * 64,
"civitai": civitai_payload,
},
{
# Legacy entry without a hash: matched by extension-less name (D2)
"file_name": "model-fp32",
"file_path": "/models/loras/model-fp32.safetensors",
"sha256": "",
"civitai": civitai_payload,
},
]
lora_scanner = FakeDownloadedFilesScanner(100, entries)
async def lora_factory():
return lora_scanner
handler = ModelLibraryHandler(
ServiceRegistryAdapter(
get_lora_scanner=lora_factory,
get_checkpoint_scanner=fake_scanner_factory,
get_embedding_scanner=fake_scanner_factory,
get_downloaded_version_history_service=fake_download_history_service_factory,
),
metadata_provider_factory=fake_metadata_provider_factory,
)
response = await handler.check_model_exists(
FakeRequest(query={"modelId": "5", "modelVersionId": "100"}) # pyright: ignore[reportArgumentType]
)
payload = _json_payload(response)
assert payload == {
"success": True,
"exists": True,
"modelType": "lora",
"hasBeenDownloaded": False,
"downloadedFiles": [
{
"fileId": 1001,
"fileName": "model-fp16.safetensors",
"filePath": "/models/loras/model-fp16.safetensors",
},
{
"fileId": 1002,
"fileName": "model-fp32.safetensors",
"filePath": "/models/loras/model-fp32.safetensors",
},
],
}
@pytest.mark.asyncio
async def test_check_model_exists_downloaded_files_without_remote_metadata():
"""Unmatchable local files are still reported with fileId None (#1058)."""
entries = [
{
"file_name": "renamed-model",
"file_path": "/models/loras/renamed-model.safetensors",
"sha256": "c" * 64,
"civitai": {"id": 100}, # no remote files list cached
},
]
lora_scanner = FakeDownloadedFilesScanner(100, entries)
async def lora_factory():
return lora_scanner
handler = ModelLibraryHandler(
ServiceRegistryAdapter(
get_lora_scanner=lora_factory,
get_checkpoint_scanner=fake_scanner_factory,
get_embedding_scanner=fake_scanner_factory,
get_downloaded_version_history_service=fake_download_history_service_factory,
),
metadata_provider_factory=fake_metadata_provider_factory,
)
response = await handler.check_model_exists(
FakeRequest(query={"modelId": "5", "modelVersionId": "100"}) # pyright: ignore[reportArgumentType]
)
payload = _json_payload(response)
assert payload["success"] is True
assert payload["exists"] is True
assert payload["downloadedFiles"] == [
{
"fileId": None,
"fileName": "renamed-model",
"filePath": "/models/loras/renamed-model.safetensors",
}
]
@pytest.mark.asyncio
async def test_model_version_download_status_endpoints():
history_service = FakeDownloadHistoryService({"lora": {123}})
@@ -7,6 +7,7 @@ compatibility with ``id``.
import json
import sqlite3
import time
from pathlib import Path
import pytest
@@ -321,3 +322,176 @@ async def test_legacy_history_db_gains_file_params_column(tmp_path: Path) -> Non
item = await svc.retry_from_history(download_id="dl-legacy-fp")
assert item is not None
assert json.loads(item["file_params"]) == {"id": 5}
# ---------------------------------------------------------------------------
# deduplicate() — file identity in the dedup key (#1058)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_dedup_queue_keeps_distinct_files_of_same_version(tmp_path: Path) -> None:
"""Two queued downloads of different files of one version both survive."""
svc = _make_service(tmp_path)
await svc.add_to_queue(
download_id="dl-a",
model_id=1,
model_version_id=100,
file_params={"id": 1001, "type": "Model"},
)
await svc.add_to_queue(
download_id="dl-b",
model_id=1,
model_version_id=100,
file_params={"id": 1002, "type": "Model"},
)
result = await svc.deduplicate()
assert result["removed_queue"] == 0
queue = await svc.get_queue()
assert sorted(item["download_id"] for item in queue) == ["dl-a", "dl-b"]
@pytest.mark.asyncio
async def test_dedup_queue_collapses_same_file_and_legacy_rows(tmp_path: Path) -> None:
"""Same version + same/absent file id still collapses to the latest row."""
svc = _make_service(tmp_path)
# Same file id -> only the most recently enqueued row survives
await svc.add_to_queue(
download_id="dl-old",
model_id=1,
model_version_id=100,
file_params={"id": 1001},
)
await svc.add_to_queue(
download_id="dl-new",
model_id=1,
model_version_id=100,
file_params={"id": 1001},
)
# Rows without file identity keep the old per-version behavior
await svc.add_to_queue(
download_id="dl-legacy-old", model_id=2, model_version_id=200
)
await svc.add_to_queue(
download_id="dl-legacy-new", model_id=2, model_version_id=200
)
result = await svc.deduplicate()
assert result["removed_queue"] == 2
remaining = {item["download_id"] for item in await svc.get_queue()}
assert remaining == {"dl-new", "dl-legacy-new"}
@pytest.mark.asyncio
async def test_dedup_queue_does_not_mix_null_and_file_id(tmp_path: Path) -> None:
"""A row with a file id never dedups against a row without one."""
svc = _make_service(tmp_path)
await svc.add_to_queue(
download_id="dl-file",
model_id=1,
model_version_id=100,
file_params={"id": 1001},
)
await svc.add_to_queue(
download_id="dl-nofile", model_id=1, model_version_id=100
)
result = await svc.deduplicate()
assert result["removed_queue"] == 0
assert len(await svc.get_queue()) == 2
@pytest.mark.asyncio
async def test_dedup_queue_unparseable_file_params_treated_as_none(tmp_path: Path) -> None:
"""Corrupt file_params JSON falls back to the NULL file identity."""
svc = _make_service(tmp_path)
await svc.add_to_queue(
download_id="dl-plain", model_id=1, model_version_id=100
)
conn = sqlite3.connect(str(tmp_path / "queue.sqlite"))
conn.execute(
"INSERT INTO download_queue (download_id, model_id, model_version_id, "
"file_params, status, added_at) VALUES (?, ?, ?, ?, 'queued', ?)",
("dl-corrupt", 1, 100, "{not json", time.time()),
)
conn.commit()
conn.close()
result = await svc.deduplicate()
assert result["removed_queue"] == 1
assert len(await svc.get_queue()) == 1
@pytest.mark.asyncio
async def test_dedup_history_keeps_distinct_files_of_same_version(tmp_path: Path) -> None:
"""History rows of different files never collapse, even across statuses."""
svc = _make_service(tmp_path)
await svc.add_to_history(
download_id="dl-h1",
model_id=1,
model_version_id=100,
status="completed",
file_params={"id": 1001},
)
await svc.add_to_history(
download_id="dl-h2",
model_id=1,
model_version_id=100,
status="completed",
file_params={"id": 1002},
)
# A failed entry for file 1002 must not remove file 1001's completed row
await svc.add_to_history(
download_id="dl-h3",
model_id=1,
model_version_id=100,
status="failed",
file_params={"id": 1002},
)
result = await svc.deduplicate()
assert result["removed_history"] == 1 # only dl-h3 collapses (same file)
history = await svc.get_history()
remaining = {item["download_id"] for item in history["items"]}
assert remaining == {"dl-h1", "dl-h2"}
@pytest.mark.asyncio
async def test_dedup_history_collapses_same_file_and_legacy_rows(tmp_path: Path) -> None:
"""Same file id / absent file id history rows still dedup as before."""
svc = _make_service(tmp_path)
# Same file id + same status -> keep the most recent row
await svc.add_to_history(
download_id="dl-x1",
model_id=1,
model_version_id=100,
status="completed",
file_params={"id": 1001},
)
await svc.add_to_history(
download_id="dl-x2",
model_id=1,
model_version_id=100,
status="completed",
file_params={"id": 1001},
)
# Legacy rows without file identity: cross-status dedup still applies
await svc.add_to_history(
download_id="dl-y1", model_id=2, model_version_id=200, status="failed"
)
await svc.add_to_history(
download_id="dl-y2", model_id=2, model_version_id=200, status="completed"
)
result = await svc.deduplicate()
assert result["removed_history"] == 2
history = await svc.get_history()
remaining = {item["download_id"] for item in history["items"]}
assert remaining == {"dl-x2", "dl-y2"}