mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-23 12:04:09 -03:00
Compare commits
5
Commits
b80830913c
...
86aa1d8059
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
86aa1d8059 | ||
|
|
74254756ef | ||
|
|
259e08e47c | ||
|
|
6647c45731 | ||
|
|
b614a5c447 |
@@ -346,8 +346,3 @@ WeChat: [Click to view QR code](https://raw.githubusercontent.com/willmiao/Comfy
|
||||
|
||||
Join our Discord community for support, discussions, and updates:
|
||||
[Discord Server](https://discord.gg/vcqNrWVFvM)
|
||||
|
||||
---
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/?repos=willmiao%2FComfyUI-Lora-Manager&type=date&legend=top-left)
|
||||
|
||||
@@ -56,6 +56,7 @@ from ...utils.constants import (
|
||||
)
|
||||
from .hf_handlers import HfHandler
|
||||
from .agent_handlers import AgentHandler
|
||||
from .model_handlers import ModelCivitaiHandler
|
||||
from ...utils.civitai_utils import rewrite_preview_url
|
||||
from ...utils.example_images_paths import (
|
||||
find_non_compliant_items_in_example_images_root,
|
||||
@@ -2061,6 +2062,63 @@ class ModelLibraryHandler:
|
||||
enriched.append(entry)
|
||||
return enriched
|
||||
|
||||
@staticmethod
|
||||
async def _get_downloaded_files(
|
||||
scanner: Any, model_version_id: int
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return per-file downloaded state for a version in the library.
|
||||
|
||||
This handler has no CivitAI version payload, so the remote file list
|
||||
is taken from the local entries' cached ``civitai`` metadata (the
|
||||
full version payload persisted at download time, see
|
||||
``BaseModelMetadata.from_civitai_info``) and matched with the same
|
||||
D2 rule used by ``get_civitai_versions`` (#1058). Local entries that
|
||||
cannot be matched to a known remote file (e.g. missing metadata or
|
||||
renamed files) are still reported with ``fileId`` set to None.
|
||||
Returns ``[{fileId, fileName, filePath}]``.
|
||||
"""
|
||||
try:
|
||||
cache = await scanner.get_cached_data()
|
||||
except Exception: # pragma: no cover - defensive fallback
|
||||
logger.debug(
|
||||
"Failed to read cache for downloaded files of version %s",
|
||||
model_version_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return []
|
||||
|
||||
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||
local_entries = files_getter(model_version_id) if files_getter else []
|
||||
if not local_entries:
|
||||
return []
|
||||
|
||||
version_payload: Mapping[str, Any] = {}
|
||||
for entry in local_entries:
|
||||
civitai = entry.get("civitai") if isinstance(entry, Mapping) else None
|
||||
if isinstance(civitai, Mapping) and isinstance(civitai.get("files"), list):
|
||||
version_payload = civitai
|
||||
break
|
||||
|
||||
downloaded = ModelCivitaiHandler._match_downloaded_files(
|
||||
version_payload, local_entries
|
||||
)
|
||||
|
||||
# Surface local files that D2 could not map to a known remote file
|
||||
matched_paths = {item.get("filePath") for item in downloaded}
|
||||
for entry in local_entries:
|
||||
if not isinstance(entry, Mapping):
|
||||
continue
|
||||
if entry.get("file_path") in matched_paths:
|
||||
continue
|
||||
downloaded.append(
|
||||
{
|
||||
"fileId": None,
|
||||
"fileName": entry.get("file_name"),
|
||||
"filePath": entry.get("file_path"),
|
||||
}
|
||||
)
|
||||
return downloaded
|
||||
|
||||
async def check_model_exists(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
model_id_str = request.query.get("modelId")
|
||||
@@ -2096,9 +2154,11 @@ class ModelLibraryHandler:
|
||||
|
||||
exists = False
|
||||
model_type = None
|
||||
matched_scanner = None
|
||||
if await lora_scanner.check_model_version_exists(model_version_id):
|
||||
exists = True
|
||||
model_type = "lora"
|
||||
matched_scanner = lora_scanner
|
||||
elif (
|
||||
checkpoint_scanner
|
||||
and await checkpoint_scanner.check_model_version_exists(
|
||||
@@ -2107,6 +2167,7 @@ class ModelLibraryHandler:
|
||||
):
|
||||
exists = True
|
||||
model_type = "checkpoint"
|
||||
matched_scanner = checkpoint_scanner
|
||||
elif (
|
||||
embedding_scanner
|
||||
and await embedding_scanner.check_model_version_exists(
|
||||
@@ -2115,6 +2176,7 @@ class ModelLibraryHandler:
|
||||
):
|
||||
exists = True
|
||||
model_type = "embedding"
|
||||
matched_scanner = embedding_scanner
|
||||
|
||||
if exists:
|
||||
return web.json_response(
|
||||
@@ -2123,6 +2185,9 @@ class ModelLibraryHandler:
|
||||
"exists": True,
|
||||
"modelType": model_type,
|
||||
"hasBeenDownloaded": False,
|
||||
"downloadedFiles": await self._get_downloaded_files(
|
||||
matched_scanner, model_version_id
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -2144,6 +2209,7 @@ class ModelLibraryHandler:
|
||||
"exists": False,
|
||||
"modelType": history_type,
|
||||
"hasBeenDownloaded": has_been_downloaded,
|
||||
"downloadedFiles": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,15 @@ from ..utils.cache_paths import get_cache_base_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# SQL fragment extracting the CivitAI file id from the JSON ``file_params``
|
||||
# column (#1058). ``json_valid`` guards against NULL and legacy/unparseable
|
||||
# values, yielding NULL for rows without a file identity; NULL keys group
|
||||
# together so such rows keep the old version-level dedup behavior.
|
||||
_FILE_ID_SQL = (
|
||||
"CASE WHEN json_valid(file_params) "
|
||||
"THEN json_extract(file_params, '$.id') END"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_database_path() -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
@@ -866,33 +875,44 @@ class DownloadQueueService:
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
# 1. History: for each (model_id, model_version_id, status) triplet
|
||||
# keep only the row with the highest id (most recently inserted).
|
||||
conn.execute("""
|
||||
# 1. History: for each (model_id, model_version_id, file_id,
|
||||
# status) group keep only the row with the highest id (most
|
||||
# recently inserted). file_id comes from file_params (#1058)
|
||||
# so distinct files of the same version never collapse.
|
||||
conn.execute(f"""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT MAX(id)
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id, status
|
||||
GROUP BY model_id, model_version_id, status,
|
||||
{_FILE_ID_SQL}
|
||||
)
|
||||
""")
|
||||
result["removed_history"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
||||
# keep only the entry with the highest-priority terminal status.
|
||||
# 2. Cross-status dedup: for each (model_id, model_version_id,
|
||||
# file_id), keep only the entry with the highest-priority
|
||||
# terminal status.
|
||||
# Priority: completed (3) > failed (2) > canceled (1).
|
||||
# This prevents the same model version from having both a
|
||||
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
||||
# either) after the bug-created duplicates are removed.
|
||||
conn.execute("""
|
||||
# This prevents the same file of a model version from having
|
||||
# both a 'failed' and a 'canceled' entry (or a 'completed'
|
||||
# alongside either) after the bug-created duplicates are
|
||||
# removed. ``IS`` matches NULL file ids against each other so
|
||||
# rows without file identity keep the old behavior.
|
||||
conn.execute(f"""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT dh.id
|
||||
FROM download_history dh
|
||||
FROM (
|
||||
SELECT id, model_id, model_version_id, status,
|
||||
{_FILE_ID_SQL} AS file_id
|
||||
FROM download_history
|
||||
) dh
|
||||
INNER JOIN (
|
||||
SELECT model_id, model_version_id,
|
||||
{_FILE_ID_SQL} AS file_id,
|
||||
MAX(CASE status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
@@ -900,17 +920,18 @@ class DownloadQueueService:
|
||||
ELSE 0
|
||||
END) AS best_prio
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id
|
||||
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||
) best
|
||||
ON dh.model_id = best.model_id
|
||||
AND dh.model_version_id = best.model_version_id
|
||||
AND dh.file_id IS best.file_id
|
||||
AND CASE dh.status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
WHEN 'canceled' THEN 1
|
||||
ELSE 0
|
||||
END = best.best_prio
|
||||
GROUP BY dh.model_id, dh.model_version_id
|
||||
GROUP BY dh.model_id, dh.model_version_id, dh.file_id
|
||||
HAVING dh.id = MAX(dh.id)
|
||||
)
|
||||
""")
|
||||
@@ -918,15 +939,17 @@ class DownloadQueueService:
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 3. Queue: for each (model_id, model_version_id) keep only the
|
||||
# row with the latest added_at (most recently enqueued).
|
||||
conn.execute("""
|
||||
# 3. Queue: for each (model_id, model_version_id, file_id) keep
|
||||
# only the row with the latest added_at (most recently
|
||||
# enqueued). file_id comes from file_params (#1058) so
|
||||
# distinct files of the same version never collapse.
|
||||
conn.execute(f"""
|
||||
DELETE FROM download_queue
|
||||
WHERE rowid NOT IN (
|
||||
SELECT MAX(rowid)
|
||||
FROM download_queue
|
||||
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
GROUP BY model_id, model_version_id
|
||||
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||
)
|
||||
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
""")
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
position: fixed;
|
||||
top: 0;
|
||||
z-index: var(--z-header);
|
||||
height: 48px;
|
||||
height: var(--header-height, 48px);
|
||||
/* Reduced height */
|
||||
width: 100%;
|
||||
box-shadow: var(--shadow-md);
|
||||
|
||||
@@ -1,10 +1,3 @@
|
||||
/* Blurred backdrop for the model modal to match the recipe modal */
|
||||
#modelModal {
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
backdrop-filter: blur(6px);
|
||||
-webkit-backdrop-filter: blur(6px);
|
||||
}
|
||||
|
||||
/* Lora Modal Header */
|
||||
.modal-header {
|
||||
display: flex;
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: calc(100% - var(--header-height, 48px)); /* Adjust height to exclude header */
|
||||
background: rgba(0, 0, 0, 0.2);
|
||||
background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
|
||||
backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
|
||||
-webkit-backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
|
||||
z-index: var(--z-modal);
|
||||
overflow: auto; /* Change from hidden to auto to allow scrolling */
|
||||
}
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
/* Darker than --modal-backdrop-bg to stress destructive actions, but keeps the shared blur */
|
||||
background: rgba(0, 0, 0, 0.8);
|
||||
backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
|
||||
-webkit-backdrop-filter: blur(var(--modal-backdrop-blur, 6px));
|
||||
z-index: var(--z-overlay);
|
||||
}
|
||||
|
||||
|
||||
@@ -104,13 +104,6 @@
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Darker, blurred backdrop keeps the busy page behind the modal from bleeding through */
|
||||
#recipeModal {
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
backdrop-filter: blur(6px);
|
||||
-webkit-backdrop-filter: blur(6px);
|
||||
}
|
||||
|
||||
#recipeModal .modal-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
border: none;
|
||||
padding: 8px 16px;
|
||||
font-size: 0.9em;
|
||||
transform: translateX(-50%) translateY(20px);
|
||||
transform: translateY(20px);
|
||||
}
|
||||
|
||||
.toast.toast-copy.show {
|
||||
transform: translateX(-50%) translateY(0);
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
/* Toast Notifications */
|
||||
@@ -19,14 +19,15 @@
|
||||
right: 20px;
|
||||
left: auto;
|
||||
transform: translateX(120%);
|
||||
min-width: 300px;
|
||||
box-sizing: border-box;
|
||||
min-width: 200px;
|
||||
max-width: 400px;
|
||||
background: var(--lora-surface);
|
||||
color: var(--text-color);
|
||||
padding: 12px 16px;
|
||||
border-radius: var(--border-radius-sm);
|
||||
box-shadow: var(--shadow-toast);
|
||||
z-index: calc(var(--z-overlay) + 10);
|
||||
z-index: var(--z-toast);
|
||||
opacity: 0;
|
||||
transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1),
|
||||
opacity 0.3s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
@@ -130,7 +131,6 @@
|
||||
.toast {
|
||||
width: calc(100% - 40px);
|
||||
max-width: none;
|
||||
right: 20px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,16 +166,17 @@
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Toast Container for stacked notifications */
|
||||
/* Toast Container for stacked notifications (top-right, flush below the header) */
|
||||
.toast-container {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
top: var(--header-height, 48px); /* Start right below the fixed header */
|
||||
right: 0;
|
||||
z-index: calc(var(--z-overlay) + 10);
|
||||
z-index: var(--z-toast);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-end;
|
||||
gap: 10px;
|
||||
padding: 20px;
|
||||
padding: 8px 20px 0; /* Small breathing room below the header */
|
||||
pointer-events: none; /* Allow clicking through the container */
|
||||
width: 400px;
|
||||
max-width: 100%;
|
||||
@@ -215,8 +216,7 @@
|
||||
/* Responsive adjustments */
|
||||
@media (max-width: 480px) {
|
||||
.toast-container {
|
||||
width: 100%;
|
||||
padding: 10px;
|
||||
padding: 0 10px;
|
||||
}
|
||||
|
||||
.toast {
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
--shadow-dialog: 0 10px 24px rgba(0, 0, 0, 0.25);
|
||||
--shadow-inset-top: 0 -2px 8px rgba(0, 0, 0, 0.1);
|
||||
|
||||
--modal-backdrop-bg: rgba(0, 0, 0, 0.5);
|
||||
--modal-backdrop-blur: 6px;
|
||||
|
||||
--transition-fast: 150ms ease;
|
||||
--transition-base: 200ms ease;
|
||||
--transition-slow: 300ms ease;
|
||||
|
||||
@@ -908,6 +908,33 @@ class FakeExistenceScanner:
|
||||
return []
|
||||
|
||||
|
||||
class FakeVersionFilesCache:
|
||||
"""Cache stub exposing the multi-valued version->files index (#1058)."""
|
||||
|
||||
def __init__(self, entries):
|
||||
self._entries = entries
|
||||
|
||||
def get_files_by_version_id(self, _version_id):
|
||||
return list(self._entries)
|
||||
|
||||
|
||||
class FakeDownloadedFilesScanner:
|
||||
"""Scanner stub with one known version and per-file cache entries."""
|
||||
|
||||
def __init__(self, version_id, entries):
|
||||
self._version_id = version_id
|
||||
self._cache = FakeVersionFilesCache(entries)
|
||||
|
||||
async def check_model_version_exists(self, version_id):
|
||||
return version_id == self._version_id
|
||||
|
||||
async def get_model_versions_by_id(self, _model_id):
|
||||
return []
|
||||
|
||||
async def get_cached_data(self):
|
||||
return self._cache
|
||||
|
||||
|
||||
class FakeMetadataProvider:
|
||||
async def get_model_versions(self, _model_id):
|
||||
return {"modelVersions": [], "name": "", "type": "lora"}
|
||||
@@ -1524,9 +1551,125 @@ async def test_check_model_exists_returns_download_history_when_file_missing():
|
||||
"exists": False,
|
||||
"modelType": "checkpoint",
|
||||
"hasBeenDownloaded": True,
|
||||
"downloadedFiles": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_model_exists_returns_downloaded_files():
|
||||
"""The version branch exposes per-file downloaded state (#1058)."""
|
||||
civitai_payload = {
|
||||
"id": 100,
|
||||
"files": [
|
||||
{
|
||||
"id": 1001,
|
||||
"name": "model-fp16.safetensors",
|
||||
"hashes": {"SHA256": "A" * 64},
|
||||
},
|
||||
{
|
||||
"id": 1002,
|
||||
"name": "model-fp32.safetensors",
|
||||
"hashes": {"SHA256": "B" * 64},
|
||||
},
|
||||
],
|
||||
}
|
||||
entries = [
|
||||
{
|
||||
"file_name": "model-fp16",
|
||||
"file_path": "/models/loras/model-fp16.safetensors",
|
||||
"sha256": "a" * 64,
|
||||
"civitai": civitai_payload,
|
||||
},
|
||||
{
|
||||
# Legacy entry without a hash: matched by extension-less name (D2)
|
||||
"file_name": "model-fp32",
|
||||
"file_path": "/models/loras/model-fp32.safetensors",
|
||||
"sha256": "",
|
||||
"civitai": civitai_payload,
|
||||
},
|
||||
]
|
||||
lora_scanner = FakeDownloadedFilesScanner(100, entries)
|
||||
|
||||
async def lora_factory():
|
||||
return lora_scanner
|
||||
|
||||
handler = ModelLibraryHandler(
|
||||
ServiceRegistryAdapter(
|
||||
get_lora_scanner=lora_factory,
|
||||
get_checkpoint_scanner=fake_scanner_factory,
|
||||
get_embedding_scanner=fake_scanner_factory,
|
||||
get_downloaded_version_history_service=fake_download_history_service_factory,
|
||||
),
|
||||
metadata_provider_factory=fake_metadata_provider_factory,
|
||||
)
|
||||
|
||||
response = await handler.check_model_exists(
|
||||
FakeRequest(query={"modelId": "5", "modelVersionId": "100"}) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert payload == {
|
||||
"success": True,
|
||||
"exists": True,
|
||||
"modelType": "lora",
|
||||
"hasBeenDownloaded": False,
|
||||
"downloadedFiles": [
|
||||
{
|
||||
"fileId": 1001,
|
||||
"fileName": "model-fp16.safetensors",
|
||||
"filePath": "/models/loras/model-fp16.safetensors",
|
||||
},
|
||||
{
|
||||
"fileId": 1002,
|
||||
"fileName": "model-fp32.safetensors",
|
||||
"filePath": "/models/loras/model-fp32.safetensors",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_model_exists_downloaded_files_without_remote_metadata():
|
||||
"""Unmatchable local files are still reported with fileId None (#1058)."""
|
||||
entries = [
|
||||
{
|
||||
"file_name": "renamed-model",
|
||||
"file_path": "/models/loras/renamed-model.safetensors",
|
||||
"sha256": "c" * 64,
|
||||
"civitai": {"id": 100}, # no remote files list cached
|
||||
},
|
||||
]
|
||||
lora_scanner = FakeDownloadedFilesScanner(100, entries)
|
||||
|
||||
async def lora_factory():
|
||||
return lora_scanner
|
||||
|
||||
handler = ModelLibraryHandler(
|
||||
ServiceRegistryAdapter(
|
||||
get_lora_scanner=lora_factory,
|
||||
get_checkpoint_scanner=fake_scanner_factory,
|
||||
get_embedding_scanner=fake_scanner_factory,
|
||||
get_downloaded_version_history_service=fake_download_history_service_factory,
|
||||
),
|
||||
metadata_provider_factory=fake_metadata_provider_factory,
|
||||
)
|
||||
|
||||
response = await handler.check_model_exists(
|
||||
FakeRequest(query={"modelId": "5", "modelVersionId": "100"}) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["exists"] is True
|
||||
assert payload["downloadedFiles"] == [
|
||||
{
|
||||
"fileId": None,
|
||||
"fileName": "renamed-model",
|
||||
"filePath": "/models/loras/renamed-model.safetensors",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_version_download_status_endpoints():
|
||||
history_service = FakeDownloadHistoryService({"lora": {123}})
|
||||
|
||||
@@ -7,6 +7,7 @@ compatibility with ``id``.
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -321,3 +322,176 @@ async def test_legacy_history_db_gains_file_params_column(tmp_path: Path) -> Non
|
||||
item = await svc.retry_from_history(download_id="dl-legacy-fp")
|
||||
assert item is not None
|
||||
assert json.loads(item["file_params"]) == {"id": 5}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# deduplicate() — file identity in the dedup key (#1058)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_queue_keeps_distinct_files_of_same_version(tmp_path: Path) -> None:
|
||||
"""Two queued downloads of different files of one version both survive."""
|
||||
svc = _make_service(tmp_path)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-a",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
file_params={"id": 1001, "type": "Model"},
|
||||
)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-b",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
file_params={"id": 1002, "type": "Model"},
|
||||
)
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_queue"] == 0
|
||||
queue = await svc.get_queue()
|
||||
assert sorted(item["download_id"] for item in queue) == ["dl-a", "dl-b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_queue_collapses_same_file_and_legacy_rows(tmp_path: Path) -> None:
|
||||
"""Same version + same/absent file id still collapses to the latest row."""
|
||||
svc = _make_service(tmp_path)
|
||||
# Same file id -> only the most recently enqueued row survives
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-old",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-new",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
# Rows without file identity keep the old per-version behavior
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-legacy-old", model_id=2, model_version_id=200
|
||||
)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-legacy-new", model_id=2, model_version_id=200
|
||||
)
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_queue"] == 2
|
||||
remaining = {item["download_id"] for item in await svc.get_queue()}
|
||||
assert remaining == {"dl-new", "dl-legacy-new"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_queue_does_not_mix_null_and_file_id(tmp_path: Path) -> None:
|
||||
"""A row with a file id never dedups against a row without one."""
|
||||
svc = _make_service(tmp_path)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-file",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-nofile", model_id=1, model_version_id=100
|
||||
)
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_queue"] == 0
|
||||
assert len(await svc.get_queue()) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_queue_unparseable_file_params_treated_as_none(tmp_path: Path) -> None:
|
||||
"""Corrupt file_params JSON falls back to the NULL file identity."""
|
||||
svc = _make_service(tmp_path)
|
||||
await svc.add_to_queue(
|
||||
download_id="dl-plain", model_id=1, model_version_id=100
|
||||
)
|
||||
conn = sqlite3.connect(str(tmp_path / "queue.sqlite"))
|
||||
conn.execute(
|
||||
"INSERT INTO download_queue (download_id, model_id, model_version_id, "
|
||||
"file_params, status, added_at) VALUES (?, ?, ?, ?, 'queued', ?)",
|
||||
("dl-corrupt", 1, 100, "{not json", time.time()),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_queue"] == 1
|
||||
assert len(await svc.get_queue()) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_history_keeps_distinct_files_of_same_version(tmp_path: Path) -> None:
|
||||
"""History rows of different files never collapse, even across statuses."""
|
||||
svc = _make_service(tmp_path)
|
||||
await svc.add_to_history(
|
||||
download_id="dl-h1",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
status="completed",
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
await svc.add_to_history(
|
||||
download_id="dl-h2",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
status="completed",
|
||||
file_params={"id": 1002},
|
||||
)
|
||||
# A failed entry for file 1002 must not remove file 1001's completed row
|
||||
await svc.add_to_history(
|
||||
download_id="dl-h3",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
status="failed",
|
||||
file_params={"id": 1002},
|
||||
)
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_history"] == 1 # only dl-h3 collapses (same file)
|
||||
history = await svc.get_history()
|
||||
remaining = {item["download_id"] for item in history["items"]}
|
||||
assert remaining == {"dl-h1", "dl-h2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_history_collapses_same_file_and_legacy_rows(tmp_path: Path) -> None:
|
||||
"""Same file id / absent file id history rows still dedup as before."""
|
||||
svc = _make_service(tmp_path)
|
||||
# Same file id + same status -> keep the most recent row
|
||||
await svc.add_to_history(
|
||||
download_id="dl-x1",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
status="completed",
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
await svc.add_to_history(
|
||||
download_id="dl-x2",
|
||||
model_id=1,
|
||||
model_version_id=100,
|
||||
status="completed",
|
||||
file_params={"id": 1001},
|
||||
)
|
||||
# Legacy rows without file identity: cross-status dedup still applies
|
||||
await svc.add_to_history(
|
||||
download_id="dl-y1", model_id=2, model_version_id=200, status="failed"
|
||||
)
|
||||
await svc.add_to_history(
|
||||
download_id="dl-y2", model_id=2, model_version_id=200, status="completed"
|
||||
)
|
||||
|
||||
result = await svc.deduplicate()
|
||||
|
||||
assert result["removed_history"] == 2
|
||||
history = await svc.get_history()
|
||||
remaining = {item["download_id"] for item in history["items"]}
|
||||
assert remaining == {"dl-x2", "dl-y2"}
|
||||
|
||||
Reference in New Issue
Block a user