feat(download): per-file download status and multi-file selection (#1058)

This commit is contained in:
Will Miao
2026-08-19 17:51:31 +08:00
parent cef4129fc9
commit e7c26bf722
23 changed files with 22259 additions and 21031 deletions
+14
View File
@@ -976,6 +976,7 @@ class DownloadManager:
version_info,
record.get("model_version_id"),
record.get("save_path") or record.get("file_path"),
file_info=file_info,
)
await self._sync_downloaded_version(
model_type,
@@ -1860,6 +1861,7 @@ class DownloadManager:
version_info,
model_version_id,
save_path,
file_info=file_info,
)
await self._sync_downloaded_version(
model_type,
@@ -1902,6 +1904,7 @@ class DownloadManager:
version_info: Dict[str, Any],
fallback_version_id=None,
file_path: str | None = None,
file_info: Dict[str, Any] | None = None,
) -> None:
try:
history_service = await ServiceRegistry.get_downloaded_version_history_service()
@@ -1927,6 +1930,15 @@ class DownloadManager:
if version_id is None:
version_id = fallback_version_id
# Per-file identity for multi-file versions (#1058)
file_id = None
file_name = None
if isinstance(file_info, dict):
file_id = file_info.get("id")
raw_file_name = file_info.get("name")
if isinstance(raw_file_name, str) and raw_file_name.strip():
file_name = raw_file_name.strip()
try:
await history_service.mark_downloaded(
model_type,
@@ -1934,6 +1946,8 @@ class DownloadManager:
model_id=int(cast(Any, resolved_model_id)) if resolved_model_id is not None else None,
source="download",
file_path=file_path,
file_id=file_id,
file_name=file_name,
)
except (TypeError, ValueError):
logger.debug(
@@ -62,6 +62,14 @@ class DownloadedVersionHistoryService:
);
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
ON downloaded_model_versions(model_type, model_id);
CREATE TABLE IF NOT EXISTS downloaded_version_files (
model_type TEXT NOT NULL,
version_id INTEGER NOT NULL,
file_id INTEGER NOT NULL,
file_name TEXT,
downloaded_at REAL NOT NULL,
PRIMARY KEY (model_type, version_id, file_id)
);
"""
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
@@ -131,10 +139,13 @@ class DownloadedVersionHistoryService:
source: str = "manual",
file_path: str | None = None,
library_name: str | None = None,
file_id: int | None = None,
file_name: str | None = None,
) -> None:
normalized_type = _normalize_model_type(model_type)
normalized_version_id = _normalize_int(version_id)
normalized_model_id = _normalize_int(model_id)
normalized_file_id = _normalize_int(file_id)
if normalized_type is None or normalized_version_id is None:
return
@@ -168,6 +179,25 @@ class DownloadedVersionHistoryService:
active_library_name,
),
)
if normalized_file_id is not None:
# Per-file history for multi-file versions (#1058)
conn.execute(
"""
INSERT INTO downloaded_version_files (
model_type, version_id, file_id, file_name, downloaded_at
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(model_type, version_id, file_id) DO UPDATE SET
file_name = COALESCE(excluded.file_name, downloaded_version_files.file_name),
downloaded_at = excluded.downloaded_at
""",
(
normalized_type,
normalized_version_id,
normalized_file_id,
file_name,
timestamp,
),
)
conn.commit()
async def mark_downloaded_bulk(
@@ -255,8 +285,63 @@ class DownloadedVersionHistoryService:
self._get_active_library_name(),
),
)
# Whole-version deletion also clears the per-file records (#1058)
conn.execute(
"""
DELETE FROM downloaded_version_files
WHERE model_type = ? AND version_id = ?
""",
(normalized_type, normalized_version_id),
)
conn.commit()
async def mark_file_deleted(
self, model_type: str, version_id: int, file_id: int
) -> None:
"""Drop a single file record of a version, keeping siblings (#1058)."""
normalized_type = _normalize_model_type(model_type)
normalized_version_id = _normalize_int(version_id)
normalized_file_id = _normalize_int(file_id)
if (
normalized_type is None
or normalized_version_id is None
or normalized_file_id is None
):
return
async with self._lock:
conn = self._get_conn()
conn.execute(
"""
DELETE FROM downloaded_version_files
WHERE model_type = ? AND version_id = ? AND file_id = ?
""",
(normalized_type, normalized_version_id, normalized_file_id),
)
conn.commit()
async def get_downloaded_file_ids(
self, model_type: str, version_id: int
) -> list[int]:
"""Return the CivitAI file ids recorded as downloaded for a version."""
normalized_type = _normalize_model_type(model_type)
normalized_version_id = _normalize_int(version_id)
if normalized_type is None or normalized_version_id is None:
return []
async with self._lock:
conn = self._get_conn()
rows = conn.execute(
"""
SELECT file_id
FROM downloaded_version_files
WHERE model_type = ? AND version_id = ?
ORDER BY file_id ASC
""",
(normalized_type, normalized_version_id),
).fetchall()
return [int(row["file_id"]) for row in rows]
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
normalized_type = _normalize_model_type(model_type)
normalized_version_id = _normalize_int(version_id)
+65 -1
View File
@@ -35,6 +35,10 @@ class ModelCache:
folders: List[str]
version_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
model_id_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
# Multi-valued companion to version_index: every local file entry of a
# CivitAI model version, so versions with several downloaded files stay
# consistent (#1058).
version_files_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
name_display_mode: str = "model_name"
_lock: Any = field(init=False, repr=False, default=None)
# Cache for last sort: (sort_key, order, seed) -> sorted list
@@ -116,6 +120,7 @@ class ModelCache:
self.version_index = {}
self.model_id_index = {}
self.version_files_index = {}
for item in self.raw_data:
self.add_to_version_index(item)
@@ -132,6 +137,17 @@ class ModelCache:
self.version_index[version_id] = item
# Register in the multi-valued index, deduplicated by file_path (#1058)
files = self.version_files_index.setdefault(version_id, [])
for entry in files:
if entry is item or (
isinstance(entry, dict)
and entry.get('file_path') == item.get('file_path')
):
break
else:
files.append(item)
model_id = self._normalize_version_id(civitai_data.get('modelId'))
if model_id is None:
return
@@ -159,12 +175,37 @@ class ModelCache:
if version_id is None:
return
# Drop only this file's entry from the multi-valued index (#1058)
files = self.version_files_index.get(version_id)
if files:
remaining = [
entry
for entry in files
if not (
entry is item
or (
isinstance(entry, dict)
and entry.get('file_path') == item.get('file_path')
)
)
]
if remaining:
self.version_files_index[version_id] = remaining
else:
self.version_files_index.pop(version_id, None)
# A surviving sibling file keeps the version present in the indexes
sibling = (self.version_files_index.get(version_id) or [None])[0]
existing = self.version_index.get(version_id)
if existing is item or (
isinstance(existing, dict)
and existing.get('file_path') == item.get('file_path')
):
self.version_index.pop(version_id, None)
if sibling is not None:
self.version_index[version_id] = sibling
else:
self.version_index.pop(version_id, None)
model_id = self._normalize_version_id(civitai_data.get('modelId'))
if model_id is None:
@@ -174,6 +215,20 @@ class ModelCache:
if not versions:
return
if sibling is not None:
# Update the descriptor to reflect the surviving sibling file
descriptor = self._build_version_descriptor(
sibling,
sibling.get('civitai') if isinstance(sibling, dict) else {},
version_id,
)
for index, existing_desc in enumerate(versions):
if existing_desc.get('versionId') == version_id:
if descriptor is not None:
versions[index] = descriptor
break
return
filtered = [v for v in versions if v.get('versionId') != version_id]
if filtered:
self.model_id_index[model_id] = filtered
@@ -206,6 +261,15 @@ class ModelCache:
versions = self.model_id_index.get(normalized_id, [])
return [dict(version) for version in versions]
def get_files_by_version_id(self, version_id: Any) -> List[Dict[str, Any]]:
"""Return every local file entry for a CivitAI model version (#1058)."""
normalized_id = self._normalize_version_id(version_id)
if normalized_id is None:
return []
return list(self.version_files_index.get(normalized_id, []))
async def resort(self):
"""Resort cached data according to last sort mode if set"""
async with self._lock:
+33
View File
@@ -2446,6 +2446,39 @@ class ModelScanner:
logger.error(f"Error checking model version existence: {e}")
return False
async def get_files_for_version(self, model_version_id: int) -> List[Dict[str, Any]]:
"""Get all local file entries for a specific model version (#1058).
A Civitai model version can have several weight files downloaded;
unlike the single-valued version_index this returns every entry.
Args:
model_version_id: Civitai model version ID
Returns:
List[Dict]: Cache entries (may be empty)
"""
try:
normalized_id = int(model_version_id)
except (TypeError, ValueError):
return []
try:
cache = await self.get_cached_data()
if not cache:
return []
getter = getattr(cache, "get_files_by_version_id", None)
if getter is not None:
return getter(normalized_id)
# Fallback for cache implementations without the multi-file index
entry = cache.version_index.get(normalized_id)
return [entry] if entry is not None else []
except Exception as e:
logger.error(f"Error getting files for model version: {e}")
return []
async def get_model_versions_by_id(self, model_id: int) -> List[Dict[str, Any]]:
"""Get all versions of a model by its ID