mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-29 08:51:27 -03:00
fix(downloads): preserve aria2 partial pair and refresh expired CivitAI signed URLs
A failed aria2 transfer deleted the partial payload while keeping its .aria2 control file, and "No URI available" (expired CivitAI signed URL) was treated as a permanent failure, wasting nearly-complete downloads. - Re-schedule the transfer with a freshly resolved signed URL and continue=true when aria2 reports "No URI available", bounded by MAX_TRANSFER_RECOVERY_ATTEMPTS - Keep payload and .aria2 control file together as a resumable pair after a failed transfer instead of deleting the payload - Report and remove orphaned .aria2 control files that have no payload, both after failures and when restoring persisted downloads Fixes #1088
This commit is contained in:
Will Miao
@@ -82,6 +82,17 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
||||
)
|
||||
|
||||
|
||||
def _is_no_uri_available_error(message: str) -> bool:
|
||||
"""Return True for aria2's "No URI available" transfer failure.
|
||||
|
||||
aria2 reports this when every URI for the transfer has become unusable.
|
||||
For CivitAI downloads this typically means the temporary signed URL
|
||||
expired mid-download; the transfer can be recovered by resolving a fresh
|
||||
signed URL and re-scheduling with ``continue=true``.
|
||||
"""
|
||||
return "no uri available" in message.lower()
|
||||
|
||||
|
||||
class Aria2Error(RuntimeError):
|
||||
"""Raised when aria2 integration fails."""
|
||||
|
||||
@@ -145,8 +156,11 @@ class Aria2Downloader:
|
||||
disappears (e.g. another download restarted the daemon and
|
||||
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
||||
the transfer is re-scheduled with ``continue=true`` so the download
|
||||
resumes from the on-disk ``.aria2`` control file. Recovery is bounded
|
||||
by ``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||
resumes from the on-disk ``.aria2`` control file. The same
|
||||
re-scheduling happens when aria2 fails with "No URI available"
|
||||
(typically an expired CivitAI signed URL): a fresh URL is resolved
|
||||
and the partial download continues. Recovery is bounded by
|
||||
``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||
"""
|
||||
|
||||
await self._ensure_process()
|
||||
@@ -201,7 +215,36 @@ class Aria2Downloader:
|
||||
completed_path = self._resolve_completed_path(status, save_path)
|
||||
return True, completed_path
|
||||
if state == "error":
|
||||
return False, status.get("errorMessage") or "aria2 download failed"
|
||||
error_message = status.get("errorMessage") or "aria2 download failed"
|
||||
if (
|
||||
_is_no_uri_available_error(error_message)
|
||||
and recovery_attempts < MAX_TRANSFER_RECOVERY_ATTEMPTS
|
||||
):
|
||||
# The signed URL (e.g. CivitAI's) expired before the
|
||||
# transfer finished. Re-registering resolves a fresh
|
||||
# URL and resumes from the on-disk partial payload and
|
||||
# .aria2 control file via ``continue=true``.
|
||||
recovery_attempts += 1
|
||||
logger.warning(
|
||||
"aria2 transfer %s failed with %r; refreshing the "
|
||||
"URL and resuming the partial download "
|
||||
"(attempt %d/%d)",
|
||||
download_id,
|
||||
error_message,
|
||||
recovery_attempts,
|
||||
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||
)
|
||||
await asyncio.sleep(1.0)
|
||||
await self._ensure_process()
|
||||
async with self._register_lock:
|
||||
transfer = await self._register_transfer(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
continue
|
||||
return False, error_message
|
||||
if state == "removed":
|
||||
return False, "Download was cancelled"
|
||||
|
||||
|
||||
@@ -717,6 +717,47 @@ class DownloadManager:
|
||||
await asyncio.sleep(delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _reconcile_failed_aria2_partial(save_path: str) -> None:
|
||||
"""Reconcile on-disk partial state after a failed aria2 transfer.
|
||||
|
||||
The payload and its ``.aria2`` control file form a resumable pair and
|
||||
are preserved together so a retry (with a refreshed URL when needed)
|
||||
can resume via aria2's ``continue=true``. A control file without its
|
||||
payload cannot resume anything, so the orphan is reported and removed.
|
||||
"""
|
||||
control_path = f"{save_path}.aria2"
|
||||
payload_exists = os.path.exists(save_path)
|
||||
control_exists = os.path.exists(control_path)
|
||||
|
||||
if payload_exists and not control_exists:
|
||||
# If the .aria2 control file is missing, aria2 considers the
|
||||
# download complete. A transient RPC failure may have made us
|
||||
# think the download failed even though the file is fully on disk.
|
||||
# Keep the file so a retry can find it already complete.
|
||||
logger.warning(
|
||||
"aria2 download reported failure but .aria2 file is absent "
|
||||
"for %s — the file is likely complete. Preserving it for retry.",
|
||||
save_path,
|
||||
)
|
||||
elif payload_exists and control_exists:
|
||||
logger.info(
|
||||
"Preserving aria2 partial download for resume: %s", save_path
|
||||
)
|
||||
elif control_exists:
|
||||
logger.warning(
|
||||
"Orphaned aria2 control file without payload: %s — removing it",
|
||||
control_path,
|
||||
)
|
||||
try:
|
||||
os.remove(control_path)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to remove orphaned aria2 control file %s: %s",
|
||||
control_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
async def _cleanup_cancelled_download_files(
|
||||
self,
|
||||
download_id: str,
|
||||
@@ -1226,6 +1267,24 @@ class DownloadManager:
|
||||
)
|
||||
continue
|
||||
|
||||
if not os.path.exists(save_path) and os.path.exists(control_path):
|
||||
# A control file without its payload cannot resume
|
||||
# anything; report it and clean up the orphan.
|
||||
logger.warning(
|
||||
"Orphaned aria2 control file without payload for %s: "
|
||||
"%s — removing it",
|
||||
download_id,
|
||||
control_path,
|
||||
)
|
||||
try:
|
||||
os.remove(control_path)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to remove orphaned aria2 control file %s: %s",
|
||||
control_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
await self._aria2_state_store.remove(download_id)
|
||||
|
||||
self._restored_persisted_downloads = True
|
||||
@@ -2423,20 +2482,8 @@ class DownloadManager:
|
||||
break
|
||||
|
||||
last_error = result
|
||||
# For aria2: if the .aria2 control file is missing, aria2 considers
|
||||
# the download complete. A transient RPC failure may have made us
|
||||
# think the download failed even though the file is fully on disk.
|
||||
# Keep the file so a retry can find it already complete.
|
||||
if (
|
||||
transfer_backend == "aria2"
|
||||
and os.path.exists(save_path)
|
||||
and not os.path.exists(f"{save_path}.aria2")
|
||||
):
|
||||
logger.warning(
|
||||
"aria2 download reported failure but .aria2 file is absent "
|
||||
"for %s — the file is likely complete. Preserving it for retry.",
|
||||
save_path,
|
||||
)
|
||||
if transfer_backend == "aria2":
|
||||
self._reconcile_failed_aria2_partial(save_path)
|
||||
elif os.path.exists(save_path):
|
||||
try:
|
||||
os.remove(save_path)
|
||||
|
||||
Reference in New Issue
Block a user