alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-08-28 08:21:27 -03:00
Code Issues Packages Projects Releases Wiki Activity

feat(recipes): skip rate-limited batch-import items and register download 429s (#1085)

Browse Source 
Phase 2 of docs/plans/issue-1085-rate-limit-design.md:

- Batch import: items that fail due to vendor rate limiting are now
  SKIPPED with a "re-run the import later" hint instead of FAILED, so a
  transient 429 no longer pollutes failure accounting; the progress
  broadcast carries a rate_limited flag.
- Batch import UI: show a one-time "rate limited — slowing down" toast
  and swap the running status text while rate_limited; i18n keys synced
  to all locales.
- Downloader: download_file / download_to_memory / get_response_headers
  register 429 cooldowns with the RateLimitCoordinator, so subsequent
  API calls queue behind a download-triggered rate-limit window.
This commit is contained in:
Will Miao
2026-08-27 10:08:32 +08:00
parent c2a2048c8b
commit df34efafbc
16 changed files with 273 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/services/batch_import_service.py
+34 -3
View File
@@ -71,6 +71,9 @@ class BatchImportProgress:
tags: List[str] = field(default_factory=list)
skip_no_metadata: bool = False
skip_duplicates: bool = False
# Set once any item is skipped due to vendor rate limiting (#1085); lets
# the UI surface a "slowing down / try again later" hint.
rate_limited: bool = False
def to_dict(self) -> Dict[str, Any]:
return {
@@ -82,6 +85,7 @@ class BatchImportProgress:
"skipped": self.skipped,
"current_item": self.current_item,
"status": self.status,
"rate_limited": self.rate_limited,
"started_at": self.started_at,
"finished_at": self.finished_at,
"progress_percent": round((self.completed / self.total) * 100, 1)
@@ -383,6 +387,13 @@ class BatchImportService:
ext = os.path.splitext(filename)[1].lower()
return ext in self.SUPPORTED_EXTENSIONS
@staticmethod
def _is_rate_limit_error(error: Optional[str]) -> bool:
"""Return True when an error payload represents vendor rate limiting."""
if not error:
return False
return "rate limit" in error.lower()
async def _run_batch_import(
self,
*,
@@ -441,6 +452,17 @@ class BatchImportService:
item.status = ImportStatus.SKIPPED
item.error_message = result.get("error")
progress.skipped += 1
elif self._is_rate_limit_error(result.get("error")):
# Vendor rate limit is a transient, external condition —
# do not pollute the failure count with it (#1085). The
# import can simply be re-run later.
item.status = ImportStatus.SKIPPED
item.error_message = (
f"Rate limited by metadata provider; "
f"re-run the import later ({result.get('error')})"
)
progress.skipped += 1
progress.rate_limited = True
else:
item.status = ImportStatus.FAILED
item.error_message = result.get("error")
@@ -448,10 +470,19 @@ class BatchImportService:
except Exception as e:
self._logger.error(f"Error importing {item.source}: {e}")
item.status = ImportStatus.FAILED
item.error_message = str(e)
item.duration = time.time() - start_time
progress.failed += 1
if self._is_rate_limit_error(str(e)):
item.status = ImportStatus.SKIPPED
item.error_message = (
f"Rate limited by metadata provider; "
f"re-run the import later ({e})"
)
progress.skipped += 1
progress.rate_limited = True
else:
item.status = ImportStatus.FAILED
item.error_message = str(e)
progress.failed += 1
self._concurrency_controller.record_result(item.duration, False)
await self._concurrency_controller.apply_concurrency()
py/services/downloader.py
+28
View File
@@ -596,6 +596,21 @@ class Downloader:
False,
"File not found - the download link may be invalid or expired.",
)
elif response.status == 429:
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085). The download
# itself fails as before; retry policy stays with the
# caller (download manager).
retry_after = self._extract_retry_after(response.headers)
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(
self._guard_destination(url), retry_after
)
logger.warning(
f"Rate limited (429) for {url}, retry_after={retry_after}"
)
return False, f"Download rate limited (429), retry after {retry_after}s"
else:
logger.error(
f"Download failed for {url} with status {response.status}"
@@ -973,6 +988,11 @@ class Downloader:
elif response.status == 429:
raw_retry_after = response.headers.get("Retry-After")
retry_after = _parse_retry_after(raw_retry_after or "")
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085).
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(destination, retry_after)
if raw_retry_after:
logger.warning(
"Rate limited (429) for %s, Retry-After: %ss", url, retry_after
@@ -1042,6 +1062,14 @@ class Downloader:
if response.status == 200:
guard.register_success(destination)
return True, dict(response.headers)
elif response.status == 429:
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085).
retry_after = self._extract_retry_after(response.headers)
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(destination, retry_after)
return False, f"Head request rate limited (429), retry after {retry_after}s"
else:
return False, f"Head request failed with status {response.status}"