feat(recipes): skip rate-limited batch-import items and register download 429s (#1085)

Phase 2 of docs/plans/issue-1085-rate-limit-design.md:

- Batch import: items that fail due to vendor rate limiting are now
  SKIPPED with a "re-run the import later" hint instead of FAILED, so a
  transient 429 no longer pollutes failure accounting; the progress
  broadcast carries a rate_limited flag.
- Batch import UI: show a one-time "rate limited — slowing down" toast
  and swap the running status text while rate_limited; i18n keys synced
  to all locales.
- Downloader: download_file / download_to_memory / get_response_headers
  register 429 cooldowns with the RateLimitCoordinator, so subsequent
  API calls queue behind a download-triggered rate-limit window.
This commit is contained in:
Will Miao
2026-08-27 10:08:32 +08:00
parent c2a2048c8b
commit df34efafbc
16 changed files with 273 additions and 9 deletions
+28
View File
@@ -596,6 +596,21 @@ class Downloader:
False,
"File not found - the download link may be invalid or expired.",
)
elif response.status == 429:
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085). The download
# itself fails as before; retry policy stays with the
# caller (download manager).
retry_after = self._extract_retry_after(response.headers)
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(
self._guard_destination(url), retry_after
)
logger.warning(
f"Rate limited (429) for {url}, retry_after={retry_after}"
)
return False, f"Download rate limited (429), retry after {retry_after}s"
else:
logger.error(
f"Download failed for {url} with status {response.status}"
@@ -973,6 +988,11 @@ class Downloader:
elif response.status == 429:
raw_retry_after = response.headers.get("Retry-After")
retry_after = _parse_retry_after(raw_retry_after or "")
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085).
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(destination, retry_after)
if raw_retry_after:
logger.warning(
"Rate limited (429) for %s, Retry-After: %ss", url, retry_after
@@ -1042,6 +1062,14 @@ class Downloader:
if response.status == 200:
guard.register_success(destination)
return True, dict(response.headers)
elif response.status == 429:
# Register the vendor's cooldown so API calls through
# make_request queue behind it (#1085).
retry_after = self._extract_retry_after(response.headers)
coordinator = await RateLimitCoordinator.get_instance()
if coordinator.enabled:
coordinator.register_rate_limit(destination, retry_after)
return False, f"Head request rate limited (429), retry after {retry_after}s"
else:
return False, f"Head request failed with status {response.status}"