mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-06-17 16:09:25 -03:00
fix(rate-limit): continue to next provider on CivArchive 429 to prevent bulk refresh from freezing (#983)
When CivArchive returns HTTP 429 with a large retry_after, the bulk metadata refresh would block for hours because: 1. FallbackMetadataProvider raised RateLimitError instead of continuing to the next provider (e.g., SQLite archive was never reached). 2. _RateLimitRetryHelper retried long-rate-limit 429s 3 times — all futile since the hourly cap hasn't reset. 3. The batch loop had no awareness of persistent rate-limiting, causing 192+ models to each hammer the same rate-limited endpoint. Changes: - FallbackMetadataProvider: all 6 methods now continue to next provider on RateLimitError instead of raising (model_metadata_provider.py) - fetch_and_update_model: deleted-model path also continues on RateLimitError so sqlite provider gets a chance (metadata_sync_service.py) - _RateLimitRetryHelper: when retry_after >= 120s, only 1 attempt is made — retries are futile for hour-scale rate limits - BulkMetadataRefreshUseCase: tracks consecutive rate-limit failures and aborts early after 3 (bulk_metadata_refresh_use_case.py) Tests: updated test_fallback_respects_retry_limit for new continue behavior; added tests for large/small retry_after thresholds.
This commit is contained in:
@@ -77,6 +77,9 @@ class BulkMetadataRefreshUseCase:
|
||||
|
||||
await emit("started")
|
||||
|
||||
RATE_LIMIT_ABORT_THRESHOLD = 3
|
||||
consecutive_rate_limits = 0
|
||||
|
||||
for model in to_process:
|
||||
if self._service.scanner.is_cancelled():
|
||||
self._logger.info("Bulk metadata refresh cancelled by user")
|
||||
@@ -115,12 +118,39 @@ class BulkMetadataRefreshUseCase:
|
||||
continue
|
||||
|
||||
await MetadataManager.hydrate_model_data(model)
|
||||
result, _ = await self._metadata_sync.fetch_and_update_model(
|
||||
result, error_msg = await self._metadata_sync.fetch_and_update_model(
|
||||
sha256=model["sha256"],
|
||||
file_path=model["file_path"],
|
||||
model_data=model,
|
||||
update_cache_func=self._service.scanner.update_single_model_cache,
|
||||
)
|
||||
|
||||
if not result and error_msg and "Rate limited" in error_msg:
|
||||
consecutive_rate_limits += 1
|
||||
else:
|
||||
consecutive_rate_limits = 0
|
||||
|
||||
if consecutive_rate_limits >= RATE_LIMIT_ABORT_THRESHOLD:
|
||||
self._logger.warning(
|
||||
"Bulk metadata refresh aborted: %d consecutive rate limits detected. "
|
||||
"Processed %d/%d models.",
|
||||
consecutive_rate_limits,
|
||||
processed,
|
||||
total_to_process,
|
||||
)
|
||||
await emit(
|
||||
"rate_limited",
|
||||
processed=processed,
|
||||
success=success,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"Rate limit detected; {total_to_process - processed} models skipped",
|
||||
"processed": processed,
|
||||
"updated": success,
|
||||
"total": total_models,
|
||||
}
|
||||
|
||||
if result:
|
||||
success += 1
|
||||
if original_name != model.get("model_name"):
|
||||
|
||||
Reference in New Issue
Block a user