alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-08-28 08:21:27 -03:00
Code Issues Packages Projects Releases Wiki Activity

feat(services): add per-destination rate-limit gate for API traffic (#1085)

Browse Source 
Implement Phase 1 of docs/plans/issue-1085-rate-limit-design.md:

- New RateLimitCoordinator: per-host shared Retry-After gate with
  exponential backoff (30s base, 1800s cap), minimum inter-request pacing
  (default 0.75s), herd-free waiter serialization via per-destination
  locks, and a bounded wait (default 300s) that raises instead of parking.
- Downloader.make_request: connectivity-guard fail-fast first, then gate
  pacing; on 429 register the cooldown and wait-and-resend (bounded);
  errors that passed through the gate are marked gate_handled.
- FallbackMetadataProvider / MetadataSyncService: a network provider 429
  no longer fails over to other network providers (stops the CivArchive
  flood); sqlite stays as local last resort. Rate-limited lookups now
  report "Rate limited" instead of "Model not found", so transient 429s
  no longer mark models civitai_deleted.
- _RateLimitRetryHelper skips its own sleep for gate_handled errors,
  removing the double wait.
- New settings: rate_limit_gate_enabled, rate_limit_max_wait_seconds,
  rate_limit_min_interval_seconds.
This commit is contained in:
Will Miao
2026-08-27 09:53:07 +08:00
parent 1e1921cabb
commit c2a2048c8b
9 changed files with 909 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/services/downloader.py
+97 -57
View File
@@ -32,6 +32,7 @@ from .connectivity_guard import (
ConnectivityGuard,
)
from .errors import RateLimitError
from .rate_limit_coordinator import RateLimitCoordinator
logger = logging.getLogger(__name__)
@@ -1074,74 +1075,113 @@ class Downloader:
Returns:
Tuple[bool, Union[Dict, str]]: (success, response data or error message)
When the rate-limit gate is enabled (``rate_limit_gate_enabled``),
requests are paced per destination and 429 responses are honored by
waiting out the ``Retry-After`` window (bounded by
``rate_limit_max_wait_seconds``) before re-sending. A ``RateLimitError``
returned after gate involvement is marked with ``gate_handled = True``
so downstream retry helpers do not wait a second time.
"""
guard = await ConnectivityGuard.get_instance()
destination = self._guard_destination(url)
# Fail fast on transport-level outages before pacing: there is no
# point waiting out a vendor cooldown while the network is down.
if guard.should_block_request(destination):
return False, OFFLINE_COOLDOWN_ERROR
try:
session = await self.session
# Debug log for proxy mode at request time
if self.proxy_url:
logger.debug(f"[make_request] Using app-level proxy: {self.proxy_url}")
else:
logger.debug(
"[make_request] Using system-level proxy (trust_env) if configured."
)
coordinator = await RateLimitCoordinator.get_instance()
gate_enabled = coordinator.enabled
# Safety bound on the wait-and-resend loop; each 429 normally exits
# via the wait cap in wait_for_slot, this covers pathological 429s
# with tiny Retry-After values.
max_resend_attempts = 5
attempt = 0
# Prepare headers
headers = self._get_auth_headers(use_auth)
if custom_headers:
headers.update(custom_headers)
while True:
if gate_enabled:
try:
await coordinator.wait_for_slot(destination)
except RateLimitError as exc:
exc.gate_handled = True
return False, exc
# Add proxy to kwargs if not already present
if "proxy" not in kwargs:
kwargs["proxy"] = self.proxy_url
async with session.request(
method, url, headers=headers, **kwargs
) as response:
if response.status == 200:
guard.register_success(destination)
# Try to parse as JSON, fall back to text
try:
data = await response.json()
return True, data
except:
text = await response.text()
return True, text
elif response.status == 401:
return False, "Unauthorized access - invalid or missing API key"
elif response.status == 403:
return False, "Access forbidden"
elif response.status == 404:
return False, "Resource not found"
elif response.status == 429:
retry_after = self._extract_retry_after(response.headers)
error_msg = "Request rate limited"
logger.warning(
"Rate limit encountered for %s %s; retry_after=%s",
method,
url,
retry_after,
)
return False, RateLimitError(
error_msg,
retry_after=retry_after,
)
try:
session = await self.session
# Debug log for proxy mode at request time
if self.proxy_url:
logger.debug(f"[make_request] Using app-level proxy: {self.proxy_url}")
else:
return False, f"Request failed with status {response.status}"
logger.debug(
"[make_request] Using system-level proxy (trust_env) if configured."
)
except Exception as e:
if guard.is_network_unreachable_error(e):
guard.register_network_failure(e, destination)
if guard.should_block_request(destination):
return False, OFFLINE_COOLDOWN_ERROR
logger.debug("Network unavailable for %s %s: %s", method, url, e)
# Prepare headers
headers = self._get_auth_headers(use_auth)
if custom_headers:
headers.update(custom_headers)
# Add proxy to kwargs if not already present
if "proxy" not in kwargs:
kwargs["proxy"] = self.proxy_url
async with session.request(
method, url, headers=headers, **kwargs
) as response:
if response.status == 200:
guard.register_success(destination)
if gate_enabled:
coordinator.register_success(destination)
# Try to parse as JSON, fall back to text
try:
data = await response.json()
return True, data
except:
text = await response.text()
return True, text
elif response.status == 401:
return False, "Unauthorized access - invalid or missing API key"
elif response.status == 403:
return False, "Access forbidden"
elif response.status == 404:
return False, "Resource not found"
elif response.status == 429:
retry_after = self._extract_retry_after(response.headers)
error_msg = "Request rate limited"
if not gate_enabled:
logger.warning(
"Rate limit encountered for %s %s; retry_after=%s",
method,
url,
retry_after,
)
return False, RateLimitError(
error_msg,
retry_after=retry_after,
)
# The coordinator logs the cooldown notice (INFO once
# per window, DEBUG on extension).
coordinator.register_rate_limit(destination, retry_after)
attempt += 1
if attempt >= max_resend_attempts:
error = RateLimitError(error_msg, retry_after=retry_after)
error.gate_handled = True
return False, error
# Loop back: wait_for_slot blocks until the cooldown
# elapses (or raises once the wait exceeds the cap).
continue
else:
return False, f"Request failed with status {response.status}"
except Exception as e:
if guard.is_network_unreachable_error(e):
guard.register_network_failure(e, destination)
if guard.should_block_request(destination):
return False, OFFLINE_COOLDOWN_ERROR
logger.debug("Network unavailable for %s %s: %s", method, url, e)
return False, str(e)
logger.error(f"Error making {method} request to {url}: {e}")
return False, str(e)
logger.error(f"Error making {method} request to {url}: {e}")
return False, str(e)
async def close(self):
"""Close the HTTP session"""
py/services/metadata_sync_service.py
+8 -1
View File
@@ -245,16 +245,23 @@ class MetadataSyncService:
civitai_api_not_found = False
any_rate_limited = False
skip_network_providers = False
for provider_name, provider in provider_attempts:
if skip_network_providers and provider_name != "sqlite":
# A network provider was already rate-limited; failing
# over to another network provider just spreads the flood
# (#1085). The local sqlite archive stays as last resort.
continue
try:
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
except RateLimitError as exc:
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
provider_name or provider.__class__.__name__,
exc.retry_after or 0,
)
any_rate_limited = True
skip_network_providers = True
continue
except Exception as exc: # pragma: no cover - defensive logging
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
py/services/model_metadata_provider.py
+58 -7
View File
@@ -66,6 +66,14 @@ class _RateLimitRetryHelper:
except RateLimitError as exc:
attempt += 1
# The downloader's rate-limit gate already applied the wait
# policy for this request (waited out the vendor window or
# deliberately refused because it exceeds the cap). Sleeping
# again here would double the wait — just propagate.
if getattr(exc, "gate_handled", False):
exc.provider = exc.provider or label
raise
# Determine effective retry limit based on rate-limit magnitude
effective_retry_limit = self._retry_limit # default: 3
if exc.retry_after is not None and exc.retry_after >= 120.0:
@@ -101,6 +109,12 @@ class _RateLimitRetryHelper:
return min(self._max_delay, max(0.0, base_delay))
# Labels of providers that are free to consult even while a network provider
# is rate-limited (local lookups, no vendor cost).
_LOCAL_PROVIDER_LABELS = frozenset({"sqlite"})
class ModelMetadataProvider(ABC):
"""Base abstract class for all model metadata providers"""
@@ -451,7 +465,14 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
return None
class FallbackMetadataProvider(ModelMetadataProvider):
"""Try providers in order, return first successful result."""
"""Try providers in order, return first successful result.
Rate-limit policy (#1085): once a *network* provider raises
``RateLimitError``, the chain stops consulting further network providers —
failing over would just spread the flood to the next vendor. Local-only
providers (see ``_LOCAL_PROVIDER_LABELS``) are still allowed as a last
resort because they cost the vendor nothing.
"""
def __init__(
self,
@@ -486,7 +507,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
)
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result, error = await self._call_with_rate_limit(
label,
@@ -496,8 +520,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
if result:
return result, error
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
@@ -505,11 +530,18 @@ class FallbackMetadataProvider(ModelMetadataProvider):
except Exception as e:
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
continue
if rate_limited:
# Distinct from "Model not found": callers must not mistake a
# rate-limited lookup for a confirmed deletion.
return None, "Rate limited"
return None, "Model not found"
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
not_found_confirmed = False
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result = await self._call_with_rate_limit(
label,
@@ -519,8 +551,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
if result:
return result
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
@@ -539,7 +572,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
return None
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result = await self._call_with_rate_limit(
label,
@@ -550,8 +586,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
if result:
return result
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
@@ -562,7 +599,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
return None
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result, error = await self._call_with_rate_limit(
label,
@@ -572,8 +612,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
if result:
return result, error
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
@@ -581,12 +622,17 @@ class FallbackMetadataProvider(ModelMetadataProvider):
except Exception as e:
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
continue
if rate_limited:
return None, "Rate limited"
return None, "No provider could retrieve the data"
async def get_model_versions_by_hashes(
self, hashes: List[str]
) -> Optional[List[Dict[str, Any]]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result = await self._call_with_rate_limit(
label,
@@ -598,8 +644,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
except NotImplementedError:
continue
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
@@ -614,7 +661,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
return None
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result = await self._call_with_rate_limit(
label,
@@ -625,8 +675,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
if result is not None:
return result
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
py/services/rate_limit_coordinator.py
+213
View File
@@ -0,0 +1,213 @@
"""Process-wide, per-destination rate-limit gate for outbound API traffic.
Implements the pacing/gating layer designed in
``docs/plans/issue-1085-rate-limit-design.md``:
- **Reactive gate**: a 429 response arms ``next_allowed_send`` from the
vendor's ``Retry-After`` (or exponential backoff when the header is
missing); subsequent requests to the same destination wait out the window.
- **Preemptive pacing**: a minimum inter-request interval per destination
spaces consecutive sends so bursts never form in the first place.
- **Herd-free**: waiters are serialized through a per-destination lock, so
each one claims a distinct send slot instead of thousands of coroutines
waking up together.
- **Bounded**: waits longer than ``rate_limit_max_wait_seconds`` are refused
by raising :class:`RateLimitError`, leaving the final decision to callers.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from typing import Dict, Optional
from .errors import RateLimitError
logger = logging.getLogger(__name__)
DEFAULT_MIN_INTERVAL_SECONDS = 0.75
DEFAULT_MAX_WAIT_SECONDS = 300.0
BASE_BACKOFF_SECONDS = 30.0
MAX_BACKOFF_SECONDS = 1800.0
@dataclass
class _DestinationState:
"""Rate-limit bookkeeping for one destination (hostname)."""
next_allowed_send: float = 0.0 # time.monotonic() timestamp
consecutive_429: int = 0
last_send_at: float = 0.0 # time.monotonic() timestamp
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
class RateLimitCoordinator:
"""Coordinates outbound request pacing per destination.
Singleton mirroring :class:`ConnectivityGuard`'s pattern. All waits are
bounded by the ``rate_limit_max_wait_seconds`` setting; when the required
wait exceeds the cap, :meth:`wait_for_slot` raises :class:`RateLimitError`
instead of parking the caller.
"""
_instance: "RateLimitCoordinator | None" = None
_instance_lock = asyncio.Lock()
@classmethod
async def get_instance(cls) -> "RateLimitCoordinator":
async with cls._instance_lock:
if cls._instance is None:
cls._instance = cls()
return cls._instance
def __init__(self) -> None:
if hasattr(self, "_initialized"):
return
self._initialized = True
self._states: Dict[str, _DestinationState] = {}
# ------------------------------------------------------------------
# Settings (read live so settings edits apply without a restart)
@staticmethod
def _setting(key: str, default):
try:
from .settings_manager import get_settings_manager
return get_settings_manager().get(key, default)
except Exception: # pragma: no cover - defensive: settings unavailable
return default
@property
def enabled(self) -> bool:
return bool(self._setting("rate_limit_gate_enabled", True))
@property
def min_interval_seconds(self) -> float:
try:
return max(0.0, float(self._setting("rate_limit_min_interval_seconds", DEFAULT_MIN_INTERVAL_SECONDS)))
except (TypeError, ValueError):
return DEFAULT_MIN_INTERVAL_SECONDS
@property
def max_wait_seconds(self) -> float:
try:
return max(0.0, float(self._setting("rate_limit_max_wait_seconds", DEFAULT_MAX_WAIT_SECONDS)))
except (TypeError, ValueError):
return DEFAULT_MAX_WAIT_SECONDS
# ------------------------------------------------------------------
# State helpers
@staticmethod
def _normalize(destination: Optional[str]) -> str:
if destination is None or not destination.strip():
return "__global__"
return destination.lower().strip()
def _state_for(self, destination: Optional[str]) -> _DestinationState:
key = self._normalize(destination)
if key not in self._states:
self._states[key] = _DestinationState()
return self._states[key]
def reset(self) -> None:
"""Drop all per-destination state. Test seam."""
self._states.clear()
def in_cooldown(self, destination: Optional[str] = None) -> bool:
return self.remaining_seconds(destination) > 0
def remaining_seconds(self, destination: Optional[str] = None) -> float:
state = self._state_for(destination)
return max(0.0, state.next_allowed_send - time.monotonic())
# ------------------------------------------------------------------
# Gate operations
async def wait_for_slot(self, destination: Optional[str] = None) -> None:
"""Block until this caller may send the next request to *destination*.
Waits for both the rate-limit cooldown (``next_allowed_send``) and the
minimum inter-request interval (``last_send_at + min_interval``).
Waiters queue on the per-destination lock, so concurrent callers are
spaced out instead of stampeding when a cooldown expires.
Raises:
RateLimitError: when the required wait exceeds
``rate_limit_max_wait_seconds``.
"""
state = self._state_for(destination)
deadline = time.monotonic() + self.max_wait_seconds
async with state.lock:
now = time.monotonic()
wake_at = max(
state.next_allowed_send,
state.last_send_at + self.min_interval_seconds,
)
if wake_at > deadline:
raise RateLimitError(
f"Rate limit wait for '{self._normalize(destination)}' "
f"exceeds the {self.max_wait_seconds:.0f}s cap",
retry_after=wake_at - now,
)
delay = wake_at - now
if delay > 0:
logger.debug(
"Rate-limit gate: pacing request to '%s' by %.2fs",
self._normalize(destination),
delay,
)
await asyncio.sleep(delay)
state.last_send_at = time.monotonic()
def register_rate_limit(
self,
destination: Optional[str],
retry_after: Optional[float] = None,
) -> float:
"""Record a 429 for *destination* and arm the cooldown window.
Honors the vendor's ``Retry-After`` when present; otherwise grows an
exponential backoff (30s base, doubling per consecutive 429, capped at
1800s). Returns the cooldown duration in seconds.
"""
state = self._state_for(destination)
state.consecutive_429 += 1
if retry_after is not None and retry_after > 0:
backoff = min(MAX_BACKOFF_SECONDS, float(retry_after))
else:
backoff = min(
MAX_BACKOFF_SECONDS,
BASE_BACKOFF_SECONDS * (2 ** (state.consecutive_429 - 1)),
)
now = time.monotonic()
already_cooling = state.next_allowed_send > now
state.next_allowed_send = max(state.next_allowed_send, now + backoff)
if already_cooling:
logger.debug(
"Rate-limit cooldown for '%s' extended by %.0fs (consecutive_429=%d)",
self._normalize(destination),
backoff,
state.consecutive_429,
)
else:
logger.info(
"Rate limited by '%s'; pausing requests for %.0fs",
self._normalize(destination),
backoff,
)
return backoff
def register_success(self, destination: Optional[str]) -> None:
"""Reset rate-limit state after a successful request.
A 200 proves the vendor is accepting traffic again, so any armed
cooldown window is cleared alongside the backoff counter (mirrors
``ConnectivityGuard.register_success`` semantics).
"""
state = self._state_for(destination)
state.consecutive_429 = 0
state.next_allowed_send = 0.0
py/services/settings_manager.py
+3
View File
@@ -70,6 +70,9 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"enable_metadata_archive_db": False,
"enable_civarchive_api": True,
"metadata_provider_order": "civitai_archive_sqlite",
"rate_limit_gate_enabled": True,
"rate_limit_max_wait_seconds": 300,
"rate_limit_min_interval_seconds": 0.75,
"proxy_enabled": False,
"proxy_host": "",
"proxy_port": "",