feat(services): add per-destination rate-limit gate for API traffic (#1085)

Implement Phase 1 of docs/plans/issue-1085-rate-limit-design.md:

- New RateLimitCoordinator: per-host shared Retry-After gate with
  exponential backoff (30s base, 1800s cap), minimum inter-request pacing
  (default 0.75s), herd-free waiter serialization via per-destination
  locks, and a bounded wait (default 300s) that raises instead of parking.
- Downloader.make_request: connectivity-guard fail-fast first, then gate
  pacing; on 429 register the cooldown and wait-and-resend (bounded);
  errors that passed through the gate are marked gate_handled.
- FallbackMetadataProvider / MetadataSyncService: a network provider 429
  no longer fails over to other network providers (stops the CivArchive
  flood); sqlite stays as local last resort. Rate-limited lookups now
  report "Rate limited" instead of "Model not found", so transient 429s
  no longer mark models civitai_deleted.
- _RateLimitRetryHelper skips its own sleep for gate_handled errors,
  removing the double wait.
- New settings: rate_limit_gate_enabled, rate_limit_max_wait_seconds,
  rate_limit_min_interval_seconds.
This commit is contained in:
Will Miao
2026-08-27 09:53:07 +08:00
parent 1e1921cabb
commit c2a2048c8b
9 changed files with 909 additions and 104 deletions
@@ -812,3 +812,56 @@ async def test_fetch_and_update_model_does_not_overwrite_api_metadata_with_archi
helpers.metadata_manager.save_metadata.assert_awaited()
update_cache.assert_awaited()
@pytest.mark.asyncio
async def test_fetch_and_update_model_keeps_sqlite_last_resort_after_civarchive_rate_limit(tmp_path):
"""A CivArchive 429 must not block the local sqlite last resort (#1085)."""
civarchive_provider = SimpleNamespace(
get_model_by_hash=AsyncMock(
side_effect=RateLimitError("limited", retry_after=30)
),
get_model_version=AsyncMock(),
)
sqlite_payload = {
"source": "archive_db",
"model": {"name": "Recovered", "description": "", "tags": []},
"images": [],
"baseModel": "sdxl",
}
sqlite_provider = SimpleNamespace(
get_model_by_hash=AsyncMock(return_value=(sqlite_payload, None)),
get_model_version=AsyncMock(),
)
async def select_provider(name: str):
if name == "civarchive_api":
return civarchive_provider
if name == "sqlite":
return sqlite_provider
raise AssertionError(f"unexpected provider request: {name}")
helpers = build_service(
settings_values={"enable_metadata_archive_db": True},
provider_selector=AsyncMock(side_effect=select_provider),
)
model_path = tmp_path / "model.safetensors"
model_data = {
"civitai_deleted": True,
"db_checked": False,
"file_path": str(model_path),
}
update_cache = AsyncMock()
ok, error = await helpers.service.fetch_and_update_model(
sha256="cafe",
file_path=str(model_path),
model_data=model_data,
update_cache_func=update_cache,
)
assert ok and error is None
civarchive_provider.get_model_by_hash.assert_awaited_once()
sqlite_provider.get_model_by_hash.assert_awaited_once()
assert model_data["metadata_source"] == "archive_db"