fix(recipes): enforce batch-import concurrency bound and harden ingest errors (#1085)

Address the rate-limit flood and secondary errors seen during large
recipe ingestion (example-images directory import):

- batch import: share one adaptive-concurrency semaphore across the whole
  batch (previously each item got a fresh semaphore, so the min/max
  concurrency bounds never applied and every item ran concurrently);
  synchronize the shared semaphore capacity after each completed item.
- comfy parser: guard ckpt_name against list/None values so re.search no
  longer raises TypeError and fails the whole image import.
- civarchive client: normalize empty-string failure payloads to
  "Request failed" and treat a missing payload as an error, fixing the
  "'NoneType' object has no attribute 'get'" crash.
- civarchive client: log connectivity-guard offline-cooldown
  short-circuits at DEBUG instead of one ERROR per request.
This commit is contained in:
Will Miao
2026-08-27 07:58:48 +08:00
parent 574dfbbe55
commit ee233548e5
6 changed files with 285 additions and 26 deletions
+47
View File
@@ -1,4 +1,5 @@
import copy
import logging
from typing import Any, Dict
from unittest.mock import AsyncMock
@@ -257,3 +258,49 @@ async def test_get_model_by_hash_propagates_rate_limit(downloader):
assert exc_info.value.retry_after == 5
assert exc_info.value.provider == "civarchive_api"
async def test_get_model_by_hash_empty_error_payload(downloader):
"""An empty-string failure payload must surface as a proper error.
Regression test: (None, "") used to fall through the falsy-error check and
crash in _resolve_version_from_files with "'NoneType' object has no
attribute 'get'".
"""
async def fake_make_request(method, url, use_auth=False, **kwargs):
return False, ""
downloader.make_request = fake_make_request
client = await CivArchiveClient.get_instance()
result, error = await client.get_model_by_hash("empty-error")
assert result is None
assert error == "Request failed"
async def test_get_model_version_offline_cooldown_logged_as_debug(downloader, caplog):
"""Cooldown short-circuits must not spam an ERROR per request."""
async def fake_make_request(method, url, use_auth=False, **kwargs):
return False, "offline_cooldown"
downloader.make_request = fake_make_request
client = await CivArchiveClient.get_instance()
with caplog.at_level(logging.DEBUG, logger="py.services.civarchive_client"):
result = await client.get_model_version(model_id=1, version_id=123)
assert result is None
module_records = [r for r in caplog.records if r.name == "py.services.civarchive_client"]
assert not any(
r.levelno >= logging.ERROR and "Error fetching CivArchive model version" in r.getMessage()
for r in module_records
)
assert any(
r.levelno == logging.DEBUG and "while offline" in r.getMessage()
for r in module_records
)