fix(recipes): enforce batch-import concurrency bound and harden ingest errors (#1085)

Address the rate-limit flood and secondary errors seen during large
recipe ingestion (example-images directory import):

- batch import: share one adaptive-concurrency semaphore across the whole
  batch (previously each item got a fresh semaphore, so the min/max
  concurrency bounds never applied and every item ran concurrently);
  synchronize the shared semaphore capacity after each completed item.
- comfy parser: guard ckpt_name against list/None values so re.search no
  longer raises TypeError and fails the whole image import.
- civarchive client: normalize empty-string failure payloads to
  "Request failed" and treat a missing payload as an error, fixing the
  "'NoneType' object has no attribute 'get'" crash.
- civarchive client: log connectivity-guard offline-cooldown
  short-circuits at DEBUG instead of one ERROR per request.
This commit is contained in:
Will Miao
2026-08-27 07:58:48 +08:00
parent 574dfbbe55
commit ee233548e5
6 changed files with 285 additions and 26 deletions
@@ -277,3 +277,72 @@ async def test_parse_metadata_without_extra_metadata(monkeypatch):
assert "error" not in result
assert result["loras"] == []
assert result["checkpoint"]["id"] == "456"
@pytest.mark.asyncio
async def test_parse_metadata_with_list_ckpt_name(monkeypatch):
"""ckpt_name serialized as a single-element list must not crash re.search."""
checkpoint_info = {
"id": 456,
"modelId": 123,
"model": {"name": "Checkpoint", "type": "checkpoint"},
"name": "v1",
"baseModel": "SDXL 1.0",
}
async def fake_metadata_provider():
class Provider:
async def get_model_version_info(self, version_id):
assert version_id == "456"
return checkpoint_info, None
return Provider()
monkeypatch.setattr(
"py.recipes.parsers.comfy.get_default_metadata_provider",
fake_metadata_provider,
)
metadata_json = {
"1": {
"class_type": "CheckpointLoaderSimple",
"inputs": {"ckpt_name": ["urn:air:sdxl:checkpoint:civitai:123@456"]},
}
}
result = await ComfyMetadataParser().parse_metadata(json.dumps(metadata_json))
assert "error" not in result
assert result["checkpoint"] is not None
assert int(result["checkpoint"]["id"]) == 456
assert int(result["checkpoint"]["modelId"]) == 123
@pytest.mark.asyncio
async def test_parse_metadata_with_none_ckpt_name(monkeypatch):
"""Missing (None) ckpt_name must not crash re.search with a TypeError."""
async def fake_metadata_provider():
class Provider:
async def get_model_version_info(self, version_id):
raise AssertionError("Checkpoint lookup must be skipped")
return Provider()
monkeypatch.setattr(
"py.recipes.parsers.comfy.get_default_metadata_provider",
fake_metadata_provider,
)
metadata_json = {
"1": {
"class_type": "CheckpointLoaderSimple",
"inputs": {"ckpt_name": None},
}
}
result = await ComfyMetadataParser().parse_metadata(json.dumps(metadata_json))
assert "error" not in result
assert result["checkpoint"] is None
assert result["loras"] == []