feat(fetch): skip CivArchive API for HuggingFace-sourced models

- Bulk refresh filter now excludes models with hf_url
- Individual refresh for HF models only checks CivitAI API
- CivArchive client validates model IDs before querying
This commit is contained in:
Will Miao
2026-07-11 20:29:54 +08:00
parent f362ed585b
commit 4f016a8024
3 changed files with 33 additions and 1 deletions

View File

@@ -304,6 +304,20 @@ class CivArchiveClient:
version_id = file_data.get("model_version_id") or file_data.get("modelVersionId")
if model_id is None or version_id is None:
continue
# CivitAI / CivArchive model IDs are small integers (typically ≤ 7
# digits). Reject suspiciously large values that indicate the API
# returned a malformed payload (e.g. a hash reinterpreted as an ID)
# to avoid pointless HTTP 500 errors from CivArchive.
_MAX_VALID_CIVITAI_ID = 100_000_000
try:
if int(model_id) >= _MAX_VALID_CIVITAI_ID or int(version_id) >= _MAX_VALID_CIVITAI_ID:
logger.debug(
"Skipping implausible CivArchive model_id=%s / version_id=%s",
model_id, version_id,
)
continue
except (TypeError, ValueError):
continue
resolved = await self.get_model_version(model_id, version_id)
if resolved:
return resolved

View File

@@ -209,7 +209,21 @@ class MetadataSyncService:
error_msg = "CivitAI model is deleted and no archive provider is available"
return False, error_msg
else:
provider_attempts.append((None, await self._get_default_provider()))
is_hf_source = bool(model_data.get("hf_url"))
if is_hf_source:
# HF-sourced model: only check CivitAI API directly.
# CivArchive is almost guaranteed to have no record, and
# hitting it wastes rate-limit budget.
# Use a distinct provider name ("civitai_api" not None) so
# downstream code does NOT interpret a "Model not found"
# response as civitai_api_not_found — which would mark the
# model civitai_deleted=True when it was never on CivitAI.
try:
provider_attempts.append(("civitai_api", await self._get_provider("civitai_api")))
except Exception as exc: # pragma: no cover - provider resolution fault
logger.debug("Unable to resolve civitai_api provider: %s", exc)
if not provider_attempts:
provider_attempts.append((None, await self._get_default_provider()))
civitai_metadata: Optional[Dict[str, Any]] = None
metadata_provider: Optional[MetadataProviderProtocol] = None

View File

@@ -51,6 +51,10 @@ class BulkMetadataRefreshUseCase:
if not model.get("skip_metadata_refresh", False)
and not self._is_in_skip_path(model.get("folder", ""), skip_paths)
and (not model.get("civitai") or not model["civitai"].get("id"))
# Skip models downloaded from Hugging Face — they are not on
# CivitAI / CivArchive. Users can still refresh them individually
# via the right-click context menu.
and not model.get("hf_url", "")
and not (
# Skip models confirmed not on CivitAI when no need to retry
model.get("from_civitai") is False