alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-10-05 09:35:31 -03:00
Code Issues Packages Projects Releases Wiki Activity

fix(update): read model-page prices from a host that answers

Browse Source 
End-to-end verification against the live site found the price capture broken for
a whole class of users: the civitai page hosts are not interchangeable, and the
user's civitai_host preference was silently fatal. With civitai_host=civitai.red
the update DB held zero prices even with tracking enabled.

- civitai.red refuses non-browser HTTP clients outright (Cloudflare challenge,
  403 for any User-Agent, aiohttp and httpx alike), while civitai.com and
  civitai.green answer normally for anonymously visible models and 404 for
  mature ones. An earlier manual check with curl passed on TLS fingerprint luck,
  which is why this was missed.
- get_model_prices now tries the configured host first, then the others, and
  takes the first parseable payload. The host that worked is remembered, and a
  host that refuses outright is parked for 15 minutes so a library full of
  mature models does not pay three requests each; a 404 is model-specific and
  does not park the host. Links keep using the configured host, which is where
  the user's own browser has clearance.
- Mature models still have no price source anywhere, so that is now stated
  instead of silent: price_check_attempted_at separates "tried and unreadable"
  from "never looked", gated versions show a muted "Price unavailable" badge,
  and the alerts panel reports unavailableCount.
- Failures are logged at warning level, once per host per TTL, with the
  per-host reason, instead of only at debug level.
- The recorded alternatives (internal tRPC with the user's API key, or an
  extension-assisted fetch from the user's browser) and the strengthened
  upstream ask for a public price field are documented in the plan.
This commit is contained in:
Will Miao
2026-10-04 20:00:29 +08:00
parent 7ed19c185c
commit ec5fef512b
23 changed files with 653 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/services/civitai_client.py
+123 -31
View File
@@ -20,7 +20,11 @@ from .model_metadata_provider import (
)
from .downloader import get_downloader
from .errors import RateLimitError, ResourceNotFoundError
from ..utils.civitai_utils import build_civitai_model_page_url, resolve_license_payload
from ..utils.civitai_utils import (
build_civitai_model_page_url,
civitai_page_host_candidates,
resolve_license_payload,
)
from ..utils.civitai_page_prices import parse_model_page_prices
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
@@ -32,6 +36,24 @@ logger = logging.getLogger(__name__)
_CREATOR_COUNT_CACHE_TTL_SECONDS = 600
_creator_model_count_cache: Dict[str, Tuple[float, Optional[int]]] = {}
# How long a page host stays on the skip list after refusing a request outright
# (Cloudflare's challenge, surfaced as 403 "Access forbidden"). Long enough to
# cover a whole update refresh, short enough to recover within a session.
_PAGE_HOST_BLOCK_TTL = 15 * 60
def _is_host_level_refusal(message: str) -> bool:
"""Whether a failed request means "this host refuses us" rather than "this
model is unavailable".
``downloader.make_request`` collapses statuses into prose, and 403 ("Access
forbidden") is the one a Cloudflare challenge produces. A 404 ("Resource not
found") is model-specific — mature pages are hidden from anonymous visitors —
so it must not put the whole host on the skip list.
"""
return "forbidden" in message.lower()
class CivitaiClient:
_instance = None
@@ -67,6 +89,10 @@ class CivitaiClient:
str, Tuple[Optional[Dict[str, Any]], Optional[str]]
] = OrderedDict()
self._MAX_CACHE_ENTRIES = 500
# Model-page host bookkeeping: which host last worked, and which ones are
# currently refusing us (see get_model_prices).
self._page_host_preference: Optional[str] = None
self._page_host_blocked: Dict[str, float] = {}
def _build_image_info_url(self, image_id: str) -> str:
return f"{self.base_url}/images?imageId={image_id}&nsfw=X&withMeta=true"
@@ -396,6 +422,19 @@ class CivitaiClient:
empty dict when the page loads but lists no gated version, or None when
the page could not be read or understood — callers keep any stored price.
Several hosts are tried in order, because the hosts are not equivalent:
* ``civitai.red`` serves mature model pages that ``civitai.com`` hides from
anonymous visitors, but it is behind a Cloudflare challenge that refuses
non-browser clients outright (403 for any User-Agent).
* ``civitai.com`` / ``civitai.green`` answer normally for anonymously
visible models, and 404 for the mature ones.
So the user's ``civitai_host`` preference is a starting point, not the only
option. Mature models whose page cannot be read from any host stay
priceless — see the known limitation in
``docs/plans/paid-model-price-tracking.md``.
This is a public anonymous page fetch: no API key and no internal
endpoint, so a failure here must never fail the update check itself.
"""
@@ -405,39 +444,92 @@ class CivitaiClient:
except (TypeError, ValueError):
return None
url = build_civitai_model_page_url(normalized_id, host=self._page_host())
if not url:
return None
candidates = self._page_host_candidates()
failures: List[str] = []
try:
success, result = await self._make_request(
"GET",
url,
use_auth=False,
custom_headers={"Accept": "text/html"},
)
except RateLimitError:
# The shared rate-limit gate already recorded it; skip this model.
raise
except Exception as exc: # pragma: no cover - defensive
logger.debug("Failed to fetch model page for %s: %s", model_id, exc)
return None
for host in candidates:
url = build_civitai_model_page_url(normalized_id, host=host)
if not url:
continue
if not success or not isinstance(result, str):
logger.debug(
"No model page payload for %s (success=%s, type=%s)",
model_id,
success,
type(result).__name__,
)
return None
try:
success, result = await self._make_request(
"GET",
url,
use_auth=False,
custom_headers={"Accept": "text/html"},
)
except RateLimitError:
# The shared rate-limit gate already recorded it; skip this model.
raise
except Exception as exc: # pragma: no cover - defensive
failures.append(f"{host}: {exc}")
continue
prices = parse_model_page_prices(result)
if prices is None:
logger.debug(
"Model page for %s carried no usable price payload", model_id
)
return prices
if not success or not isinstance(result, str):
message = result if isinstance(result, str) else type(result).__name__
failures.append(f"{host}: {message}")
if isinstance(result, str) and _is_host_level_refusal(result):
# A refusal like Cloudflare's "Access forbidden" applies to the
# host, not to this model, so stop paying for it for a while.
self._block_page_host(host)
continue
prices = parse_model_page_prices(result)
if prices is None:
failures.append(f"{host}: no usable price payload")
continue
self._remember_page_host(host)
return prices
logger.warning(
"No price source for model %s; tried %s. Mature models are only served "
"by civitai.red, which challenges non-browser clients.",
model_id,
"; ".join(failures) or "no candidate hosts",
)
return None
def _page_host_candidates(self) -> List[str]:
"""Ordered hosts to try: the last one that worked, then the preference."""
preferred = self._page_host()
ordered = list(civitai_page_host_candidates(preferred))
if self._page_host_preference and self._page_host_preference in ordered:
ordered.remove(self._page_host_preference)
ordered.insert(0, self._page_host_preference)
now = time.time()
usable = [
host
for host in ordered
if now - self._page_host_blocked.get(host, 0.0) >= _PAGE_HOST_BLOCK_TTL
]
# Never return an empty list: a blocked host is still better than no attempt
# once the preference and the memo disagree.
return usable or ordered
def _remember_page_host(self, host: str) -> None:
if self._page_host_preference != host:
logger.info("CivitAI model pages are being read from %s", host)
self._page_host_preference = host
self._page_host_blocked.pop(host, None)
def _block_page_host(self, host: str) -> None:
if host in self._page_host_blocked:
return
# Log once per host per TTL: the failure is host-wide, so repeating it for
# every mature model in the library would be pure noise.
logger.warning(
"CivitAI model pages on %s refused the request (likely a Cloudflare "
"challenge); skipping that host for %d minutes",
host,
_PAGE_HOST_BLOCK_TTL // 60,
)
self._page_host_blocked[host] = time.time()
if self._page_host_preference == host:
self._page_host_preference = None
def _page_host(self) -> Optional[str]:
"""Resolve the page host from the ``civitai_host`` setting."""
py/services/model_update_service.py
+90 -19
View File
@@ -128,6 +128,10 @@ class ModelVersionRecord:
# price rises back above the threshold. Gives the panel "dropped X ago" and
# the unread count something to compare against.
price_alert_since: Optional[float] = None
# When a price fetch was last *attempted* (success or failure). Distinguishes
# "never tried" from "tried and no price is readable", which is what lets the
# UI say "price unavailable" for mature models instead of showing nothing.
price_check_attempted_at: Optional[float] = None
@dataclass
@@ -379,6 +383,7 @@ class ModelUpdateService:
price_checked_at REAL,
price_alert_state INTEGER NOT NULL DEFAULT 0,
price_alert_since REAL,
price_check_attempted_at REAL,
PRIMARY KEY (model_id, version_id),
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
);
@@ -666,6 +671,10 @@ class ModelUpdateService:
"ALTER TABLE model_update_versions "
"ADD COLUMN price_alert_since REAL"
),
"price_check_attempted_at": (
"ALTER TABLE model_update_versions "
"ADD COLUMN price_check_attempted_at REAL"
),
}
for column, statement in migrations.items():
@@ -779,6 +788,7 @@ class ModelUpdateService:
price_checked_at REAL,
price_alert_state INTEGER NOT NULL DEFAULT 0,
price_alert_since REAL,
price_check_attempted_at REAL,
PRIMARY KEY (model_id, version_id),
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
)
@@ -810,6 +820,7 @@ class ModelUpdateService:
"price_checked_at",
"price_alert_state",
"price_alert_since",
"price_check_attempted_at",
]
defaults = {
"sort_index": "0",
@@ -834,6 +845,7 @@ class ModelUpdateService:
"price_checked_at": "NULL",
"price_alert_state": "0",
"price_alert_since": "NULL",
"price_check_attempted_at": "NULL",
}
select_parts = []
@@ -1350,6 +1362,39 @@ class ModelUpdateService:
return None
return float(row["newest"])
def count_unavailable_prices(self, model_type: Optional[str] = None) -> int:
"""Gated versions whose price we tried to read and could not.
Mostly mature models: their pages are served only by ``civitai.red``, which
refuses non-browser clients, while ``civitai.com`` hides them from
anonymous visitors. Reported so the UI can be honest instead of silent.
"""
params: List[Any] = []
type_filter = ""
if model_type:
type_filter = "AND s.model_type = ?"
params.append(model_type)
with self._connect() as conn:
row = conn.execute(
f"""
SELECT COUNT(*) AS unavailable
FROM model_update_versions v
JOIN model_update_status s ON s.model_id = v.model_id
WHERE v.should_ignore = 0
AND s.should_ignore_model = 0
{type_filter}
AND v.price_check_attempted_at IS NOT NULL
AND v.price_buzz IS NULL
AND (v.paid_access IS NOT NULL OR v.is_paid = 1 OR v.is_early_access = 1)
""",
tuple(params),
).fetchone()
if row is None:
return 0
return int(row["unavailable"])
async def _refresh_single_model(
self,
model_type: str,
@@ -2079,6 +2124,7 @@ class ModelUpdateService:
"price_checked_at": None,
"price_alert_state": False,
"price_alert_since": None,
"price_check_attempted_at": None,
}
if remote_version.price_checked_at is not None:
@@ -2095,6 +2141,7 @@ class ModelUpdateService:
"price_checked_at": source.price_checked_at,
"price_alert_state": source.price_alert_state,
"price_alert_since": source.price_alert_since,
"price_check_attempted_at": source.price_check_attempted_at,
}
def _price_tracking_enabled(self) -> bool:
@@ -2186,42 +2233,60 @@ class ModelUpdateService:
Never raises for a provider problem: price tracking is a convenience, and
an unreadable page (or a provider that has no prices at all) must leave
the update check exactly as it was.
A failed attempt is still recorded (``price_check_attempted_at``) so the UI
can distinguish "we could not read a price" from "we never looked" — that
is the honest state for mature models, whose pages are served only by
civitai.red, which refuses non-browser clients.
"""
getter = getattr(metadata_provider, "get_model_prices", None)
if not callable(getter):
return list(versions)
attempted_at = time.time()
try:
prices = await getter(model_id)
except RateLimitError:
raise
except Exception as exc: # pragma: no cover - defensive
logger.debug("Price fetch failed for model %s: %s", model_id, exc)
return list(versions)
prices = None
if not isinstance(prices, Mapping) or not prices:
return list(versions)
if not isinstance(prices, Mapping):
prices = {}
checked_at = time.time()
enriched: List[ModelVersionRecord] = []
for version in versions:
if not self._has_structural_gate(version):
enriched.append(version)
continue
fields = prices.get(version.version_id)
if not isinstance(fields, Mapping):
# Not priced (or not understood): leave the stored values alone so
# the next refresh retries instead of recording a blank price.
enriched.append(version)
continue
recognized = {
key: value
for key, value in fields.items()
if key in _PRICE_FIELD_NAMES
}
recognized = (
{
key: value
for key, value in fields.items()
if key in _PRICE_FIELD_NAMES
}
if isinstance(fields, Mapping)
else {}
)
if not recognized:
enriched.append(version)
# Keep the stored price (there may be none) but remember the try, so
# the UI can say "unavailable" instead of showing nothing at all.
enriched.append(
replace(version, price_check_attempted_at=attempted_at)
)
continue
enriched.append(
replace(version, price_checked_at=checked_at, **recognized)
replace(
version,
price_checked_at=attempted_at,
price_check_attempted_at=attempted_at,
**recognized,
)
)
return enriched
@@ -2468,7 +2533,7 @@ class ModelUpdateService:
is_early_access, usage_control, paid_access, is_paid, file_count,
gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz,
accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state,
price_alert_since
price_alert_since, price_check_attempted_at
FROM model_update_versions
WHERE model_id IN ({placeholders})
ORDER BY model_id ASC, sort_index ASC, version_id ASC
@@ -2517,6 +2582,11 @@ class ModelUpdateService:
if row["price_alert_since"] is not None
else None
),
price_check_attempted_at=(
float(row["price_check_attempted_at"])
if row["price_check_attempted_at"] is not None
else None
),
)
)
@@ -2582,8 +2652,8 @@ class ModelUpdateService:
is_early_access, usage_control, paid_access, is_paid, file_count,
gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz,
accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state,
price_alert_since
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
price_alert_since, price_check_attempted_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
version.version_id,
@@ -2611,6 +2681,7 @@ class ModelUpdateService:
version.price_checked_at,
1 if version.price_alert_state else 0,
version.price_alert_since,
version.price_check_attempted_at,
),
)
conn.commit()