diff --git a/docs/plans/paid-model-price-tracking.md b/docs/plans/paid-model-price-tracking.md index 95c054c7..8555e6b6 100644 --- a/docs/plans/paid-model-price-tracking.md +++ b/docs/plans/paid-model-price-tracking.md @@ -600,3 +600,55 @@ Verification: `pytest` 3652 passed / 7 skipped, `npm run test:js` 1444 passed, p standalone server run that seeded the update DB and confirmed the payload shape, the `kind` split, the `civitaiUrl`, and that changing `price_alert_threshold_buzz` through `POST /api/lm/settings` changes panel membership immediately with no refresh. + +### 11.10 Known limitation: mature (NSFW) model prices, and the host fallback + +End-to-end verification against the live site found that the page fetch was broken for a whole class +of users: **the hosts are not interchangeable**, and the user's `civitai_host` preference was +silently fatal. + +Measured with the app's own HTTP stack (aiohttp) and with httpx, browser User-Agent in both cases: + +| Target | civitai.com | civitai.green | civitai.red | +| --- | --- | --- | --- | +| Anonymously visible model page | 200, prices parse | 200, prices parse (same bytes as .com) | **403 Cloudflare challenge** | +| Mature (NSFW) model page | 404 | 404 | **403 Cloudflare challenge** | + +* `civitai.red` refuses non-browser clients outright — User-Agent, `Accept-Language`, `Sec-Fetch-*` + and switching HTTP library all make no difference. (An earlier manual check with `curl` passed by + luck of TLS fingerprint, which is why this was missed: it was a false positive.) +* A real user on `civitai_host=civitai.red` therefore captured **zero** prices + (`price_checked_at = 0` in their update DB) even with tracking enabled. +* Mature models are hidden from anonymous visitors on `.com`/`.green` and only served by `.red`, so + **no host can currently read their price.** The public API cannot help: it trims `paidAccess` to + `{permanent, endsAt}` by design, and the public `mini/{id}` endpoint exposes only per-generation + `fees`. + +What P5a now does about it: + +* **Host fallback** (`CivitaiClient.get_model_prices`): the configured host is tried first, then the + others (`civitai_page_host_candidates`), first parseable payload wins. The host that worked is + remembered, and a host that refuses outright (403) is parked for 15 minutes — a host-wide failure + must not cost three requests per mature model in the library. A 404 is model-specific and does + **not** park the host. +* **Honest "unavailable" state**: `price_check_attempted_at` separates "we tried and could not read + a price" from "we never looked". Gated versions in that state show a muted `Price unavailable` + badge in the versions tab, and the panel reports `unavailableCount`. +* **Diagnostics**: a host refusal is warned once per host per TTL, and a model with no price source + logs the per-host reasons (404 vs challenge) instead of failing silently at debug level. + +Recorded options for mature models, deliberately **not** implemented: + +1. **Internal tRPC with the user's API key** (`modelVersion.getById` on `.red`): the route is a + `publicProcedure` with `requiredScope: ModelsRead`, and `isBearerAuth` satisfies + `acceptableOrigin`, so the user's own key would work and `.red`'s `/api` paths are not challenged. + Rejected as the default because the endpoint is undocumented and its own 401 message says to use + the public API. If ever wanted, it belongs behind an off-by-default setting. +2. **Extension-assisted fetch**: `lm-civitai-extension` runs inside the user's browser, so it has + both the Cloudflare clearance and the login session needed to read mature pages. This is the only + route that would work without an undocumented API, but it is a cross-component design of its own. + +**Strengthened upstream ask:** the public API should expose the price. The argument is no longer +"convenience" — the page route is demonstrably unreliable (one host challenges non-browser clients, +the other two hide mature models from anonymous visitors), so a supported field is the only way for +any third-party tool to show prices for the models where creators monetize most. diff --git a/locales/de.json b/locales/de.json index 8b65c215..c432c06d 100644 --- a/locales/de.json +++ b/locales/de.json @@ -2037,7 +2037,9 @@ "ignored": "Ignoriert", "ignoredTooltip": "Für diese Version sind Update-Benachrichtigungen deaktiviert", "onSiteOnly": "Nur On-Site", - "onSiteOnlyTooltip": "Diese Version ist nur für die On-Site-Generierung auf CivitAI verfügbar" + "onSiteOnlyTooltip": "Diese Version ist nur für die On-Site-Generierung auf CivitAI verfügbar", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "Herunterladen", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/en.json b/locales/en.json index e877ea94..b29554f0 100644 --- a/locales/en.json +++ b/locales/en.json @@ -2037,7 +2037,9 @@ "ignored": "Ignored", "ignoredTooltip": "Update notifications are disabled for this version", "onSiteOnly": "On-Site Only", - "onSiteOnlyTooltip": "This version is only available for on-site generation on CivitAI" + "onSiteOnlyTooltip": "This version is only available for on-site generation on CivitAI", + "priceUnavailable": "Price unavailable", + "priceUnavailableTooltip": "CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "Download", @@ -2408,7 +2410,8 @@ "notInLibrary": "Not in library", "droppedAgo": "dropped {when}", "openCivitai": "Open on CivitAI", - "openLocal": "Open" + "openLocal": "Open", + "unavailable": "{count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/es.json b/locales/es.json index aea10e71..f7f12ee2 100644 --- a/locales/es.json +++ b/locales/es.json @@ -2037,7 +2037,9 @@ "ignored": "Ignorada", "ignoredTooltip": "Las notificaciones de actualización están desactivadas para esta versión", "onSiteOnly": "Solo en Sitio", - "onSiteOnlyTooltip": "Esta versión solo está disponible para generación en el sitio de CivitAI" + "onSiteOnlyTooltip": "Esta versión solo está disponible para generación en el sitio de CivitAI", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "Descargar", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/fr.json b/locales/fr.json index 81b44219..5f19be0a 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -2037,7 +2037,9 @@ "ignored": "Ignorée", "ignoredTooltip": "Les notifications de mise à jour sont désactivées pour cette version", "onSiteOnly": "Uniquement sur Site", - "onSiteOnlyTooltip": "Cette version n'est disponible que pour la génération sur le site CivitAI" + "onSiteOnlyTooltip": "Cette version n'est disponible que pour la génération sur le site CivitAI", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "Télécharger", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/he.json b/locales/he.json index fd9249ef..7001a5c6 100644 --- a/locales/he.json +++ b/locales/he.json @@ -2037,7 +2037,9 @@ "ignored": "התעלם", "ignoredTooltip": "התראות העדכון מושבתות עבור גרסה זו", "onSiteOnly": "רק באתר", - "onSiteOnlyTooltip": "גרסה זו זמינה רק ליצירה באתר CivitAI" + "onSiteOnlyTooltip": "גרסה זו זמינה רק ליצירה באתר CivitAI", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "הורדה", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/ja.json b/locales/ja.json index f7c2e006..e537c576 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -2037,7 +2037,9 @@ "ignored": "無視中", "ignoredTooltip": "このバージョンの更新通知は無効です", "onSiteOnly": "サイト内のみ", - "onSiteOnlyTooltip": "このバージョンはCivitAIサイト内でのみ利用可能で、ダウンロードはできません" + "onSiteOnlyTooltip": "このバージョンはCivitAIサイト内でのみ利用可能で、ダウンロードはできません", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "ダウンロード", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/ko.json b/locales/ko.json index a16157ad..d82f271b 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -2037,7 +2037,9 @@ "ignored": "무시됨", "ignoredTooltip": "이 버전은 업데이트 알림이 비활성화되어 있습니다", "onSiteOnly": "사이트 내 전용", - "onSiteOnlyTooltip": "이 버전은 CivitAI 사이트 내에서만 사용 가능하며 다운로드할 수 없습니다" + "onSiteOnlyTooltip": "이 버전은 CivitAI 사이트 내에서만 사용 가능하며 다운로드할 수 없습니다", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "다운로드", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/ru.json b/locales/ru.json index 83312489..986e9eba 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -2037,7 +2037,9 @@ "ignored": "Игнорируется", "ignoredTooltip": "Уведомления об обновлениях для этой версии отключены", "onSiteOnly": "Только на Сайте", - "onSiteOnlyTooltip": "Эта версия доступна только для генерации на сайте CivitAI" + "onSiteOnlyTooltip": "Эта версия доступна только для генерации на сайте CivitAI", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "Скачать", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 0d68a8f5..9a854ad4 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -2037,7 +2037,9 @@ "ignored": "已忽略", "ignoredTooltip": "此版本已关闭更新通知", "onSiteOnly": "仅站内生成", - "onSiteOnlyTooltip": "此版本仅在 CivitAI 站内可用，无法下载" + "onSiteOnlyTooltip": "此版本仅在 CivitAI 站内可用，无法下载", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "下载", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 9b650d42..0d3162fb 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -2037,7 +2037,9 @@ "ignored": "已忽略", "ignoredTooltip": "此版本已關閉更新通知", "onSiteOnly": "僅站內生成", - "onSiteOnlyTooltip": "此版本僅在 CivitAI 站內可用，無法下載" + "onSiteOnlyTooltip": "此版本僅在 CivitAI 站內可用，無法下載", + "priceUnavailable": "[TODO: Translate] Price unavailable", + "priceUnavailableTooltip": "[TODO: Translate] CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser" }, "actions": { "download": "下載", @@ -2408,7 +2410,8 @@ "notInLibrary": "[TODO: Translate] Not in library", "droppedAgo": "[TODO: Translate] dropped {when}", "openCivitai": "[TODO: Translate] Open on CivitAI", - "openLocal": "[TODO: Translate] Open" + "openLocal": "[TODO: Translate] Open", + "unavailable": "[TODO: Translate] {count} paid version(s) have no readable price (mature models can only be read in a browser)" } }, "support": { diff --git a/py/routes/handlers/misc_handlers.py b/py/routes/handlers/misc_handlers.py index 0044cc12..d49abbde 100644 --- a/py/routes/handlers/misc_handlers.py +++ b/py/routes/handlers/misc_handlers.py @@ -4384,6 +4384,7 @@ class PriceAlertsHandler: None, threshold_buzz=threshold, limit=limit ) newest_checked_at = update_service.newest_price_checked_at() + unavailable_count = update_service.count_unavailable_prices() except Exception as exc: logger.error("Failed to load price alerts: %s", exc, exc_info=True) return web.json_response( @@ -4406,6 +4407,9 @@ class PriceAlertsHandler: "enabled": bool(self._setting("price_tracking_enabled", False)), "thresholdBuzz": threshold, "newestCheckedAt": newest_checked_at, + # Gated versions whose price could not be read (mature models + # are served only by the challenged civitai.red host). + "unavailableCount": unavailable_count, "alerts": alerts, } ) diff --git a/py/routes/handlers/model_handlers.py b/py/routes/handlers/model_handlers.py index 2640a6e0..4d090821 100644 --- a/py/routes/handlers/model_handlers.py +++ b/py/routes/handlers/model_handlers.py @@ -3511,6 +3511,10 @@ class ModelUpdateHandler: "acceptsBlueBuzz": bool(getattr(version, "accepts_blue_buzz", False)), "priceSaleEndsAt": getattr(version, "price_sale_ends_at", None), "priceCheckedAt": getattr(version, "price_checked_at", None), + # Set when a price fetch was attempted (success or not): a gated + # version with no price and an attempt marker is "unavailable", + # which is the honest state for mature models. + "priceAttemptedAt": getattr(version, "price_check_attempted_at", None), "priceAlert": bool(getattr(version, "price_alert_state", False)), "filePath": context.get("file_path"), "fileName": context.get("file_name"), diff --git a/py/services/civitai_client.py b/py/services/civitai_client.py index f0ddffce..ed8a3c30 100644 --- a/py/services/civitai_client.py +++ b/py/services/civitai_client.py @@ -20,7 +20,11 @@ from .model_metadata_provider import ( ) from .downloader import get_downloader from .errors import RateLimitError, ResourceNotFoundError -from ..utils.civitai_utils import build_civitai_model_page_url, resolve_license_payload +from ..utils.civitai_utils import ( + build_civitai_model_page_url, + civitai_page_host_candidates, + resolve_license_payload, +) from ..utils.civitai_page_prices import parse_model_page_prices from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash @@ -32,6 +36,24 @@ logger = logging.getLogger(__name__) _CREATOR_COUNT_CACHE_TTL_SECONDS = 600 _creator_model_count_cache: Dict[str, Tuple[float, Optional[int]]] = {} +# How long a page host stays on the skip list after refusing a request outright +# (Cloudflare's challenge, surfaced as 403 "Access forbidden"). Long enough to +# cover a whole update refresh, short enough to recover within a session. +_PAGE_HOST_BLOCK_TTL = 15 * 60 + + +def _is_host_level_refusal(message: str) -> bool: + """Whether a failed request means "this host refuses us" rather than "this + model is unavailable". + + ``downloader.make_request`` collapses statuses into prose, and 403 ("Access + forbidden") is the one a Cloudflare challenge produces. A 404 ("Resource not + found") is model-specific — mature pages are hidden from anonymous visitors — + so it must not put the whole host on the skip list. + """ + + return "forbidden" in message.lower() + class CivitaiClient: _instance = None @@ -67,6 +89,10 @@ class CivitaiClient: str, Tuple[Optional[Dict[str, Any]], Optional[str]] ] = OrderedDict() self._MAX_CACHE_ENTRIES = 500 + # Model-page host bookkeeping: which host last worked, and which ones are + # currently refusing us (see get_model_prices). + self._page_host_preference: Optional[str] = None + self._page_host_blocked: Dict[str, float] = {} def _build_image_info_url(self, image_id: str) -> str: return f"{self.base_url}/images?imageId={image_id}&nsfw=X&withMeta=true" @@ -396,6 +422,19 @@ class CivitaiClient: empty dict when the page loads but lists no gated version, or None when the page could not be read or understood — callers keep any stored price. + Several hosts are tried in order, because the hosts are not equivalent: + + * ``civitai.red`` serves mature model pages that ``civitai.com`` hides from + anonymous visitors, but it is behind a Cloudflare challenge that refuses + non-browser clients outright (403 for any User-Agent). + * ``civitai.com`` / ``civitai.green`` answer normally for anonymously + visible models, and 404 for the mature ones. + + So the user's ``civitai_host`` preference is a starting point, not the only + option. Mature models whose page cannot be read from any host stay + priceless — see the known limitation in + ``docs/plans/paid-model-price-tracking.md``. + This is a public anonymous page fetch: no API key and no internal endpoint, so a failure here must never fail the update check itself. """ @@ -405,39 +444,92 @@ class CivitaiClient: except (TypeError, ValueError): return None - url = build_civitai_model_page_url(normalized_id, host=self._page_host()) - if not url: - return None + candidates = self._page_host_candidates() + failures: List[str] = [] - try: - success, result = await self._make_request( - "GET", - url, - use_auth=False, - custom_headers={"Accept": "text/html"}, - ) - except RateLimitError: - # The shared rate-limit gate already recorded it; skip this model. - raise - except Exception as exc: # pragma: no cover - defensive - logger.debug("Failed to fetch model page for %s: %s", model_id, exc) - return None + for host in candidates: + url = build_civitai_model_page_url(normalized_id, host=host) + if not url: + continue - if not success or not isinstance(result, str): - logger.debug( - "No model page payload for %s (success=%s, type=%s)", - model_id, - success, - type(result).__name__, - ) - return None + try: + success, result = await self._make_request( + "GET", + url, + use_auth=False, + custom_headers={"Accept": "text/html"}, + ) + except RateLimitError: + # The shared rate-limit gate already recorded it; skip this model. + raise + except Exception as exc: # pragma: no cover - defensive + failures.append(f"{host}: {exc}") + continue - prices = parse_model_page_prices(result) - if prices is None: - logger.debug( - "Model page for %s carried no usable price payload", model_id - ) - return prices + if not success or not isinstance(result, str): + message = result if isinstance(result, str) else type(result).__name__ + failures.append(f"{host}: {message}") + if isinstance(result, str) and _is_host_level_refusal(result): + # A refusal like Cloudflare's "Access forbidden" applies to the + # host, not to this model, so stop paying for it for a while. + self._block_page_host(host) + continue + + prices = parse_model_page_prices(result) + if prices is None: + failures.append(f"{host}: no usable price payload") + continue + + self._remember_page_host(host) + return prices + + logger.warning( + "No price source for model %s; tried %s. Mature models are only served " + "by civitai.red, which challenges non-browser clients.", + model_id, + "; ".join(failures) or "no candidate hosts", + ) + return None + + def _page_host_candidates(self) -> List[str]: + """Ordered hosts to try: the last one that worked, then the preference.""" + + preferred = self._page_host() + ordered = list(civitai_page_host_candidates(preferred)) + if self._page_host_preference and self._page_host_preference in ordered: + ordered.remove(self._page_host_preference) + ordered.insert(0, self._page_host_preference) + + now = time.time() + usable = [ + host + for host in ordered + if now - self._page_host_blocked.get(host, 0.0) >= _PAGE_HOST_BLOCK_TTL + ] + # Never return an empty list: a blocked host is still better than no attempt + # once the preference and the memo disagree. + return usable or ordered + + def _remember_page_host(self, host: str) -> None: + if self._page_host_preference != host: + logger.info("CivitAI model pages are being read from %s", host) + self._page_host_preference = host + self._page_host_blocked.pop(host, None) + + def _block_page_host(self, host: str) -> None: + if host in self._page_host_blocked: + return + # Log once per host per TTL: the failure is host-wide, so repeating it for + # every mature model in the library would be pure noise. + logger.warning( + "CivitAI model pages on %s refused the request (likely a Cloudflare " + "challenge); skipping that host for %d minutes", + host, + _PAGE_HOST_BLOCK_TTL // 60, + ) + self._page_host_blocked[host] = time.time() + if self._page_host_preference == host: + self._page_host_preference = None def _page_host(self) -> Optional[str]: """Resolve the page host from the ``civitai_host`` setting.""" diff --git a/py/services/model_update_service.py b/py/services/model_update_service.py index 28ee8d32..10fdce4e 100644 --- a/py/services/model_update_service.py +++ b/py/services/model_update_service.py @@ -128,6 +128,10 @@ class ModelVersionRecord: # price rises back above the threshold. Gives the panel "dropped X ago" and # the unread count something to compare against. price_alert_since: Optional[float] = None + # When a price fetch was last *attempted* (success or failure). Distinguishes + # "never tried" from "tried and no price is readable", which is what lets the + # UI say "price unavailable" for mature models instead of showing nothing. + price_check_attempted_at: Optional[float] = None @dataclass @@ -379,6 +383,7 @@ class ModelUpdateService: price_checked_at REAL, price_alert_state INTEGER NOT NULL DEFAULT 0, price_alert_since REAL, + price_check_attempted_at REAL, PRIMARY KEY (model_id, version_id), FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE ); @@ -666,6 +671,10 @@ class ModelUpdateService: "ALTER TABLE model_update_versions " "ADD COLUMN price_alert_since REAL" ), + "price_check_attempted_at": ( + "ALTER TABLE model_update_versions " + "ADD COLUMN price_check_attempted_at REAL" + ), } for column, statement in migrations.items(): @@ -779,6 +788,7 @@ class ModelUpdateService: price_checked_at REAL, price_alert_state INTEGER NOT NULL DEFAULT 0, price_alert_since REAL, + price_check_attempted_at REAL, PRIMARY KEY (model_id, version_id), FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE ) @@ -810,6 +820,7 @@ class ModelUpdateService: "price_checked_at", "price_alert_state", "price_alert_since", + "price_check_attempted_at", ] defaults = { "sort_index": "0", @@ -834,6 +845,7 @@ class ModelUpdateService: "price_checked_at": "NULL", "price_alert_state": "0", "price_alert_since": "NULL", + "price_check_attempted_at": "NULL", } select_parts = [] @@ -1350,6 +1362,39 @@ class ModelUpdateService: return None return float(row["newest"]) + def count_unavailable_prices(self, model_type: Optional[str] = None) -> int: + """Gated versions whose price we tried to read and could not. + + Mostly mature models: their pages are served only by ``civitai.red``, which + refuses non-browser clients, while ``civitai.com`` hides them from + anonymous visitors. Reported so the UI can be honest instead of silent. + """ + + params: List[Any] = [] + type_filter = "" + if model_type: + type_filter = "AND s.model_type = ?" + params.append(model_type) + + with self._connect() as conn: + row = conn.execute( + f""" + SELECT COUNT(*) AS unavailable + FROM model_update_versions v + JOIN model_update_status s ON s.model_id = v.model_id + WHERE v.should_ignore = 0 + AND s.should_ignore_model = 0 + {type_filter} + AND v.price_check_attempted_at IS NOT NULL + AND v.price_buzz IS NULL + AND (v.paid_access IS NOT NULL OR v.is_paid = 1 OR v.is_early_access = 1) + """, + tuple(params), + ).fetchone() + if row is None: + return 0 + return int(row["unavailable"]) + async def _refresh_single_model( self, model_type: str, @@ -2079,6 +2124,7 @@ class ModelUpdateService: "price_checked_at": None, "price_alert_state": False, "price_alert_since": None, + "price_check_attempted_at": None, } if remote_version.price_checked_at is not None: @@ -2095,6 +2141,7 @@ class ModelUpdateService: "price_checked_at": source.price_checked_at, "price_alert_state": source.price_alert_state, "price_alert_since": source.price_alert_since, + "price_check_attempted_at": source.price_check_attempted_at, } def _price_tracking_enabled(self) -> bool: @@ -2186,42 +2233,60 @@ class ModelUpdateService: Never raises for a provider problem: price tracking is a convenience, and an unreadable page (or a provider that has no prices at all) must leave the update check exactly as it was. + + A failed attempt is still recorded (``price_check_attempted_at``) so the UI + can distinguish "we could not read a price" from "we never looked" — that + is the honest state for mature models, whose pages are served only by + civitai.red, which refuses non-browser clients. """ getter = getattr(metadata_provider, "get_model_prices", None) if not callable(getter): return list(versions) + attempted_at = time.time() try: prices = await getter(model_id) except RateLimitError: raise except Exception as exc: # pragma: no cover - defensive logger.debug("Price fetch failed for model %s: %s", model_id, exc) - return list(versions) + prices = None - if not isinstance(prices, Mapping) or not prices: - return list(versions) + if not isinstance(prices, Mapping): + prices = {} - checked_at = time.time() enriched: List[ModelVersionRecord] = [] for version in versions: + if not self._has_structural_gate(version): + enriched.append(version) + continue + fields = prices.get(version.version_id) - if not isinstance(fields, Mapping): - # Not priced (or not understood): leave the stored values alone so - # the next refresh retries instead of recording a blank price. - enriched.append(version) - continue - recognized = { - key: value - for key, value in fields.items() - if key in _PRICE_FIELD_NAMES - } + recognized = ( + { + key: value + for key, value in fields.items() + if key in _PRICE_FIELD_NAMES + } + if isinstance(fields, Mapping) + else {} + ) if not recognized: - enriched.append(version) + # Keep the stored price (there may be none) but remember the try, so + # the UI can say "unavailable" instead of showing nothing at all. + enriched.append( + replace(version, price_check_attempted_at=attempted_at) + ) continue + enriched.append( - replace(version, price_checked_at=checked_at, **recognized) + replace( + version, + price_checked_at=attempted_at, + price_check_attempted_at=attempted_at, + **recognized, + ) ) return enriched @@ -2468,7 +2533,7 @@ class ModelUpdateService: is_early_access, usage_control, paid_access, is_paid, file_count, gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz, accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state, - price_alert_since + price_alert_since, price_check_attempted_at FROM model_update_versions WHERE model_id IN ({placeholders}) ORDER BY model_id ASC, sort_index ASC, version_id ASC @@ -2517,6 +2582,11 @@ class ModelUpdateService: if row["price_alert_since"] is not None else None ), + price_check_attempted_at=( + float(row["price_check_attempted_at"]) + if row["price_check_attempted_at"] is not None + else None + ), ) ) @@ -2582,8 +2652,8 @@ class ModelUpdateService: is_early_access, usage_control, paid_access, is_paid, file_count, gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz, accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state, - price_alert_since - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + price_alert_since, price_check_attempted_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( version.version_id, @@ -2611,6 +2681,7 @@ class ModelUpdateService: version.price_checked_at, 1 if version.price_alert_state else 0, version.price_alert_since, + version.price_check_attempted_at, ), ) conn.commit() diff --git a/py/utils/civitai_utils.py b/py/utils/civitai_utils.py index 9466d0dd..ceacdc77 100644 --- a/py/utils/civitai_utils.py +++ b/py/utils/civitai_utils.py @@ -41,6 +41,21 @@ def normalize_civitai_page_host(hostname: str | None) -> str: return DEFAULT_CIVITAI_PAGE_HOST +def civitai_page_host_candidates(hostname: str | None = None) -> tuple[str, ...]: + """Ordered page hosts to try, the preferred one first. + + The hosts are not interchangeable: ``civitai.red`` serves mature model pages + that ``civitai.com`` hides from anonymous visitors, but it also sits behind a + Cloudflare challenge that rejects non-browser HTTP clients outright. Trying the + others when the preferred host refuses a request is what keeps a user's + ``civitai_host`` preference from disabling page reads entirely. + """ + + preferred = normalize_civitai_page_host(hostname) + rest = sorted(_SUPPORTED_CIVITAI_PAGE_HOSTS - {preferred}) + return (preferred, *rest) + + def build_civitai_model_page_url( model_id: str | int | None, version_id: str | int | None = None, diff --git a/static/js/components/shared/ModelVersionsTab.js b/static/js/components/shared/ModelVersionsTab.js index 2dea9b33..6a5fa695 100644 --- a/static/js/components/shared/ModelVersionsTab.js +++ b/static/js/components/shared/ModelVersionsTab.js @@ -609,6 +609,20 @@ function renderRow(version, options) { ) : buildPriceTooltip(version, paidBadgeLabel), })); + } else if (isGated(version) && version.priceAttemptedAt) { + // We looked and could not read a price. Staying silent would read as + // "free", which is the one thing this version is not. + badges.push(buildBadge( + translate('modals.model.versions.badges.priceUnavailable', {}, 'Price unavailable'), + 'muted', + { + title: translate( + 'modals.model.versions.badges.priceUnavailableTooltip', + {}, + 'CivitAI does not publish this price in its public API, and mature model pages can only be read in a browser' + ), + } + )); } // A version that used to be gated and no longer is. `gateLapsedAt` is diff --git a/static/js/managers/UpdateService.js b/static/js/managers/UpdateService.js index 12c77c0c..d82ece99 100644 --- a/static/js/managers/UpdateService.js +++ b/static/js/managers/UpdateService.js @@ -28,7 +28,8 @@ async function fetchPriceAlerts(limit = 200) { alerts: Array.isArray(payload.alerts) ? payload.alerts : [], enabled: payload.enabled === true, thresholdBuzz: payload.thresholdBuzz ?? 0, - newestCheckedAt: payload.newestCheckedAt ?? null + newestCheckedAt: payload.newestCheckedAt ?? null, + unavailableCount: payload.unavailableCount ?? 0 }; } @@ -112,6 +113,7 @@ export class UpdateService { this.priceAlertsEnabled = false; this.priceAlertsThreshold = 0; this.priceAlertsNewestCheckedAt = null; + this.priceAlertsUnavailableCount = 0; this.priceAlertsLoading = false; this.priceAlertSegment = 'below_threshold'; this.unreadPriceAlertCount = 0; @@ -404,6 +406,7 @@ export class UpdateService { this.priceAlertsEnabled = payload.enabled; this.priceAlertsThreshold = payload.thresholdBuzz; this.priceAlertsNewestCheckedAt = payload.newestCheckedAt; + this.priceAlertsUnavailableCount = payload.unavailableCount || 0; this.refreshUnreadPriceAlertCount(); this.renderPriceAlerts(); return this.priceAlerts; @@ -467,7 +470,7 @@ export class UpdateService { } if (stale) { const checkedAt = this.priceAlertsNewestCheckedAt; - const staleText = error + let staleText = error ? translate( 'update.priceAlerts.loadFailed', {}, @@ -480,6 +483,15 @@ export class UpdateService { `Prices last checked ${formatRelativeTime(checkedAt)}` ) : ''; + const unavailable = this.priceAlertsUnavailableCount; + if (unavailable > 0) { + const unavailableText = translate( + 'update.priceAlerts.unavailable', + { count: unavailable }, + `${unavailable} paid version(s) have no readable price (mature models can only be read in a browser)` + ); + staleText = staleText ? `${staleText} · ${unavailableText}` : unavailableText; + } stale.textContent = staleText; stale.classList.toggle('hidden', !staleText); } diff --git a/tests/frontend/components/modelVersionsTab.downloadButton.test.js b/tests/frontend/components/modelVersionsTab.downloadButton.test.js index 5510ebac..a4ffd5fe 100644 --- a/tests/frontend/components/modelVersionsTab.downloadButton.test.js +++ b/tests/frontend/components/modelVersionsTab.downloadButton.test.js @@ -283,3 +283,96 @@ describe('ModelVersionsTab download button visibility', () => { expect(openFileSelectionForVersion).not.toHaveBeenCalled(); }); }); + +describe('ModelVersionsTab price badges', () => { + let getModelApiClient; + let fetchModelUpdateVersions; + + beforeEach(async () => { + vi.resetModules(); + document.body.innerHTML = ` +
+
+
+
+
+ `; + ({ getModelApiClient } = await import(API_FACTORY_MODULE)); + fetchModelUpdateVersions = vi.fn(); + getModelApiClient.mockReturnValue({ + fetchModelUpdateVersions, + fetchModelRoots: vi.fn(), + setModelUpdateIgnore: vi.fn(), + setVersionUpdateIgnore: vi.fn(), + deleteModel: vi.fn(), + }); + }); + + afterEach(() => { + document.body.innerHTML = ''; + }); + + function rowFor(versionId) { + return document.querySelector(`.model-version-row[data-version-id="${versionId}"]`); + } + + it('shows the price for a gated version whose price was captured', async () => { + fetchModelUpdateVersions.mockResolvedValue(buildRecord([ + { + versionId: 20, + name: 'Alpha', + isInLibrary: false, + shouldIgnore: false, + isPaid: true, + paidAccess: { permanent: true, endsAt: null }, + priceBuzz: 5000, + listPriceBuzz: 5000, + priceCheckedAt: 1791039694.5, + priceAttemptedAt: 1791039694.5, + }, + ])); + + await renderVersions(); + + expect(rowFor(20).textContent).toContain('5,000 Buzz'); + }); + + it('marks a gated version with no readable price as unavailable', async () => { + // The mature-model case: we looked, no host would serve the page. + fetchModelUpdateVersions.mockResolvedValue(buildRecord([ + { + versionId: 21, + name: 'Beta', + isInLibrary: false, + shouldIgnore: false, + isPaid: true, + paidAccess: { permanent: true, endsAt: null }, + priceBuzz: null, + priceAttemptedAt: 1791039694.5, + }, + ])); + + await renderVersions(); + + expect(rowFor(21).textContent).toContain('Price unavailable'); + }); + + it('stays quiet when the price was never looked up', async () => { + fetchModelUpdateVersions.mockResolvedValue(buildRecord([ + { + versionId: 22, + name: 'Gamma', + isInLibrary: false, + shouldIgnore: false, + isPaid: true, + paidAccess: { permanent: true, endsAt: null }, + priceBuzz: null, + priceAttemptedAt: null, + }, + ])); + + await renderVersions(); + + expect(rowFor(22).textContent).not.toContain('Price unavailable'); + }); +}); diff --git a/tests/frontend/managers/updateServicePriceAlerts.test.js b/tests/frontend/managers/updateServicePriceAlerts.test.js index e94b165b..a772dd82 100644 --- a/tests/frontend/managers/updateServicePriceAlerts.test.js +++ b/tests/frontend/managers/updateServicePriceAlerts.test.js @@ -203,6 +203,21 @@ describe('UpdateService price alerts panel', () => { expect(document.querySelectorAll('#priceAlertsList .price-alert-item')).toHaveLength(0); }); + it('reports how many gated versions have no readable price', async () => { + global.fetch = vi.fn().mockResolvedValue( + createFetchResponse( + alertPayload([BELOW_THRESHOLD_ALERT], { unavailableCount: 3 }) + ) + ); + + await service.loadPriceAlerts({ force: true }); + + expect(service.priceAlertsUnavailableCount).toBe(3); + const note = document.getElementById('priceAlertsStale'); + expect(note.classList.contains('hidden')).toBe(false); + expect(note.textContent).toContain('3 paid version(s) have no readable price'); + }); + it('keeps the last known list when the request fails', async () => { global.fetch = vi .fn() diff --git a/tests/routes/test_misc_routes.py b/tests/routes/test_misc_routes.py index adae23d6..8719cf23 100644 --- a/tests/routes/test_misc_routes.py +++ b/tests/routes/test_misc_routes.py @@ -2950,9 +2950,10 @@ def _price_alerts_adapter(update_service, scanners=None): class _FakeUpdateService: - def __init__(self, alerts): + def __init__(self, alerts, *, unavailable_count=0): self.alerts = alerts self.calls = [] + self.unavailable_count = unavailable_count async def get_price_alerts(self, model_type=None, *, threshold_buzz=None, limit=200): self.calls.append((model_type, threshold_buzz, limit)) @@ -2961,6 +2962,9 @@ class _FakeUpdateService: def newest_price_checked_at(self): return 1791039694.5 + def count_unavailable_prices(self, model_type=None): + return self.unavailable_count + @pytest.mark.asyncio async def test_price_alerts_handler_returns_the_global_list(): @@ -2997,6 +3001,7 @@ async def test_price_alerts_handler_returns_the_global_list(): assert payload["enabled"] is True assert payload["thresholdBuzz"] == 300 assert payload["newestCheckedAt"] == 1791039694.5 + assert payload["unavailableCount"] == 0 assert payload["alerts"][0]["civitaiUrl"] == ( "https://civitai.com/models/2981320?modelVersionId=3379626" ) diff --git a/tests/services/test_civitai_client.py b/tests/services/test_civitai_client.py index a0aa3af1..a023ee23 100644 --- a/tests/services/test_civitai_client.py +++ b/tests/services/test_civitai_client.py @@ -1,4 +1,5 @@ import copy +import pathlib from unittest.mock import AsyncMock import pytest @@ -921,3 +922,104 @@ async def test_get_model_prices_propagates_rate_limit(downloader): with pytest.raises(RateLimitError): await client.get_model_prices(7) + + +# --- Model page host fallback ------------------------------------------------- + +_PAGE_FIXTURE = ( + pathlib.Path(__file__).resolve().parents[1] + / "utils" + / "fixtures" + / "civitai_model_page_paid.html" +) + + +def _page_html() -> str: + return _PAGE_FIXTURE.read_text(encoding="utf-8") + + +async def test_get_model_prices_falls_back_when_the_preferred_host_refuses( + downloader, monkeypatch +): + """civitai.red refuses non-browser clients (Cloudflare); the price must still + be readable from a host that answers. This is the user-visible bug: with + civitai_host=civitai.red every fetch used to fail.""" + + client = await CivitaiClient.get_instance() + monkeypatch.setattr(client, "_page_host", lambda: "civitai.red") + seen = [] + + async def fake_make_request(method, url, use_auth=True, **kwargs): + seen.append(url) + if "civitai.red" in url: + return False, "Access forbidden" + return True, _page_html() + + downloader.make_request = fake_make_request + + result = await client.get_model_prices(4242) + + assert result is not None + assert result[1001]["price_buzz"] == 5000 + assert seen[0].startswith("https://civitai.red/") + assert any("civitai.com" in url for url in seen) + # The working host is remembered and the refusing one is parked. + assert client._page_host_preference == "civitai.com" + assert "civitai.red" in client._page_host_blocked + + +async def test_get_model_prices_reuses_the_working_host(downloader, monkeypatch): + client = await CivitaiClient.get_instance() + monkeypatch.setattr(client, "_page_host", lambda: "civitai.red") + calls = [] + + async def fake_make_request(method, url, use_auth=True, **kwargs): + calls.append(url) + if "civitai.red" in url: + return False, "Access forbidden" + return True, _page_html() + + downloader.make_request = fake_make_request + + await client.get_model_prices(4242) + calls.clear() + await client.get_model_prices(4242) + + # Second time around the parked host is not retried. + assert calls and all("civitai.red" not in url for url in calls) + + +async def test_get_model_prices_returns_none_when_no_host_can_serve(downloader, monkeypatch): + """Mature models: 404 on civitai.com/green, challenge on civitai.red.""" + + client = await CivitaiClient.get_instance() + monkeypatch.setattr(client, "_page_host", lambda: "civitai.com") + + async def fake_make_request(method, url, use_auth=True, **kwargs): + if "civitai.red" in url: + return False, "Access forbidden" + return False, "Resource not found" + + downloader.make_request = fake_make_request + + assert await client.get_model_prices(2981320) is None + + +async def test_get_model_prices_does_not_park_a_host_on_404(downloader, monkeypatch): + """A 404 is model-specific (mature content hidden anonymously), not a reason + to stop using the host for other models.""" + + client = await CivitaiClient.get_instance() + monkeypatch.setattr(client, "_page_host", lambda: "civitai.com") + + async def fake_make_request(method, url, use_auth=True, **kwargs): + if "civitai.com" in url: + return False, "Resource not found" + return False, "Access forbidden" + + downloader.make_request = fake_make_request + + await client.get_model_prices(1) + + assert "civitai.com" not in client._page_host_blocked + assert "civitai.red" in client._page_host_blocked diff --git a/tests/services/test_model_update_service.py b/tests/services/test_model_update_service.py index 733edca0..33d15d57 100644 --- a/tests/services/test_model_update_service.py +++ b/tests/services/test_model_update_service.py @@ -1884,3 +1884,74 @@ async def test_newest_price_checked_at_reports_the_latest_fetch(tmp_path): assert newest is not None assert newest > 0 + + +@pytest.mark.asyncio +async def test_failed_price_attempt_is_recorded_as_unavailable(tmp_path): + """A gated version we tried to price and could not must be distinguishable + from one we never looked at — that is the honest "unavailable" state for + mature models, whose pages no host will serve anonymously.""" + + service = _price_service(tmp_path, price_tracking_enabled=True) + scanner = DummyScanner(LOCAL_RAW_DATA) + failing = PriceProvider(GATED_RESPONSE, prices=None) + + await service.refresh_for_model_type("lora", scanner, failing) + record = await service.get_record("lora", 1) + version = next(v for v in record.versions if v.version_id == 12) + + assert failing.price_calls == 1 + assert version.price_buzz is None + assert version.price_checked_at is None + assert version.price_check_attempted_at is not None + assert service.count_unavailable_prices("lora") == 1 + assert service.count_unavailable_prices("checkpoint") == 0 + # Nothing to alert on, and no price alert state. + assert await service.get_price_alerts("lora") == [] + assert version.price_alert_state is False + + +@pytest.mark.asyncio +async def test_successful_price_attempt_sets_both_markers(tmp_path): + service = _price_service(tmp_path, price_tracking_enabled=True) + scanner = DummyScanner(LOCAL_RAW_DATA) + + await service.refresh_for_model_type( + "lora", scanner, PriceProvider(GATED_RESPONSE, prices=PRICE_PAYLOAD) + ) + record = await service.get_record("lora", 1) + version = next(v for v in record.versions if v.version_id == 12) + + assert version.price_buzz == 250 + assert version.price_checked_at is not None + assert version.price_check_attempted_at is not None + assert service.count_unavailable_prices("lora") == 0 + + +@pytest.mark.asyncio +async def test_no_price_attempt_is_recorded_while_tracking_is_off(tmp_path): + service = _price_service(tmp_path) # tracking off + scanner = DummyScanner(LOCAL_RAW_DATA) + + await service.refresh_for_model_type("lora", scanner, DummyProvider(GATED_RESPONSE)) + record = await service.get_record("lora", 1) + + assert record.versions[0].price_check_attempted_at is None + assert service.count_unavailable_prices("lora") == 0 + + +@pytest.mark.asyncio +async def test_unavailable_marker_clears_when_the_version_becomes_free(tmp_path): + service = _price_service(tmp_path, price_tracking_enabled=True) + scanner = DummyScanner(LOCAL_RAW_DATA) + + await service.refresh_for_model_type( + "lora", scanner, PriceProvider(GATED_RESPONSE, prices=None) + ) + assert service.count_unavailable_prices("lora") == 1 + + await service.refresh_for_model_type("lora", scanner, DummyProvider(FREE_RESPONSE)) + record = await service.get_record("lora", 1) + + assert record.versions[0].price_check_attempted_at is None + assert service.count_unavailable_prices("lora") == 0