fix(update): read model-page prices from a host that answers

End-to-end verification against the live site found the price capture broken for
a whole class of users: the civitai page hosts are not interchangeable, and the
user's civitai_host preference was silently fatal. With civitai_host=civitai.red
the update DB held zero prices even with tracking enabled.

- civitai.red refuses non-browser HTTP clients outright (Cloudflare challenge,
  403 for any User-Agent, aiohttp and httpx alike), while civitai.com and
  civitai.green answer normally for anonymously visible models and 404 for
  mature ones. An earlier manual check with curl passed on TLS fingerprint luck,
  which is why this was missed.
- get_model_prices now tries the configured host first, then the others, and
  takes the first parseable payload. The host that worked is remembered, and a
  host that refuses outright is parked for 15 minutes so a library full of
  mature models does not pay three requests each; a 404 is model-specific and
  does not park the host. Links keep using the configured host, which is where
  the user's own browser has clearance.
- Mature models still have no price source anywhere, so that is now stated
  instead of silent: price_check_attempted_at separates "tried and unreadable"
  from "never looked", gated versions show a muted "Price unavailable" badge,
  and the alerts panel reports unavailableCount.
- Failures are logged at warning level, once per host per TTL, with the
  per-host reason, instead of only at debug level.
- The recorded alternatives (internal tRPC with the user's API key, or an
  extension-assisted fetch from the user's browser) and the strengthened
  upstream ask for a public price field are documented in the plan.

py/services/civitai_client.py

@@ -20,7 +20,11 @@ from .model_metadata_provider import (
 )
 from .downloader import get_downloader
 from .errors import RateLimitError, ResourceNotFoundError
-from ..utils.civitai_utils import build_civitai_model_page_url, resolve_license_payload
+from ..utils.civitai_utils import (
+    build_civitai_model_page_url,
+    civitai_page_host_candidates,
+    resolve_license_payload,
+)
 from ..utils.civitai_page_prices import parse_model_page_prices
 from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
 
@@ -32,6 +36,24 @@ logger = logging.getLogger(__name__)
 _CREATOR_COUNT_CACHE_TTL_SECONDS = 600
 _creator_model_count_cache: Dict[str, Tuple[float, Optional[int]]] = {}
 
+# How long a page host stays on the skip list after refusing a request outright
+# (Cloudflare's challenge, surfaced as 403 "Access forbidden"). Long enough to
+# cover a whole update refresh, short enough to recover within a session.
+_PAGE_HOST_BLOCK_TTL = 15 * 60
+
+
+def _is_host_level_refusal(message: str) -> bool:
+    """Whether a failed request means "this host refuses us" rather than "this
+    model is unavailable".
+
+    ``downloader.make_request`` collapses statuses into prose, and 403 ("Access
+    forbidden") is the one a Cloudflare challenge produces. A 404 ("Resource not
+    found") is model-specific — mature pages are hidden from anonymous visitors —
+    so it must not put the whole host on the skip list.
+    """
+
+    return "forbidden" in message.lower()
+
 
 class CivitaiClient:
     _instance = None
@@ -67,6 +89,10 @@ class CivitaiClient:
             str, Tuple[Optional[Dict[str, Any]], Optional[str]]
         ] = OrderedDict()
         self._MAX_CACHE_ENTRIES = 500
+        # Model-page host bookkeeping: which host last worked, and which ones are
+        # currently refusing us (see get_model_prices).
+        self._page_host_preference: Optional[str] = None
+        self._page_host_blocked: Dict[str, float] = {}
 
     def _build_image_info_url(self, image_id: str) -> str:
         return f"{self.base_url}/images?imageId={image_id}&nsfw=X&withMeta=true"
@@ -396,6 +422,19 @@ class CivitaiClient:
         empty dict when the page loads but lists no gated version, or None when
         the page could not be read or understood — callers keep any stored price.
 
+        Several hosts are tried in order, because the hosts are not equivalent:
+
+        * ``civitai.red`` serves mature model pages that ``civitai.com`` hides from
+          anonymous visitors, but it is behind a Cloudflare challenge that refuses
+          non-browser clients outright (403 for any User-Agent).
+        * ``civitai.com`` / ``civitai.green`` answer normally for anonymously
+          visible models, and 404 for the mature ones.
+
+        So the user's ``civitai_host`` preference is a starting point, not the only
+        option. Mature models whose page cannot be read from any host stay
+        priceless — see the known limitation in
+        ``docs/plans/paid-model-price-tracking.md``.
+
         This is a public anonymous page fetch: no API key and no internal
         endpoint, so a failure here must never fail the update check itself.
         """
@@ -405,39 +444,92 @@ class CivitaiClient:
         except (TypeError, ValueError):
             return None
 
-        url = build_civitai_model_page_url(normalized_id, host=self._page_host())
-        if not url:
-            return None
+        candidates = self._page_host_candidates()
+        failures: List[str] = []
 
-        try:
-            success, result = await self._make_request(
-                "GET",
-                url,
-                use_auth=False,
-                custom_headers={"Accept": "text/html"},
-            )
-        except RateLimitError:
-            # The shared rate-limit gate already recorded it; skip this model.
-            raise
-        except Exception as exc:  # pragma: no cover - defensive
-            logger.debug("Failed to fetch model page for %s: %s", model_id, exc)
-            return None
+        for host in candidates:
+            url = build_civitai_model_page_url(normalized_id, host=host)
+            if not url:
+                continue
 
-        if not success or not isinstance(result, str):
-            logger.debug(
-                "No model page payload for %s (success=%s, type=%s)",
-                model_id,
-                success,
-                type(result).__name__,
-            )
-            return None
+            try:
+                success, result = await self._make_request(
+                    "GET",
+                    url,
+                    use_auth=False,
+                    custom_headers={"Accept": "text/html"},
+                )
+            except RateLimitError:
+                # The shared rate-limit gate already recorded it; skip this model.
+                raise
+            except Exception as exc:  # pragma: no cover - defensive
+                failures.append(f"{host}: {exc}")
+                continue
 
-        prices = parse_model_page_prices(result)
-        if prices is None:
-            logger.debug(
-                "Model page for %s carried no usable price payload", model_id
-            )
-        return prices
+            if not success or not isinstance(result, str):
+                message = result if isinstance(result, str) else type(result).__name__
+                failures.append(f"{host}: {message}")
+                if isinstance(result, str) and _is_host_level_refusal(result):
+                    # A refusal like Cloudflare's "Access forbidden" applies to the
+                    # host, not to this model, so stop paying for it for a while.
+                    self._block_page_host(host)
+                continue
+
+            prices = parse_model_page_prices(result)
+            if prices is None:
+                failures.append(f"{host}: no usable price payload")
+                continue
+
+            self._remember_page_host(host)
+            return prices
+
+        logger.warning(
+            "No price source for model %s; tried %s. Mature models are only served "
+            "by civitai.red, which challenges non-browser clients.",
+            model_id,
+            "; ".join(failures) or "no candidate hosts",
+        )
+        return None
+
+    def _page_host_candidates(self) -> List[str]:
+        """Ordered hosts to try: the last one that worked, then the preference."""
+
+        preferred = self._page_host()
+        ordered = list(civitai_page_host_candidates(preferred))
+        if self._page_host_preference and self._page_host_preference in ordered:
+            ordered.remove(self._page_host_preference)
+            ordered.insert(0, self._page_host_preference)
+
+        now = time.time()
+        usable = [
+            host
+            for host in ordered
+            if now - self._page_host_blocked.get(host, 0.0) >= _PAGE_HOST_BLOCK_TTL
+        ]
+        # Never return an empty list: a blocked host is still better than no attempt
+        # once the preference and the memo disagree.
+        return usable or ordered
+
+    def _remember_page_host(self, host: str) -> None:
+        if self._page_host_preference != host:
+            logger.info("CivitAI model pages are being read from %s", host)
+        self._page_host_preference = host
+        self._page_host_blocked.pop(host, None)
+
+    def _block_page_host(self, host: str) -> None:
+        if host in self._page_host_blocked:
+            return
+        # Log once per host per TTL: the failure is host-wide, so repeating it for
+        # every mature model in the library would be pure noise.
+        logger.warning(
+            "CivitAI model pages on %s refused the request (likely a Cloudflare "
+            "challenge); skipping that host for %d minutes",
+            host,
+            _PAGE_HOST_BLOCK_TTL // 60,
+        )
+        self._page_host_blocked[host] = time.time()
+        if self._page_host_preference == host:
+            self._page_host_preference = None
 
     def _page_host(self) -> Optional[str]:
         """Resolve the page host from the ``civitai_host`` setting."""