fix(update): read model-page prices from a host that answers

End-to-end verification against the live site found the price capture broken for
a whole class of users: the civitai page hosts are not interchangeable, and the
user's civitai_host preference was silently fatal. With civitai_host=civitai.red
the update DB held zero prices even with tracking enabled.

- civitai.red refuses non-browser HTTP clients outright (Cloudflare challenge,
  403 for any User-Agent, aiohttp and httpx alike), while civitai.com and
  civitai.green answer normally for anonymously visible models and 404 for
  mature ones. An earlier manual check with curl passed on TLS fingerprint luck,
  which is why this was missed.
- get_model_prices now tries the configured host first, then the others, and
  takes the first parseable payload. The host that worked is remembered, and a
  host that refuses outright is parked for 15 minutes so a library full of
  mature models does not pay three requests each; a 404 is model-specific and
  does not park the host. Links keep using the configured host, which is where
  the user's own browser has clearance.
- Mature models still have no price source anywhere, so that is now stated
  instead of silent: price_check_attempted_at separates "tried and unreadable"
  from "never looked", gated versions show a muted "Price unavailable" badge,
  and the alerts panel reports unavailableCount.
- Failures are logged at warning level, once per host per TTL, with the
  per-host reason, instead of only at debug level.
- The recorded alternatives (internal tRPC with the user's API key, or an
  extension-assisted fetch from the user's browser) and the strengthened
  upstream ask for a public price field are documented in the plan.
This commit is contained in:
Will Miao
2026-10-04 20:00:29 +08:00
parent 7ed19c185c
commit ec5fef512b
23 changed files with 653 additions and 73 deletions
+102
View File
@@ -1,4 +1,5 @@
import copy
import pathlib
from unittest.mock import AsyncMock
import pytest
@@ -921,3 +922,104 @@ async def test_get_model_prices_propagates_rate_limit(downloader):
with pytest.raises(RateLimitError):
await client.get_model_prices(7)
# --- Model page host fallback -------------------------------------------------
_PAGE_FIXTURE = (
pathlib.Path(__file__).resolve().parents[1]
/ "utils"
/ "fixtures"
/ "civitai_model_page_paid.html"
)
def _page_html() -> str:
return _PAGE_FIXTURE.read_text(encoding="utf-8")
async def test_get_model_prices_falls_back_when_the_preferred_host_refuses(
downloader, monkeypatch
):
"""civitai.red refuses non-browser clients (Cloudflare); the price must still
be readable from a host that answers. This is the user-visible bug: with
civitai_host=civitai.red every fetch used to fail."""
client = await CivitaiClient.get_instance()
monkeypatch.setattr(client, "_page_host", lambda: "civitai.red")
seen = []
async def fake_make_request(method, url, use_auth=True, **kwargs):
seen.append(url)
if "civitai.red" in url:
return False, "Access forbidden"
return True, _page_html()
downloader.make_request = fake_make_request
result = await client.get_model_prices(4242)
assert result is not None
assert result[1001]["price_buzz"] == 5000
assert seen[0].startswith("https://civitai.red/")
assert any("civitai.com" in url for url in seen)
# The working host is remembered and the refusing one is parked.
assert client._page_host_preference == "civitai.com"
assert "civitai.red" in client._page_host_blocked
async def test_get_model_prices_reuses_the_working_host(downloader, monkeypatch):
client = await CivitaiClient.get_instance()
monkeypatch.setattr(client, "_page_host", lambda: "civitai.red")
calls = []
async def fake_make_request(method, url, use_auth=True, **kwargs):
calls.append(url)
if "civitai.red" in url:
return False, "Access forbidden"
return True, _page_html()
downloader.make_request = fake_make_request
await client.get_model_prices(4242)
calls.clear()
await client.get_model_prices(4242)
# Second time around the parked host is not retried.
assert calls and all("civitai.red" not in url for url in calls)
async def test_get_model_prices_returns_none_when_no_host_can_serve(downloader, monkeypatch):
"""Mature models: 404 on civitai.com/green, challenge on civitai.red."""
client = await CivitaiClient.get_instance()
monkeypatch.setattr(client, "_page_host", lambda: "civitai.com")
async def fake_make_request(method, url, use_auth=True, **kwargs):
if "civitai.red" in url:
return False, "Access forbidden"
return False, "Resource not found"
downloader.make_request = fake_make_request
assert await client.get_model_prices(2981320) is None
async def test_get_model_prices_does_not_park_a_host_on_404(downloader, monkeypatch):
"""A 404 is model-specific (mature content hidden anonymously), not a reason
to stop using the host for other models."""
client = await CivitaiClient.get_instance()
monkeypatch.setattr(client, "_page_host", lambda: "civitai.com")
async def fake_make_request(method, url, use_auth=True, **kwargs):
if "civitai.com" in url:
return False, "Resource not found"
return False, "Access forbidden"
downloader.make_request = fake_make_request
await client.get_model_prices(1)
assert "civitai.com" not in client._page_host_blocked
assert "civitai.red" in client._page_host_blocked