feat(update): track buzz prices and alert below a threshold

CivitAI's public API deliberately omits prices — paidAccess is trimmed to
{permanent, endsAt} because "pricing belongs to the purchase flow" — but the
public model page embeds the site's own model.getById result, including
paidAccess.terms, in its server-rendered payload. That is read anonymously
(no API key, no internal endpoint, no forged Origin), one request per gated
model, so only the ~2% of models that actually carry a gate pay for it.

- optional capture, off by default: price_tracking_enabled,
  price_alert_threshold_buzz (0 = alert on "became free" only) and
  price_check_ttl_hours; prices refresh on their own TTL and immediately when a
  gate changes, and a failed fetch keeps the stored price instead of blanking it
- versions that stop carrying a gate are marked free (persisted gate_lapsed_at)
  and gate transitions are reported as events on the refresh response, so a
  version already in the library can announce that it became free
- price_alert_state plus a price_drop edge event; new
  GET /api/lm/{type}/updates/price-alerts lists what is under the threshold
- versions tab shows the price (effective, with the list price struck through
  and a Blue Buzz note) and a Free Now badge; an update check toasts the
  transitions in one message
- the parser and the alerts query are unit-tested against a trimmed page
  fixture, and every route definition is now asserted to resolve to a handler

Plan, verification notes and the deviations from it are in
docs/plans/paid-model-price-tracking.md.
This commit is contained in:
Will Miao
2026-10-04 08:53:24 +08:00
parent 5f4054265d
commit ad2402724b
31 changed files with 2829 additions and 33 deletions
+73 -10
View File
@@ -50,6 +50,7 @@ from ...services.errors import RateLimitError, ResourceNotFoundError
from ...utils.civitai_utils import resolve_license_payload
from ...utils.file_utils import calculate_sha256
from ...utils.metadata_manager import MetadataManager
from ...utils.paid_access import is_early_access_deadline_active
from ...utils.sidecar_paths import get_metadata_path
from ...utils.url_utils import relative_root_prefix
@@ -2941,6 +2942,20 @@ class ModelUpdateHandler:
same_base_scope = self._uses_same_base_update_scope()
# Gate/price transitions are reported for every refreshed model, not only
# for the ones that qualify as updates: "this version became free" matters
# for a version the user already has, which never shows up as an update.
events = []
for record in records.values():
for event in getattr(record, "events", None) or []:
events.append(
{
"modelId": record.model_id,
"modelType": record.model_type,
**event,
}
)
serialized_records = []
for record in records.values():
has_update_fn = getattr(record, "has_update", None)
@@ -2962,6 +2977,7 @@ class ModelUpdateHandler:
{
"success": True,
"records": serialized_records,
"events": events,
}
)
@@ -3054,6 +3070,46 @@ class ModelUpdateHandler:
{"success": True, "record": self._serialize_record(record)}
)
async def get_price_alerts(self, request: web.Request) -> web.Response:
"""List gated versions whose stored price crosses the alert threshold.
Backed by the persisted alert state, so it matches the badges; the
threshold (and whether price tracking runs at all) comes from settings.
"""
try:
limit = int(request.query.get("limit") or 200)
except (TypeError, ValueError):
limit = 200
alerts = await self._update_service.get_price_alerts(
self._service.model_type, limit=limit
)
threshold = 0
enabled = False
if self._settings is not None:
try:
threshold = int(
self._settings.get("price_alert_threshold_buzz", 0) or 0
)
except (TypeError, ValueError):
threshold = 0
try:
enabled = bool(
self._settings.get("price_tracking_enabled", False)
)
except Exception:
enabled = False
return web.json_response(
{
"success": True,
"enabled": enabled,
"thresholdBuzz": threshold,
"alerts": alerts,
}
)
async def get_model_versions(self, request: web.Request) -> web.Response:
model_id = self._normalize_model_id(request.match_info.get("model_id"))
if model_id is None:
@@ -3391,6 +3447,7 @@ class ModelUpdateHandler:
hide_early_access=hide_early_access,
hide_paid=hide_paid,
),
"events": list(getattr(record, "events", None) or []),
"versions": [
self._serialize_version(version, context.get(version.version_id))
for version in record.versions
@@ -3414,16 +3471,11 @@ class ModelUpdateHandler:
if getattr(version, "is_paid", False) and not version.early_access_ends_at:
is_early_access = False
elif version.early_access_ends_at:
try:
from datetime import datetime, timezone
ea_date = datetime.fromisoformat(
version.early_access_ends_at.replace("Z", "+00:00")
)
is_early_access = ea_date > datetime.now(timezone.utc)
except (ValueError, AttributeError):
# If date parsing fails, treat as active EA (conservative)
is_early_access = True
# Shared with the update service and the download gate so the badge,
# the update filter and the download warning cannot disagree.
is_early_access = is_early_access_deadline_active(
version.early_access_ends_at
)
elif getattr(version, "is_early_access", False):
# Fallback to basic EA flag from bulk API
is_early_access = True
@@ -3450,6 +3502,16 @@ class ModelUpdateHandler:
"usageControl": version.usage_control,
"isPaid": bool(getattr(version, "is_paid", False)),
"paidAccess": paid_access_payload,
# Set when a version that used to be gated became free, so the UI can
# keep showing "Free" long after the transition.
"gateLapsedAt": getattr(version, "gate_lapsed_at", None),
"priceBuzz": getattr(version, "price_buzz", None),
"listPriceBuzz": getattr(version, "list_price_buzz", None),
"generationPriceBuzz": getattr(version, "generation_price_buzz", None),
"acceptsBlueBuzz": bool(getattr(version, "accepts_blue_buzz", False)),
"priceSaleEndsAt": getattr(version, "price_sale_ends_at", None),
"priceCheckedAt": getattr(version, "price_checked_at", None),
"priceAlert": bool(getattr(version, "price_alert_state", False)),
"filePath": context.get("file_path"),
"fileName": context.get("file_name"),
# Weight-file variant count (None when unknown); lets the UI hide
@@ -3612,5 +3674,6 @@ class ModelHandlerSet:
"set_version_update_ignore": self.updates.set_version_update_ignore,
"get_model_update_status": self.updates.get_model_update_status,
"get_model_versions": self.updates.get_model_versions,
"get_price_alerts": self.updates.get_price_alerts,
"cancel_task": self.query.cancel_task,
}
+3
View File
@@ -110,6 +110,9 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
RouteDefinition(
"GET", "/api/lm/{prefix}/updates/versions/{model_id}", "get_model_versions"
),
RouteDefinition(
"GET", "/api/lm/{prefix}/updates/price-alerts", "get_price_alerts"
),
RouteDefinition("POST", "/api/lm/download-model", "download_model"),
RouteDefinition("GET", "/api/lm/download-model-get", "download_model_get"),
RouteDefinition("GET", "/api/lm/cancel-download-get", "cancel_download_get"),
+68 -1
View File
@@ -20,7 +20,8 @@ from .model_metadata_provider import (
)
from .downloader import get_downloader
from .errors import RateLimitError, ResourceNotFoundError
from ..utils.civitai_utils import resolve_license_payload
from ..utils.civitai_utils import build_civitai_model_page_url, resolve_license_payload
from ..utils.civitai_page_prices import parse_model_page_prices
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
logger = logging.getLogger(__name__)
@@ -383,6 +384,72 @@ class CivitaiClient:
logger.error(f"Error fetching model versions in bulk: {exc}")
return None
async def get_model_prices(
self, model_id: int
) -> Optional[Dict[int, Dict[str, Any]]]:
"""Fetch per-version buzz prices for one model from its public page.
CivitAI's public REST API deliberately omits prices, but the model page
embeds the site's own ``model.getById`` result (including
``paidAccess.terms``) in its server-rendered payload. One request covers
every version of the model. Returns ``{version_id: price fields}``, an
empty dict when the page loads but lists no gated version, or None when
the page could not be read or understood — callers keep any stored price.
This is a public anonymous page fetch: no API key and no internal
endpoint, so a failure here must never fail the update check itself.
"""
try:
normalized_id = int(model_id)
except (TypeError, ValueError):
return None
url = build_civitai_model_page_url(normalized_id, host=self._page_host())
if not url:
return None
try:
success, result = await self._make_request(
"GET",
url,
use_auth=False,
custom_headers={"Accept": "text/html"},
)
except RateLimitError:
# The shared rate-limit gate already recorded it; skip this model.
raise
except Exception as exc: # pragma: no cover - defensive
logger.debug("Failed to fetch model page for %s: %s", model_id, exc)
return None
if not success or not isinstance(result, str):
logger.debug(
"No model page payload for %s (success=%s, type=%s)",
model_id,
success,
type(result).__name__,
)
return None
prices = parse_model_page_prices(result)
if prices is None:
logger.debug(
"Model page for %s carried no usable price payload", model_id
)
return prices
def _page_host(self) -> Optional[str]:
"""Resolve the page host from the ``civitai_host`` setting."""
try:
from .settings_manager import get_settings_manager
settings = get_settings_manager()
return settings.get("civitai_host") if settings else None
except Exception:
return None
async def get_model_version(
self, model_id: int | None = None, version_id: int | None = None
) -> Optional[Dict[str, Any]]:
+53
View File
@@ -183,6 +183,17 @@ class ModelMetadataProvider(ABC):
"""
return None
async def get_model_prices(
self, model_id: int
) -> Optional[Dict[int, Dict[str, Any]]]:
"""Fetch per-version buzz prices for a model, when the provider has them.
CivitAI publishes prices only inside its public model page payload;
providers that cannot read it (CivArchive, SQLite, OpenModelDB) keep the
default of None, which callers treat as "no price information".
"""
return None
class CivitaiModelMetadataProvider(ModelMetadataProvider):
"""Provider that uses Civitai API for metadata"""
@@ -222,6 +233,11 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
) -> Optional[Dict[str, Any]]:
return await self.client.get_version_file_mini(version_id, file_id)
async def get_model_prices(
self, model_id: int
) -> Optional[Dict[int, Dict[str, Any]]]:
return await self.client.get_model_prices(model_id)
class CivArchiveModelMetadataProvider(ModelMetadataProvider):
"""Provider that uses CivArchive API for metadata"""
@@ -783,6 +799,34 @@ class FallbackMetadataProvider(ModelMetadataProvider):
def _iter_providers(self):
return zip(self.providers, self._provider_labels)
async def get_model_prices(
self, model_id: int
) -> Optional[Dict[int, Dict[str, Any]]]:
rate_limited = False
for provider, label in self._iter_providers():
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
continue
try:
result = await self._call_with_rate_limit(
label,
provider.get_model_prices,
model_id,
)
if result:
return result
except RateLimitError as exc:
rate_limited = True
logger.warning(
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
label,
exc.retry_after or 0,
)
continue
except Exception as e:
logger.debug("Provider %s failed for get_model_prices: %s", label, e)
continue
return None
def excluding(self, labels: "frozenset[str] | set[str]") -> "FallbackMetadataProvider":
"""Return a copy of this chain without the providers named in *labels*.
@@ -907,6 +951,15 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
file_id,
)
async def get_model_prices(
self, model_id: int
) -> Optional[Dict[int, Dict[str, Any]]]:
return await self._rate_limit_helper.run(
self._label,
self._provider.get_model_prices,
model_id,
)
class ModelMetadataProviderManager:
"""Manager for selecting and using model metadata providers"""
+465 -5
View File
@@ -11,7 +11,7 @@ import logging
import os
import sqlite3
import time
from dataclasses import dataclass, replace
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone
from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence
@@ -28,6 +28,17 @@ from ..utils.preview_selection import resolve_mature_threshold, select_preview_m
logger = logging.getLogger(__name__)
# Version fields a price payload may set (see py/utils/civitai_page_prices.py).
_PRICE_FIELD_NAMES = frozenset(
{
"price_buzz",
"list_price_buzz",
"generation_price_buzz",
"accepts_blue_buzz",
"price_sale_ends_at",
}
)
def _normalize_int(value) -> Optional[int]:
"""Safely convert a value to an integer."""
@@ -40,6 +51,16 @@ def _normalize_int(value) -> Optional[int]:
return None
def _format_utc_timestamp(value: float) -> str:
"""Format a POSIX timestamp the way CivitAI serializes its timestamps."""
return (
datetime.fromtimestamp(value, tz=timezone.utc)
.isoformat()
.replace("+00:00", "Z")
)
def _normalize_string(value) -> Optional[str]:
"""Return a stripped string or None if the value is empty."""
@@ -86,6 +107,23 @@ class ModelVersionRecord:
# e.g. records persisted before this field existed or locally-synthesized
# entries). Mirrors the frontend isModelWeightFile() filter.
file_count: Optional[int] = None
# When a version that used to carry a paid gate stopped carrying one. Kept so
# the UI can mark a version "free" long after the transition happened; cleared
# again if the version is gated once more.
gate_lapsed_at: Optional[str] = None
# Download price in Buzz for the gated version, and friends. All None until a
# price refresh succeeds (see the price tracking feature); `price_buzz` is the
# effective price a buyer pays now (sale-adjusted) and `list_price_buzz` the
# undiscounted stored price.
price_buzz: Optional[int] = None
list_price_buzz: Optional[int] = None
generation_price_buzz: Optional[int] = None
accepts_blue_buzz: bool = False
price_sale_ends_at: Optional[str] = None
price_checked_at: Optional[float] = None
# Last computed "price alert threshold hit" state, used to detect the edge
# (0 -> 1) that fires a one-shot price_drop event.
price_alert_state: bool = False
@dataclass
@@ -97,6 +135,12 @@ class ModelUpdateRecord:
versions: List[ModelVersionRecord]
last_checked_at: Optional[float]
should_ignore_model: bool
# Gate-state and price transitions observed during the refresh that produced
# this record. Derived, never persisted: a record read back from SQLite has an
# empty list. Each entry is {"versionId", "kind", "versionName",
# "isInLibrary", "priceBuzz"?} with kind in
# {"new_gate", "became_free", "price_drop"}.
events: List[Dict[str, Any]] = field(default_factory=list)
@property
def largest_version_id(self) -> Optional[int]:
@@ -322,6 +366,14 @@ class ModelUpdateService:
paid_access TEXT,
is_paid INTEGER NOT NULL DEFAULT 0,
file_count INTEGER,
gate_lapsed_at TEXT,
price_buzz INTEGER,
list_price_buzz INTEGER,
generation_price_buzz INTEGER,
accepts_blue_buzz INTEGER NOT NULL DEFAULT 0,
price_sale_ends_at TEXT,
price_checked_at REAL,
price_alert_state INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (model_id, version_id),
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
);
@@ -573,6 +625,38 @@ class ModelUpdateService:
"ALTER TABLE model_update_versions "
"ADD COLUMN file_count INTEGER"
),
"gate_lapsed_at": (
"ALTER TABLE model_update_versions "
"ADD COLUMN gate_lapsed_at TEXT"
),
"price_buzz": (
"ALTER TABLE model_update_versions "
"ADD COLUMN price_buzz INTEGER"
),
"list_price_buzz": (
"ALTER TABLE model_update_versions "
"ADD COLUMN list_price_buzz INTEGER"
),
"generation_price_buzz": (
"ALTER TABLE model_update_versions "
"ADD COLUMN generation_price_buzz INTEGER"
),
"accepts_blue_buzz": (
"ALTER TABLE model_update_versions "
"ADD COLUMN accepts_blue_buzz INTEGER NOT NULL DEFAULT 0"
),
"price_sale_ends_at": (
"ALTER TABLE model_update_versions "
"ADD COLUMN price_sale_ends_at TEXT"
),
"price_checked_at": (
"ALTER TABLE model_update_versions "
"ADD COLUMN price_checked_at REAL"
),
"price_alert_state": (
"ALTER TABLE model_update_versions "
"ADD COLUMN price_alert_state INTEGER NOT NULL DEFAULT 0"
),
}
for column, statement in migrations.items():
@@ -677,6 +761,14 @@ class ModelUpdateService:
paid_access TEXT,
is_paid INTEGER NOT NULL DEFAULT 0,
file_count INTEGER,
gate_lapsed_at TEXT,
price_buzz INTEGER,
list_price_buzz INTEGER,
generation_price_buzz INTEGER,
accepts_blue_buzz INTEGER NOT NULL DEFAULT 0,
price_sale_ends_at TEXT,
price_checked_at REAL,
price_alert_state INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (model_id, version_id),
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
)
@@ -699,6 +791,14 @@ class ModelUpdateService:
"paid_access",
"is_paid",
"file_count",
"gate_lapsed_at",
"price_buzz",
"list_price_buzz",
"generation_price_buzz",
"accepts_blue_buzz",
"price_sale_ends_at",
"price_checked_at",
"price_alert_state",
]
defaults = {
"sort_index": "0",
@@ -714,6 +814,14 @@ class ModelUpdateService:
"paid_access": "NULL",
"is_paid": "0",
"file_count": "NULL",
"gate_lapsed_at": "NULL",
"price_buzz": "NULL",
"list_price_buzz": "NULL",
"generation_price_buzz": "NULL",
"accepts_blue_buzz": "0",
"price_sale_ends_at": "NULL",
"price_checked_at": "NULL",
"price_alert_state": "0",
}
select_parts = []
@@ -1094,6 +1202,73 @@ class ModelUpdateService:
async with self._lock:
return self._get_records_bulk(model_type, normalized_ids)
async def get_price_alerts(
self, model_type: str, limit: int = 200
) -> List[Dict[str, Any]]:
"""Return the gated versions whose price currently crosses the threshold.
Reads the persisted ``price_alert_state`` rather than recomputing, so the
list matches the badges the user sees. Cheapest versions first, since that
is what a price watch is for.
"""
try:
normalized_limit = max(1, min(int(limit), 1000))
except (TypeError, ValueError):
normalized_limit = 200
with self._connect() as conn:
rows = conn.execute(
"""
SELECT v.model_id,
v.version_id,
v.name,
v.price_buzz,
v.list_price_buzz,
v.accepts_blue_buzz,
v.price_sale_ends_at,
v.price_checked_at,
v.is_in_library,
v.early_access_ends_at,
v.paid_access,
v.is_paid,
v.is_early_access,
s.model_type
FROM model_update_versions v
JOIN model_update_status s ON s.model_id = v.model_id
WHERE v.price_alert_state = 1
AND v.should_ignore = 0
AND s.should_ignore_model = 0
AND s.model_type = ?
ORDER BY v.price_buzz ASC, v.model_id ASC, v.version_id ASC
LIMIT ?
""",
(model_type, normalized_limit),
).fetchall()
return [
{
"modelId": int(row["model_id"]),
"modelType": row["model_type"],
"versionId": int(row["version_id"]),
"versionName": row["name"],
"priceBuzz": _normalize_int(row["price_buzz"]),
"listPriceBuzz": _normalize_int(row["list_price_buzz"]),
"acceptsBlueBuzz": bool(row["accepts_blue_buzz"]),
"priceSaleEndsAt": row["price_sale_ends_at"],
"priceCheckedAt": (
float(row["price_checked_at"])
if row["price_checked_at"] is not None
else None
),
"isInLibrary": bool(row["is_in_library"]),
"earlyAccessEndsAt": row["early_access_ends_at"],
"isPaid": bool(row["is_paid"]),
"isEarlyAccess": bool(row["is_early_access"]),
}
for row in rows
]
async def _refresh_single_model(
self,
model_type: str,
@@ -1168,6 +1343,20 @@ class ModelUpdateService:
elif fallback_attempted and fallback_error_message is None:
fallback_error_message = "no response"
# Optional price capture. Runs after the version list is known so only the
# models that actually carry a gate pay for a second (page) request, and
# stays outside the lock along with the other network I/O.
if (
refresh_succeeded
and isinstance(fetched_versions, list)
and fetched_versions
and metadata_provider is not None
and self._should_fetch_prices(fetched_versions, existing)
):
fetched_versions = await self._apply_model_prices(
metadata_provider, model_id, fetched_versions
)
if fallback_attempted:
if refresh_succeeded and isinstance(fetched_versions, list):
logger.info(
@@ -1631,10 +1820,71 @@ class ModelUpdateService:
existing_map = {version.version_id: version for version in existing.versions} if existing else {}
versions: List[ModelVersionRecord] = []
events: List[Dict[str, Any]] = []
seen_ids: set[int] = set()
lapse_timestamp = _format_utc_timestamp(timestamp)
for index, remote_version in enumerate(remote_versions):
version_id = remote_version.version_id
seen_ids.add(version_id)
existing_version = existing_map.get(version_id)
is_gated = self._has_structural_gate(remote_version)
was_gated = (
self._has_structural_gate(existing_version)
if existing_version is not None
else None
)
should_ignore = ignore_map.get(version_id, remote_version.should_ignore)
# `gate_lapsed_at` survives refreshes so a version that became free keeps
# its "free since" marker, and is cleared if the gate comes back.
if is_gated:
gate_lapsed_at = None
elif was_gated:
gate_lapsed_at = lapse_timestamp
elif existing_version is not None:
gate_lapsed_at = existing_version.gate_lapsed_at
else:
gate_lapsed_at = None
price_fields = self._price_fields_for(
remote_version, existing_version, is_gated=is_gated
)
price_fields["price_alert_state"] = self._evaluate_price_alert(
price_fields.get("price_buzz")
)
if (
existing_version is not None
and not should_ignore
and was_gated != is_gated
):
events.append(
{
"versionId": version_id,
"kind": "new_gate" if is_gated else "became_free",
"versionName": remote_version.name,
"isInLibrary": version_id in effective_local_set,
}
)
# Only the edge fires: a version that keeps sitting under the threshold
# must not re-announce itself on every refresh.
if (
existing_version is not None
and not should_ignore
and price_fields["price_alert_state"]
and not existing_version.price_alert_state
):
events.append(
{
"versionId": version_id,
"kind": "price_drop",
"versionName": remote_version.name,
"isInLibrary": version_id in effective_local_set,
"priceBuzz": price_fields.get("price_buzz"),
}
)
versions.append(
ModelVersionRecord(
version_id=version_id,
@@ -1644,7 +1894,7 @@ class ModelUpdateService:
size_bytes=remote_version.size_bytes,
preview_url=remote_version.preview_url or preview_map.get(version_id),
is_in_library=version_id in effective_local_set,
should_ignore=ignore_map.get(version_id, remote_version.should_ignore),
should_ignore=should_ignore,
sort_index=sort_map.get(version_id, index),
early_access_ends_at=remote_version.early_access_ends_at,
is_early_access=remote_version.is_early_access,
@@ -1656,6 +1906,8 @@ class ModelUpdateService:
if remote_version.file_count is not None
else file_count_map.get(version_id)
),
gate_lapsed_at=gate_lapsed_at,
**price_fields,
)
)
@@ -1694,8 +1946,192 @@ class ModelUpdateService:
versions=self._sorted_versions(versions),
last_checked_at=timestamp,
should_ignore_model=existing.should_ignore_model if existing else False,
events=events,
)
@staticmethod
def _has_structural_gate(version: Optional[ModelVersionRecord]) -> bool:
"""True when a version carries a paid gate, ignoring the clock.
Transition detection uses this rather than a time-based check so a gate
that lapsed since the previous refresh still reads as "was gated" — the
public API reports no ``paidAccess`` at all once a gate expires, so the
time-based helper alone would silently see "free before, free now".
"""
if version is None:
return False
return bool(version.paid_access) or version.is_paid or version.is_early_access
@staticmethod
def _price_fields_for(
remote_version: ModelVersionRecord,
existing_version: Optional[ModelVersionRecord],
*,
is_gated: bool,
) -> Dict[str, Any]:
"""Resolve the price columns for a refreshed version.
A refresh only carries prices when a price fetch actually ran (recognizable
by ``price_checked_at``), so otherwise the previously stored price is kept —
but only while the version is still gated: once it is free the stored
numbers are stale and must not keep firing alerts.
"""
if not is_gated:
return {
"price_buzz": None,
"list_price_buzz": None,
"generation_price_buzz": None,
"accepts_blue_buzz": False,
"price_sale_ends_at": None,
"price_checked_at": None,
"price_alert_state": False,
}
if remote_version.price_checked_at is not None:
source = remote_version
else:
source = existing_version or remote_version
return {
"price_buzz": source.price_buzz,
"list_price_buzz": source.list_price_buzz,
"generation_price_buzz": source.generation_price_buzz,
"accepts_blue_buzz": source.accepts_blue_buzz,
"price_sale_ends_at": source.price_sale_ends_at,
"price_checked_at": source.price_checked_at,
"price_alert_state": source.price_alert_state,
}
def _price_tracking_enabled(self) -> bool:
"""Whether the optional (page-fetching) price capture is switched on."""
try:
return bool(self._settings.get("price_tracking_enabled", False))
except Exception:
return False
def _price_alert_threshold_buzz(self) -> int:
"""Buzz threshold for a price alert; 0 means "free only"."""
try:
value = int(self._settings.get("price_alert_threshold_buzz", 0) or 0)
except (TypeError, ValueError):
return 0
return max(0, value)
def _evaluate_price_alert(self, price_buzz: Optional[int]) -> bool:
"""Whether a version's price crosses the user's alert threshold.
Free versions never carry a price, so a threshold of 0 alerts only via the
``became_free`` event — which is exactly the "or free" half of the request.
"""
if not self._price_tracking_enabled():
return False
if price_buzz is None:
return False
return price_buzz <= self._price_alert_threshold_buzz()
def _price_check_ttl_seconds(self) -> float:
try:
hours = float(self._settings.get("price_check_ttl_hours", 24))
except (TypeError, ValueError):
hours = 24.0
if hours <= 0:
hours = 24.0
return hours * 3600.0
def _should_fetch_prices(
self,
remote_versions: Sequence[ModelVersionRecord],
existing: Optional[ModelUpdateRecord],
) -> bool:
"""Whether this model needs a price fetch this round.
Only gated versions are considered, so an unmodified free library pays
nothing. A stored price is refreshed once its TTL lapses, and immediately
when the gate itself changed (a new end date or sale window is a reason to
believe the price moved).
"""
if not self._price_tracking_enabled():
return False
existing_map = (
{version.version_id: version for version in existing.versions}
if existing
else {}
)
now = time.time()
ttl = self._price_check_ttl_seconds()
for remote_version in remote_versions:
if not self._has_structural_gate(remote_version):
continue
stored = existing_map.get(remote_version.version_id)
if stored is None:
return True
if stored.price_checked_at is None:
return True
if (now - stored.price_checked_at) >= ttl:
return True
if (stored.paid_access or None) != (remote_version.paid_access or None):
return True
return False
async def _apply_model_prices(
self,
metadata_provider,
model_id: int,
versions: Sequence[ModelVersionRecord],
) -> List[ModelVersionRecord]:
"""Attach freshly fetched prices to the versions that have them.
Never raises for a provider problem: price tracking is a convenience, and
an unreadable page (or a provider that has no prices at all) must leave
the update check exactly as it was.
"""
getter = getattr(metadata_provider, "get_model_prices", None)
if not callable(getter):
return list(versions)
try:
prices = await getter(model_id)
except RateLimitError:
raise
except Exception as exc: # pragma: no cover - defensive
logger.debug("Price fetch failed for model %s: %s", model_id, exc)
return list(versions)
if not isinstance(prices, Mapping) or not prices:
return list(versions)
checked_at = time.time()
enriched: List[ModelVersionRecord] = []
for version in versions:
fields = prices.get(version.version_id)
if not isinstance(fields, Mapping):
# Not priced (or not understood): leave the stored values alone so
# the next refresh retries instead of recording a blank price.
enriched.append(version)
continue
recognized = {
key: value
for key, value in fields.items()
if key in _PRICE_FIELD_NAMES
}
if not recognized:
enriched.append(version)
continue
enriched.append(
replace(version, price_checked_at=checked_at, **recognized)
)
return enriched
def _sorted_versions(self, versions: Sequence[ModelVersionRecord]) -> List[ModelVersionRecord]:
ordered = sorted(versions, key=lambda version: (version.sort_index, version.version_id))
normalized: List[ModelVersionRecord] = []
@@ -1936,7 +2372,9 @@ class ModelUpdateService:
f"""
SELECT model_id, version_id, sort_index, name, base_model, released_at,
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
is_early_access, usage_control, paid_access, is_paid, file_count
is_early_access, usage_control, paid_access, is_paid, file_count,
gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz,
accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state
FROM model_update_versions
WHERE model_id IN ({placeholders})
ORDER BY model_id ASC, sort_index ASC, version_id ASC
@@ -1968,6 +2406,18 @@ class ModelUpdateService:
paid_access=row["paid_access"],
is_paid=bool(row["is_paid"]),
file_count=_normalize_int(row["file_count"]),
gate_lapsed_at=row["gate_lapsed_at"],
price_buzz=_normalize_int(row["price_buzz"]),
list_price_buzz=_normalize_int(row["list_price_buzz"]),
generation_price_buzz=_normalize_int(row["generation_price_buzz"]),
accepts_blue_buzz=bool(row["accepts_blue_buzz"]),
price_sale_ends_at=row["price_sale_ends_at"],
price_checked_at=(
float(row["price_checked_at"])
if row["price_checked_at"] is not None
else None
),
price_alert_state=bool(row["price_alert_state"]),
)
)
@@ -2030,8 +2480,10 @@ class ModelUpdateService:
INSERT INTO model_update_versions (
version_id, model_id, sort_index, name, base_model, released_at,
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
is_early_access, usage_control, paid_access, is_paid, file_count
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
is_early_access, usage_control, paid_access, is_paid, file_count,
gate_lapsed_at, price_buzz, list_price_buzz, generation_price_buzz,
accepts_blue_buzz, price_sale_ends_at, price_checked_at, price_alert_state
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
version.version_id,
@@ -2050,6 +2502,14 @@ class ModelUpdateService:
paid_access_value,
1 if version.is_paid else 0,
version.file_count,
version.gate_lapsed_at,
version.price_buzz,
version.list_price_buzz,
version.generation_price_buzz,
1 if version.accepts_blue_buzz else 0,
version.price_sale_ends_at,
version.price_checked_at,
1 if version.price_alert_state else 0,
),
)
conn.commit()
+5
View File
@@ -129,6 +129,11 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"model_card_footer_action": "replace_preview",
"show_version_on_card": True,
"version_grouping": "same_base",
# Buzz price tracking for paid/early-access versions. Opt-in because reading a
# price costs one extra (public) model-page request per gated model.
"price_tracking_enabled": False,
"price_alert_threshold_buzz": 0, # 0 = alert on "became free" only
"price_check_ttl_hours": 24,
"auto_organize_exclusions": [],
"metadata_refresh_skip_paths": [],
"skip_previously_downloaded_model_versions": False,
+257
View File
@@ -0,0 +1,257 @@
"""Extract buzz prices from a public CivitAI model page.
CivitAI's public REST API deliberately omits prices: ``paidAccess`` is trimmed to
``{permanent, endsAt}`` because "pricing belongs to the purchase flow" (see the
upstream ``toPublicPaidAccessDto``). The model *page*, however, ships the site's
own ``model.getById`` result inside its server-rendered Next.js payload, and that
payload carries the full ``paidAccess.terms`` — download price, generation price,
sale and the Blue Buzz flag.
Reading it is a plain anonymous page fetch: no API key, no internal endpoint, no
forged ``Origin``. It is still page data rather than a contract, so nothing in
this module may raise: an unrecognized shape degrades to ``None`` and the caller
keeps whatever price it already stored.
Shape this understands (verified against civitai.com and civitai.red)::
<script id="__NEXT_DATA__" type="application/json">
{"props": {"pageProps": {"trpcState": {"json": {"queries": [
{"queryKey": [["model", "getById"], {...}],
"state": {"data": {"modelVersions": [
{"id": 3379626, "paidAccess": {
"endsAt": null, "timeframeDays": null,
"terms": {"download": {"price": 5000},
"generation": {"price": 100, "trialLimit": 5}},
"sale": null}}
]}}}
]}}}}}
</script>
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any, Dict, Iterator, Mapping, Optional
from .paid_access import is_gate_active, normalize_paid_access
logger = logging.getLogger(__name__)
__all__ = ["parse_model_page_prices", "MAX_PAGE_BYTES"]
# Bound the payload before parsing: a model page is a few hundred KB, so anything
# far larger is not a page we want to hold in memory.
MAX_PAGE_BYTES = 8 * 1024 * 1024
_NEXT_DATA_RE = re.compile(
r'<script[^>]*id="__NEXT_DATA__"[^>]*>(.*?)</script>',
re.DOTALL,
)
# The procedure that carries per-version pricing on a model page.
_MODEL_QUERY_PATH = ("model", "getById")
def parse_model_page_prices(html: Any) -> Optional[Dict[int, Dict[str, Any]]]:
"""Return ``{version_id: price fields}`` for a model page, or None.
Only versions with an *active* gate appear: the page payload also carries
lapsed gates as tombstones (a version with ``endsAt`` in the past is freely
downloadable), and those must not be reported as priced.
Returns None when the page carries no usable payload at all (missing script
tag, malformed JSON, unexpected structure, challenge page).
"""
if not isinstance(html, str) or not html:
return None
if len(html) > MAX_PAGE_BYTES:
logger.debug("CivitAI model page too large to parse (%d bytes)", len(html))
return None
match = _NEXT_DATA_RE.search(html)
if match is None:
return None
try:
payload = json.loads(match.group(1))
except (TypeError, ValueError):
return None
versions = _find_model_versions(payload)
if versions is None:
return None
prices: Dict[int, Dict[str, Any]] = {}
for entry in versions:
if not isinstance(entry, Mapping):
continue
version_id = _coerce_int(entry.get("id"))
if version_id is None:
continue
price_fields = _price_fields(entry.get("paidAccess"))
if price_fields is not None:
prices[version_id] = price_fields
return prices
def _find_model_versions(payload: Any) -> Optional[list]:
"""Locate ``modelVersions`` inside the dehydrated tRPC state.
The query order is not stable (several procedures are dehydrated per page), so
the query is selected by key rather than position.
"""
if not isinstance(payload, Mapping):
return None
for state in _iter_trpc_states(payload):
queries = state.get("queries")
if not isinstance(queries, list):
continue
for query in queries:
if not isinstance(query, Mapping):
continue
if not _matches_model_query(query.get("queryKey")):
continue
data = (query.get("state") or {}).get("data")
if isinstance(data, Mapping):
versions = data.get("modelVersions")
if isinstance(versions, list):
return versions
return None
def _iter_trpc_states(payload: Mapping[str, Any]) -> Iterator[Mapping[str, Any]]:
"""Yield the dehydrated tRPC state objects found under ``props.pageProps``."""
props = payload.get("props")
if not isinstance(props, Mapping):
return
page_props = props.get("pageProps")
if not isinstance(page_props, Mapping):
return
candidates = [page_props.get("trpcState")]
# Older/other Next.js builds keep the state one level deeper.
dehydrated = page_props.get("dehydratedState")
if dehydrated is not None:
candidates.append(dehydrated)
for candidate in candidates:
if not isinstance(candidate, Mapping):
continue
for key in ("json", "superjson", "dehydratedState"):
nested = candidate.get(key)
if isinstance(nested, Mapping):
yield nested
yield candidate
def _matches_model_query(query_key: Any) -> bool:
if not isinstance(query_key, (list, tuple)) or not query_key:
return False
first = query_key[0]
if not isinstance(first, (list, tuple)) or len(first) < 2:
return False
return str(first[0]) == _MODEL_QUERY_PATH[0] and str(first[1]) == _MODEL_QUERY_PATH[1]
def _price_fields(raw_paid_access: Any) -> Optional[Dict[str, Any]]:
"""Build the price fields for one version, or None when it is not gated."""
dto = _coerce_dto(raw_paid_access)
if dto is None:
return None
info = normalize_paid_access(dto)
if not info or not is_gate_active(info):
return None
terms = dto.get("terms")
if not isinstance(terms, Mapping):
terms = {}
list_price = _download_price(terms)
effective_price = list_price
sale = dto.get("sale")
sale_ends_at = None
if isinstance(sale, Mapping):
sale_ends_at = _coerce_str(sale.get("endsAt"))
buyer_terms = sale.get("buyerTerms")
if isinstance(buyer_terms, Mapping):
sale_price = _download_price(buyer_terms)
if sale_price is not None:
effective_price = sale_price
return {
"price_buzz": effective_price,
"list_price_buzz": list_price,
"generation_price_buzz": _generation_price(terms),
"accepts_blue_buzz": bool(terms.get("acceptsBlueBuzz")),
"price_sale_ends_at": sale_ends_at,
}
def _coerce_dto(value: Any) -> Optional[Mapping[str, Any]]:
if isinstance(value, str):
try:
value = json.loads(value)
except (TypeError, ValueError):
return None
return value if isinstance(value, Mapping) else None
def _download_price(terms: Mapping[str, Any]) -> Optional[int]:
download = terms.get("download")
if not isinstance(download, Mapping):
return None
return _coerce_price(download.get("price"))
def _generation_price(terms: Mapping[str, Any]) -> Optional[int]:
generation = terms.get("generation")
if not isinstance(generation, Mapping):
return None
# `{free: true}` has no price of its own, and a paid tier may omit `price` to
# fall back to the download price — the download price is already captured
# separately, so only an explicit number is reported here.
return _coerce_price(generation.get("price"))
def _coerce_price(value: Any) -> Optional[int]:
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
price = int(value)
return price if price >= 0 else None
if isinstance(value, str):
try:
price = int(float(value.strip()))
except (TypeError, ValueError):
return None
return price if price >= 0 else None
return None
def _coerce_int(value: Any) -> Optional[int]:
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, str):
try:
return int(value.strip())
except (TypeError, ValueError):
return None
return None
def _coerce_str(value: Any) -> Optional[str]:
if isinstance(value, str) and value.strip():
return value.strip()
return None