feat: support gated/private Hugging Face repos via access token

Add a huggingface_api_key setting (Settings UI, HF_TOKEN /
HUGGING_FACE_HUB_TOKEN env override) and attach it as a Bearer token
to Hugging Face file listing, model card fetching and downloads, so
gated and private repositories can be downloaded once the user has
accepted the repo terms.

- fetch_json/fetch_text accept custom headers; ModelSource gains an
  auth_headers() hook so handlers stay platform-agnostic
- 401/403 from the tree API now explain how to fix (configure token /
  accept gated terms)
- aria2 pre-resolves huggingface.co redirects and strips credentials
  before handing the signed CDN URL to aria2, mirroring the CivitAI
  handling so the token never leaks to the CDN
- settings API exposes huggingface_api_key_set only; the raw key joins
  _NO_SYNC_KEYS
This commit is contained in:
Will Miao
2026-09-25 18:44:06 +08:00
parent 067e605e75
commit 8b7ba59263
23 changed files with 446 additions and 23 deletions
+21 -4
View File
@@ -193,7 +193,9 @@ def is_valid_source_id(source_id: str) -> bool:
)
async def fetch_text(url: str, *, timeout: int = HTTP_TIMEOUT) -> str:
async def fetch_text(
url: str, *, timeout: int = HTTP_TIMEOUT, headers: Optional[Dict[str, str]] = None
) -> str:
"""Fetch *url* and return its body as text, or ``""`` on any failure.
Network problems are expected (offline installs, rate limits, dead
@@ -202,8 +204,11 @@ async def fetch_text(url: str, *, timeout: int = HTTP_TIMEOUT) -> str:
"""
try:
request_headers = {"User-Agent": USER_AGENT}
if headers:
request_headers.update(headers)
async with aiohttp.ClientSession(
headers={"User-Agent": USER_AGENT},
headers=request_headers,
timeout=aiohttp.ClientTimeout(total=timeout),
) as session:
async with session.get(url) as resp:
@@ -216,7 +221,7 @@ async def fetch_text(url: str, *, timeout: int = HTTP_TIMEOUT) -> str:
async def fetch_json(
url: str, *, timeout: int = HTTP_TIMEOUT
url: str, *, timeout: int = HTTP_TIMEOUT, headers: Optional[Dict[str, str]] = None
) -> tuple[int, Any]:
"""Fetch *url* and return ``(status, parsed_body)``.
@@ -227,8 +232,11 @@ async def fetch_json(
"""
try:
request_headers = {"User-Agent": USER_AGENT}
if headers:
request_headers.update(headers)
async with aiohttp.ClientSession(
headers={"User-Agent": USER_AGENT},
headers=request_headers,
timeout=aiohttp.ClientTimeout(total=timeout),
) as session:
async with session.get(url) as resp:
@@ -381,6 +389,15 @@ class ModelSource:
return []
def auth_headers(self) -> Dict[str, str]:
"""Extra request headers this site needs for API and file downloads.
Empty by default; sites with gated/private content (Hugging Face)
override it to attach the user's access token when one is configured.
"""
return {}
def file_download_url(
self, source_id: str, filename: str, revision: str = ""
) -> str:
+37 -2
View File
@@ -27,6 +27,18 @@ _STRICT_URL_PATTERN = re.compile(
)
def _hf_token() -> str:
"""Return the configured Hugging Face access token, or ``""``."""
try:
from ..settings_manager import get_settings_manager
token = get_settings_manager().get("huggingface_api_key", "")
except Exception: # pragma: no cover - settings must never break downloads
return ""
return token.strip() if isinstance(token, str) else ""
class HuggingFaceSource(ModelSource):
"""Hugging Face Hub (``huggingface.co``)."""
@@ -45,12 +57,20 @@ class HuggingFaceSource(ModelSource):
def asset_base_url(self, source_id: str, revision: str = "") -> str:
return f"https://huggingface.co/{source_id}/resolve/{self.resolve_revision(revision)}"
def auth_headers(self) -> dict[str, str]:
"""Bearer header for gated/private repositories, when a token is set."""
token = _hf_token()
return {"Authorization": f"Bearer {token}"} if token else {}
async def fetch_model_card(self, source_id: str) -> str:
"""Fetch ``README.md`` from Hugging Face (tries ``main``, then ``master``)."""
headers = self.auth_headers()
for branch in ("main", "master"):
text = await fetch_text(
f"https://huggingface.co/{source_id}/raw/{branch}/README.md"
f"https://huggingface.co/{source_id}/raw/{branch}/README.md",
headers=headers,
)
if text:
return text
@@ -67,11 +87,26 @@ class HuggingFaceSource(ModelSource):
revision = self.resolve_revision(revision)
status, payload = await fetch_json(
f"https://huggingface.co/api/models/{source_id}/tree/{revision}"
f"https://huggingface.co/api/models/{source_id}/tree/{revision}",
headers=self.auth_headers(),
)
if status == 404:
raise ModelSourceError(f"Repository '{source_id}' not found", status=404)
if status in (401, 403):
if _hf_token():
raise ModelSourceError(
f"Access to '{source_id}' was denied (HTTP {status}). For a gated "
"repository you must accept its terms on the Hugging Face page, "
"and the configured token needs read permission for it.",
status=403,
)
raise ModelSourceError(
f"'{source_id}' requires a Hugging Face access token (gated or "
"private repository). Configure one in Settings → Hugging Face "
"Access Token, and accept the repository's terms on its page.",
status=401,
)
if status != 200 or not isinstance(payload, list):
raise ModelSourceError(
f"Hugging Face API error while listing '{source_id}' (HTTP {status})"