From 1de0a5324159eec0ebd82042b16b28065d23e422 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sun, 26 Jul 2026 21:46:55 +0800 Subject: [PATCH] feat(grouping): version-group library cards by HuggingFace repo for non-Civitai sources (#1040) --- locales/de.json | 1 + locales/en.json | 1 + locales/es.json | 1 + locales/fr.json | 1 + locales/he.json | 1 + locales/ja.json | 1 + locales/ko.json | 1 + locales/ru.json | 1 + locales/zh-CN.json | 1 + locales/zh-TW.json | 1 + py/routes/handlers/model_handlers.py | 4 +- py/services/base_model_service.py | 54 ++++++++++++--- static/js/components/shared/ModelCard.js | 6 ++ static/js/components/shared/ModelModal.js | 9 ++- .../js/components/shared/ModelVersionsTab.js | 20 ++++++ tests/services/test_base_model_service.py | 66 +++++++++++++++++++ 16 files changed, 158 insertions(+), 11 deletions(-) diff --git a/locales/de.json b/locales/de.json index f7ddd226..96b57acb 100644 --- a/locales/de.json +++ b/locales/de.json @@ -1548,6 +1548,7 @@ "empty": "Noch keine Versionshistorie für dieses Modell vorhanden.", "error": "Versionen konnten nicht geladen werden.", "missingModelId": "Für dieses Modell ist keine Civitai-Model-ID vorhanden.", + "hfGroupInfo": "Dies ist eine HuggingFace-Modellgruppe. Öffnen Sie die Bibliothek, um alle Versionen im Raster zu sehen.", "confirm": { "delete": "Diese Version aus Ihrer Bibliothek löschen?" }, diff --git a/locales/en.json b/locales/en.json index a487d27a..f94b2a66 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1548,6 +1548,7 @@ "empty": "No version history available for this model yet.", "error": "Failed to load versions.", "missingModelId": "This model is missing a Civitai model id.", + "hfGroupInfo": "This is a HuggingFace model group. Open the library to see all versions in the grid.", "confirm": { "delete": "Delete this version from your library?" }, diff --git a/locales/es.json b/locales/es.json index b588be0c..465f00d4 100644 --- a/locales/es.json +++ b/locales/es.json @@ -1548,6 +1548,7 @@ "empty": "Aún no hay historial de versiones para este modelo.", "error": "No se pudieron cargar las versiones.", "missingModelId": "Este modelo no tiene un ID de modelo de Civitai.", + "hfGroupInfo": "Este es un grupo de modelos de HuggingFace. Abra la biblioteca para ver todas las versiones en la cuadrícula.", "confirm": { "delete": "¿Eliminar esta versión de tu biblioteca?" }, diff --git a/locales/fr.json b/locales/fr.json index 58a24794..e12b200f 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -1548,6 +1548,7 @@ "empty": "Aucun historique de versions n'est disponible pour ce modèle pour le moment.", "error": "Échec du chargement des versions.", "missingModelId": "Ce modèle ne possède pas d'identifiant de modèle Civitai.", + "hfGroupInfo": "Ceci est un groupe de modèles HuggingFace. Ouvrez la bibliothèque pour voir toutes les versions dans la grille.", "confirm": { "delete": "Supprimer cette version de votre bibliothèque ?" }, diff --git a/locales/he.json b/locales/he.json index 9b22296b..f9c6c3d0 100644 --- a/locales/he.json +++ b/locales/he.json @@ -1548,6 +1548,7 @@ "empty": "אין עדיין היסטוריית גרסאות למודל זה.", "error": "טעינת הגרסאות נכשלה.", "missingModelId": "למודל זה אין מזהה מודל של Civitai.", + "hfGroupInfo": "זוהי קבוצת דגמים של HuggingFace. פתח את הספרייה כדי לראות את כל הגרסאות ברשת.", "confirm": { "delete": "למחוק גרסה זו מהספרייה שלך?" }, diff --git a/locales/ja.json b/locales/ja.json index 14aae4df..eb3a47a2 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -1548,6 +1548,7 @@ "empty": "このモデルにはまだバージョン履歴がありません。", "error": "バージョンの読み込みに失敗しました。", "missingModelId": "このモデルにはCivitaiのモデルIDがありません。", + "hfGroupInfo": "これは HuggingFace モデルグループです。ライブラリを開いてグリッドですべてのバージョンを表示してください。", "confirm": { "delete": "このバージョンをライブラリから削除しますか？" }, diff --git a/locales/ko.json b/locales/ko.json index 44a4dd25..b966dc3f 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -1548,6 +1548,7 @@ "empty": "이 모델에는 아직 버전 기록이 없습니다.", "error": "버전을 불러오지 못했습니다.", "missingModelId": "이 모델에는 Civitai 모델 ID가 없습니다.", + "hfGroupInfo": "HuggingFace 모델 그룹입니다. 라이브러리를 열어 그리드에서 모든 버전을 확인하세요.", "confirm": { "delete": "이 버전을 라이브러리에서 삭제하시겠습니까?" }, diff --git a/locales/ru.json b/locales/ru.json index a1dddf90..d90a20cc 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -1548,6 +1548,7 @@ "empty": "Для этой модели пока нет истории версий.", "error": "Не удалось загрузить версии.", "missingModelId": "У этой модели отсутствует идентификатор модели Civitai.", + "hfGroupInfo": "Это группа моделей HuggingFace. Откройте библиотеку, чтобы увидеть все версии в сетке.", "confirm": { "delete": "Удалить эту версию из библиотеки?" }, diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 7b7cd999..9481f7c2 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -1548,6 +1548,7 @@ "empty": "该模型还没有版本历史。", "error": "加载版本失败。", "missingModelId": "该模型缺少 Civitai 模型 ID。", + "hfGroupInfo": "这是一个 HuggingFace 模型组。打开库页面即可在网格中查看所有版本。", "confirm": { "delete": "从库中删除此版本？" }, diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 00584cb8..347c9369 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -1548,6 +1548,7 @@ "empty": "此模型尚無版本歷史。", "error": "載入版本失敗。", "missingModelId": "此模型缺少 Civitai 模型 ID。", + "hfGroupInfo": "這是一個 HuggingFace 模型組。打開庫頁面即可在網格中查看所有版本。", "confirm": { "delete": "要從庫中刪除此版本嗎？" }, diff --git a/py/routes/handlers/model_handlers.py b/py/routes/handlers/model_handlers.py index 180a5068..5ecfab05 100644 --- a/py/routes/handlers/model_handlers.py +++ b/py/routes/handlers/model_handlers.py @@ -394,12 +394,14 @@ class ModelListingHandler: ) # View-local-versions filter: show all local versions of a specific model + # Accepts either a CivitAI modelId (int) or a HF group key like "hf:user/repo" civitai_model_id = request.query.get("civitai_model_id") if civitai_model_id is not None: try: civitai_model_id = int(civitai_model_id) except (TypeError, ValueError): - civitai_model_id = None + # Keep as string — could be an HF group key (e.g. "hf:user/repo") + pass return { "page": page, diff --git a/py/services/base_model_service.py b/py/services/base_model_service.py index 47bdb692..f85483eb 100644 --- a/py/services/base_model_service.py +++ b/py/services/base_model_service.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod import asyncio import re -from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Type, Union, TYPE_CHECKING import logging import os import time @@ -109,12 +109,15 @@ class BaseModelService(ABC): if civitai_model_id is not None: sorted_data = [ item for item in sorted_data - if self._extract_model_id(item) == civitai_model_id + if self._extract_group_key(item) == civitai_model_id ] # VLM mode: always sort by version ID descending (newest version first), # regardless of the current sort_by preference. + # Fall back to modified timestamp for non-CivitAI sources. sorted_data.sort( - key=lambda x: self._extract_version_id(x) or 0, + key=lambda x: self._extract_version_id(x) + or x.get("modified", 0) + or 0, reverse=True, ) @@ -129,18 +132,21 @@ class BaseModelService(ABC): ufs = self.settings.get("version_grouping", "same_base") group_by_base = ufs == "same_base" - dedup_map = {} # (modelId [,base_model]) -> (item, version_id) + dedup_map = {} # (modelId [,base_model]) -> (item, version_or_modified) version_counter = {} # same-key -> count standalone = [] for item in sorted_data: - mid = self._extract_model_id(item) + mid = self._extract_group_key(item) if mid is None: standalone.append(item) continue key = (mid, item.get("base_model") or "") if group_by_base else mid # Count all versions per key version_counter[key] = version_counter.get(key, 0) + 1 - vid = self._extract_version_id(item) or 0 + # Prefer CivitAI version_id; fall back to modified timestamp + vid = self._extract_version_id(item) + if vid is None: + vid = item.get("modified", 0) or 0 if key not in dedup_map or vid > dedup_map[key][1]: dedup_map[key] = (item, vid) # Attach version_count to each surviving grouped item (shallow copy @@ -174,16 +180,19 @@ class BaseModelService(ABC): model_groups: Dict[Any, List[Dict]] = {} ungrouped_standalone: List[Dict] = [] for item in sorted_data: - mid = self._extract_model_id(item) + mid = self._extract_group_key(item) if mid is None: ungrouped_standalone.append(item) continue key = (mid, item.get("base_model") or "") if group_by_base else mid model_groups.setdefault(key, []).append(item) - # Sort versions within each group by version id descending + # Sort versions within each group by version id (descending); + # fall back to modified timestamp for non-CivitAI sources. for items in model_groups.values(): items.sort( - key=lambda x: self._extract_version_id(x) or 0, + key=lambda x: self._extract_version_id(x) + or x.get("modified", 0) + or 0, reverse=True, ) # Sort groups by version count @@ -697,6 +706,33 @@ class BaseModelService(ABC): return annotated + @staticmethod + def _extract_hf_group_key(item: Dict) -> Optional[str]: + """Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None.""" + hf_url = item.get("hf_url") if isinstance(item, dict) else None + if not hf_url or not isinstance(hf_url, str): + return None + m = re.match( + r"https?://huggingface\.co/([^/]+/[^/]+)", hf_url.strip() + ) + if not m: + return None + return f"hf:{m.group(1)}" + + @staticmethod + def _extract_group_key(item: Dict) -> Union[int, str, None]: + """Return the group identity key: CivitAI modelId (int) or HF repo (str). + + Preference order: + 1. CivitAI ``modelId`` (int) + 2. HF repo identity ``hf:{owner}/{repo}`` (str) + 3. ``None`` (no known grouping source) + """ + mid = BaseModelService._extract_model_id(item) + if mid is not None: + return mid + return BaseModelService._extract_hf_group_key(item) + @staticmethod def _extract_model_id(item: Dict) -> Optional[int]: civitai = item.get("civitai") if isinstance(item, dict) else None diff --git a/static/js/components/shared/ModelCard.js b/static/js/components/shared/ModelCard.js index 5288f7d3..642b41e5 100644 --- a/static/js/components/shared/ModelCard.js +++ b/static/js/components/shared/ModelCard.js @@ -489,6 +489,12 @@ export function createModelCard(model, modelType) { const modelId = civitaiData?.modelId ?? civitaiData?.model_id; if (modelId !== undefined && modelId !== null && modelId !== '') { card.dataset.modelId = modelId; + } else if (model.hf_url) { + // For HF-only models, derive a group key from hf_url for version grouping + const match = model.hf_url.match(/https?:\/\/huggingface\.co\/([^/]+\/[^/]+)/); + if (match) { + card.dataset.modelId = 'hf:' + match[1]; + } } // LoRA specific data diff --git a/static/js/components/shared/ModelModal.js b/static/js/components/shared/ModelModal.js index d378c9c6..107e09be 100644 --- a/static/js/components/shared/ModelModal.js +++ b/static/js/components/shared/ModelModal.js @@ -473,7 +473,14 @@ export async function showModelModal(model, modelType) { const loadingExamplesText = translate('modals.model.loading.examples', {}, 'Loading examples...'); const loadingVersionsText = translate('modals.model.loading.versions', {}, 'Loading versions...'); - const civitaiModelId = modelWithFullData.civitai?.modelId || ''; + // Use CivitAI modelId, or derive HF group key for HF-only models + let civitaiModelId = modelWithFullData.civitai?.modelId || ''; + if (!civitaiModelId && modelWithFullData.hf_url) { + const match = modelWithFullData.hf_url.match(/https?:\/\/huggingface\.co\/([^/]+\/[^/]+)/); + if (match) { + civitaiModelId = 'hf:' + match[1]; + } + } const civitaiVersionId = modelWithFullData.civitai?.id || ''; const navAriaLabel = translate('modals.model.navigation.label', {}, 'Model navigation'); const previousTitle = translate('modals.model.navigation.previousWithShortcut', {}, 'Previous model (←)'); diff --git a/static/js/components/shared/ModelVersionsTab.js b/static/js/components/shared/ModelVersionsTab.js index 135ca630..98ef3c1a 100644 --- a/static/js/components/shared/ModelVersionsTab.js +++ b/static/js/components/shared/ModelVersionsTab.js @@ -950,6 +950,26 @@ export function initVersionsTab({ renderErrorState(container, translate('modals.model.versions.missingModelId', {}, 'This model is missing a Civitai model id.')); return; } + // HF group keys (e.g. "hf:user/repo") are not real CivitAI model IDs — + // skip the remote API call and show a helpful message instead. + const isHfGroupKey = typeof modelId === 'string' && modelId.startsWith('hf:'); + if (isHfGroupKey) { + controller.isLoading = false; + controller.hasLoaded = true; + controller.record = null; + const hfMsg = translate( + 'modals.model.versions.hfGroupInfo', + {}, + 'This is a HuggingFace model group. Open the library to see all versions in the grid.' + ); + container.innerHTML = ` +
+ +

${escapeHtml(hfMsg)}

+
+ `; + return; + } if (controller.hasLoaded && !forceRefresh) { return; } diff --git a/tests/services/test_base_model_service.py b/tests/services/test_base_model_service.py index a9e2a6b3..24806d12 100644 --- a/tests/services/test_base_model_service.py +++ b/tests/services/test_base_model_service.py @@ -1252,3 +1252,69 @@ async def test_get_model_civitai_url_falls_back_when_host_setting_is_not_a_strin "model_id": "123", "version_id": "456", } + + +class TestHfGroupKey: + """Tests for _extract_hf_group_key and _extract_group_key.""" + + # --- _extract_hf_group_key --- + + def test_hf_group_key_valid_url(self): + """Standard HF URL returns hf:user/repo.""" + item = {"hf_url": "https://huggingface.co/unsloth/qwen-edit"} + assert BaseModelService._extract_hf_group_key(item) == "hf:unsloth/qwen-edit" + + def test_hf_group_key_url_with_subpath(self): + """URL with subpath still extracts just owner/repo.""" + item = {"hf_url": "https://huggingface.co/user/repo/resolve/main/file.safetensors"} + assert BaseModelService._extract_hf_group_key(item) == "hf:user/repo" + + def test_hf_group_key_empty_url(self): + """Empty hf_url returns None.""" + assert BaseModelService._extract_hf_group_key({"hf_url": ""}) is None + + def test_hf_group_key_no_url(self): + """Missing hf_url key returns None.""" + assert BaseModelService._extract_hf_group_key({}) is None + + def test_hf_group_key_none_url(self): + """None hf_url returns None.""" + assert BaseModelService._extract_hf_group_key({"hf_url": None}) is None + + def test_hf_group_key_invalid_url(self): + """Malformed HF URL returns None.""" + assert BaseModelService._extract_hf_group_key({"hf_url": "not-a-url"}) is None + assert BaseModelService._extract_hf_group_key({"hf_url": "https://example.com"}) is None + + # --- _extract_group_key --- + + def test_group_key_civitai_only(self): + """CivitAI modelId returned as int.""" + item = {"civitai": {"modelId": 123}} + assert BaseModelService._extract_group_key(item) == 123 + + def test_group_key_hf_only(self): + """HF-only item returns hf:user/repo string.""" + item = {"hf_url": "https://huggingface.co/user/repo"} + assert BaseModelService._extract_group_key(item) == "hf:user/repo" + + def test_group_key_civitai_preferred(self): + """CivitAI modelId takes precedence over hf_url.""" + item = { + "civitai": {"modelId": 456}, + "hf_url": "https://huggingface.co/other/repo", + } + assert BaseModelService._extract_group_key(item) == 456 + + def test_group_key_neither(self): + """No CivitAI or HF returns None.""" + assert BaseModelService._extract_group_key({}) is None + assert BaseModelService._extract_group_key({"some": "data"}) is None + + def test_group_key_civitai_none_model_id(self): + """civitai.modelId=None falls through to HF.""" + item = { + "civitai": {"modelId": None}, + "hf_url": "https://huggingface.co/user/repo", + } + assert BaseModelService._extract_group_key(item) == "hf:user/repo"