fix: stop grouping HF/ModelScope models by repository

A repository is not a model identity: collection repos on Hugging Face
and ModelScope host many unrelated models, which were wrongly shown as
versions of each other.

- Hugging Face models no longer auto-group (the Hub exposes no
  site-native model id)
- ModelScope models group by the site's native published-model id
  (MuseInfo modelVersion.modelId), extracted during enrichment and
  persisted on the sidecar as source_model_id/source_version_id;
  unenriched models stay standalone instead of collapsing a whole repo
  into one group
- TensorArt grouping unchanged (its URL id is already model-level)
- Frontend group-key derivation mirrors the new backend semantics
This commit is contained in:
Will Miao
2026-09-25 23:29:30 +08:00
parent 8a80f82d93
commit c48feeddb6
17 changed files with 357 additions and 78 deletions
+29 -4
View File
@@ -25,7 +25,7 @@ import logging
import os
import re
from dataclasses import dataclass, field
from typing import Any, Dict, Iterable, Optional
from typing import Any, Dict, Iterable, Mapping, Optional
import aiohttp
@@ -123,6 +123,20 @@ class ModelCardContext:
trigger_words: list[str] = field(default_factory=list)
"""Trigger words the site records for the requested model file."""
source_model_id: str = ""
"""Site-native id of the *published model* the requested file belongs to.
Sites whose repository is not a model identity publish a separate,
stable id per model (ModelScope's ``modelVersion.modelId`` — identical
across every version of one published model, different between the
models of a collection repository). It is the version-grouping key,
persisted on the sidecar as ``source_model_id``.
"""
source_version_id: str = ""
"""Site-native id of the published version the requested file belongs to
(ModelScope's ``modelVersion.id``), persisted as ``source_version_id``."""
def is_empty(self) -> bool:
"""Return ``True`` when the site contributed nothing extra."""
@@ -139,6 +153,8 @@ class ModelCardContext:
self.official_tags,
self.example_images,
self.trigger_words,
self.source_model_id,
self.source_version_id,
)
)
@@ -329,11 +345,20 @@ class ModelSource:
return ""
def group_key(self, source_id: str) -> str:
"""Return the version-group key for *source_id*."""
def group_key(self, ref: SourceRef, item: Mapping[str, Any]) -> Optional[str]:
"""Return the version-group key for the model described by *item*.
The default groups by source id (``{prefix}:{owner}/{repo}``), which
is only correct when the source id already identifies a single
published model. Sources whose repository hosts many unrelated
models override this: they either derive the key from a site-native
model identity recorded in *item* (ModelScope's ``source_model_id``)
or return ``None`` when the platform has no reliable model identity
at all (Hugging Face), leaving the model ungrouped.
"""
prefix = GROUP_PREFIXES.get(self.platform, self.platform)
return f"{prefix}:{source_id}"
return f"{prefix}:{ref.source_id}"
async def fetch_model_card(self, source_id: str) -> str:
"""Fetch the raw model card (README) markdown for *source_id*."""
+12
View File
@@ -4,10 +4,12 @@ from __future__ import annotations
import logging
import re
from typing import Any, Mapping, Optional
from .base import (
ModelSource,
ModelSourceError,
SourceRef,
fetch_json,
fetch_text,
filter_weight_files,
@@ -54,6 +56,16 @@ class HuggingFaceSource(ModelSource):
def canonical_url(self, source_id: str) -> str:
return f"https://huggingface.co/{source_id}"
def group_key(self, ref: SourceRef, item: Mapping[str, Any]) -> Optional[str]:
"""Hugging Face models never auto-group.
A repository is not a model identity — collection repos host many
unrelated models — and the Hub exposes no site-native published-model
id, so there is no reliable key to group by.
"""
return None
def asset_base_url(self, source_id: str, revision: str = "") -> str:
return f"https://huggingface.co/{source_id}/resolve/{self.resolve_revision(revision)}"
+48 -1
View File
@@ -44,12 +44,15 @@ import json
import logging
import os
import re
from typing import TYPE_CHECKING, Any, Iterable, Optional
from typing import TYPE_CHECKING, Any, Iterable, Mapping, Optional
from .base import (
GROUP_PREFIXES,
ModelCardContext,
ModelSource,
ModelSourceError,
SourceRef,
clean_source_url,
fetch_json,
fetch_text,
filter_weight_files,
@@ -111,6 +114,22 @@ class ModelScopeSource(ModelSource):
def canonical_url(self, source_id: str) -> str:
return f"{self.base_url}/models/{source_id}"
def group_key(self, ref: SourceRef, item: Mapping[str, Any]) -> Optional[str]:
"""Group by ModelScope's published-model id, never by repository.
A collection repository hosts many unrelated published models, so
the repo id is not a version-group identity. Only models whose
metadata carries the site-native ``source_model_id`` (recorded at
enrichment time from ``MuseInfo.versions[].modelVersion.modelId``)
group together; unenriched models stay standalone.
"""
model_id = clean_source_url(item.get("source_model_id"))
if not model_id:
return None
prefix = GROUP_PREFIXES.get(self.platform, self.platform)
return f"{prefix}:{model_id}"
def asset_base_url(self, source_id: str, revision: str = "") -> str:
return (
f"{self.base_url}/models/{source_id}/resolve/"
@@ -350,9 +369,37 @@ def _build_card_context(
context.version_name = _version_label(versions)
context.example_images = _cover_image_urls(versions)
context.trigger_words = _version_trigger_words(versions)
context.source_model_id, context.source_version_id = _version_identity(
versions
)
return context
def _version_identity(versions: list[dict[str, Any]]) -> tuple[str, str]:
"""Return the site-native ``(model id, version id)`` of the first match.
``modelVersion.modelId`` is identical across every version of one
published model and differs between the models of a collection
repository, which makes it the version-grouping identity;
``modelVersion.id`` identifies the version itself. Both are ints in
the payload and are stored as strings.
"""
for version in versions:
model_version = version.get("modelVersion")
if not isinstance(model_version, dict):
continue
model_id = model_version.get("modelId")
version_id = model_version.get("id")
if model_id is None and version_id is None:
continue
return (
str(model_id) if model_id is not None else "",
str(version_id) if version_id is not None else "",
)
return "", ""
def _base_model_aliases(data: dict[str, Any]) -> list[str]:
"""Return the site's own names for the base model.
+8 -3
View File
@@ -196,8 +196,13 @@ def get_source_platform(item: Mapping[str, Any]) -> str:
def source_group_key(item: Mapping[str, Any]) -> Optional[str]:
"""Return the version-group key for *item*, or ``None``.
Hugging Face keeps the historical ``hf:{owner}/{repo}`` shape; other
platforms use their own short prefix (see :data:`GROUP_PREFIXES`).
Only sources with a site-native model identity yield a key: TensorArt
groups by its numeric model id (``ta:<id>``) and ModelScope by the
published-model id recorded at enrichment time (``ms:<id>`` /
``msai:<id>``). Hugging Face yields no key at all — a repository is
not a model identity — and unenriched ModelScope models stay
standalone rather than collapsing a whole collection repository into
one group.
"""
ref = resolve_source_ref(item)
@@ -206,7 +211,7 @@ def source_group_key(item: Mapping[str, Any]) -> Optional[str]:
source = get_source(ref.platform)
if source is None:
return None
return source.group_key(ref.source_id)
return source.group_key(ref, item)
__all__ = [