fix(modelscope): identify a model file by hash before filename

A file was matched to its published version by comparing basenames against
each version's `stats.fileList`.  Renaming the weights — routine once a
model is filed away, and the reason the scanner records a sha256 at all —
made the match fail silently, so the file lost its example images and its
preview with no indication why.

The detail payload's `ModelInfos.safetensor.files[]` carries a real sha256
per published file, and the local hash is already on disk, so match on that
first: it is the one identifier a rename cannot invalidate.  Exact basename
and `showName` matching remain as fallbacks, and an unknown hash falls
through to them rather than giving up, so a re-encoded file still resolves.

Verified against the live repository: a renamed `c1-st1000` file with its
hash yields the c1-st1000 image, the same rename without a hash yields
nothing, and supplying c1-st2000's hash resolves to the c1-st2000 image even
when the filename claims otherwise.
This commit is contained in:
Will Miao
2026-09-14 21:27:09 +08:00
parent e9e9ee20c6
commit 0f160e157f
5 changed files with 255 additions and 26 deletions
+6 -2
View File
@@ -474,9 +474,13 @@ class AgentService:
cache.readmes[cache_key] = readme
# Sites such as ModelScope keep part of the model card outside the
# README (author summary, curated tags, per-file example images).
# README (author summary, curated tags, per-file example images). The
# recorded hash identifies the file even after the user renames it.
card_context = await source.fetch_model_card_context(
ref.source_id, os.path.basename(model_path), cache=cache,
ref.source_id,
os.path.basename(model_path),
sha256=(metadata.get("sha256") or "").strip(),
cache=cache,
)
variables["source_description"] = card_context.description
variables["source_base_model"] = card_context.base_model
+9 -4
View File
@@ -307,19 +307,24 @@ class ModelSource:
source_id: str,
filename: str = "",
*,
sha256: str = "",
cache: Optional["ModelSourceCache"] = None,
) -> ModelCardContext:
"""Return the card extras the site keeps outside the README.
*filename* is the model file's basename (no directory) and selects
the right entry when a repository holds several models. Sites whose
model card is fully described by :meth:`fetch_model_card` need no
override and inherit this empty context.
*filename* is the model file's basename (no directory) and *sha256*
its content hash; between them they select the right entry when a
repository holds several models. A site that records per-file hashes
should prefer *sha256*, because it is the only identifier that
survives the user renaming the weights.
*cache* is an optional per-run memo (see :class:`ModelSourceCache`)
that lets a provider avoid re-fetching repository-wide data for every
file in a collection repository.
Sites whose model card is fully described by :meth:`fetch_model_card`
need no override and inherit this empty context.
Implementations must never raise: enrichment treats a missing
context as "the site had nothing extra to say".
"""
+77 -16
View File
@@ -117,6 +117,7 @@ class ModelScopeSource(ModelSource):
source_id: str,
filename: str = "",
*,
sha256: str = "",
cache: Optional["ModelSourceCache"] = None,
) -> ModelCardContext:
"""Read the model-detail API that backs the ModelScope model page.
@@ -128,10 +129,11 @@ class ModelScopeSource(ModelSource):
no further description") and put everything useful in ``Description``,
so enrichment that reads only the README comes back nearly empty.
Example images are matched to *filename* through each version's
``stats.fileList``, which means the images returned belong to the
exact ``.safetensors`` being enriched — essential for collection
repositories, where every checkpoint has its own sample image.
The wanted file is identified by its sha256 when the caller knows it
and by *filename* otherwise; see :func:`_matching_versions`. The
images and trigger words returned belong to that exact
``.safetensors`` — essential for collection repositories, where every
checkpoint has its own sample image.
The detail payload describes the whole repository and is therefore
shared across every file in it, so it is read through *cache* when the
@@ -141,7 +143,7 @@ class ModelScopeSource(ModelSource):
data = await self._fetch_detail(source_id, cache=cache)
if data is None:
return ModelCardContext()
return _build_card_context(data, filename)
return _build_card_context(data, filename, sha256)
async def _fetch_detail(
self,
@@ -245,7 +247,9 @@ def _first_string(value: Any) -> str:
return ""
def _build_card_context(data: dict[str, Any], filename: str) -> ModelCardContext:
def _build_card_context(
data: dict[str, Any], filename: str, sha256: str = ""
) -> ModelCardContext:
"""Turn a model-detail payload into a :class:`ModelCardContext`.
Separated from the HTTP fetch so the repository-wide payload can be cached
@@ -260,7 +264,12 @@ def _build_card_context(data: dict[str, Any], filename: str) -> ModelCardContext
official_tags=_official_tags(data.get("OfficialTags")),
)
versions = _matching_versions(data.get("MuseInfo"), filename)
versions = _matching_versions(
data.get("MuseInfo"),
filename,
digests=_file_digests(data),
sha256=sha256,
)
if versions:
context.example_images = _cover_image_urls(versions)
context.trigger_words = _version_trigger_words(versions)
@@ -350,16 +359,56 @@ def _version_show_name(version: dict[str, Any]) -> str:
return _clean_text(model_version.get("showName")).lower()
def _matching_versions(muse_info: Any, filename: str) -> list[dict[str, Any]]:
"""Return the ``versions`` entries that publish *filename*.
def _file_digests(data: dict[str, Any]) -> dict[str, str]:
"""Return ``basename -> sha256`` for every published weight file.
Matching is by exact basename first, then by the version's ``showName``
appearing in the file stem (which absorbs the naming drift ModelScope
sometimes applies to uploaded weights). All matches are returned so a
file re-published across several versions contributes all of its
example images. With no *filename* only an unambiguous single-version
repository is used, because a per-file image must never be attributed
to the wrong file.
``ModelInfos`` groups the repository's files by kind (``safetensor``,
…) and records a real sha256 for each, which is what makes it possible to
recognise a file the user has renamed.
"""
digests: dict[str, str] = {}
model_infos = data.get("ModelInfos")
if not isinstance(model_infos, dict):
return digests
for info in model_infos.values():
files = info.get("files") if isinstance(info, dict) else None
if not isinstance(files, list):
continue
for entry in files:
if not isinstance(entry, dict):
continue
name = _clean_text(entry.get("name"))
digest = _clean_text(entry.get("sha256"))
if name and digest:
digests.setdefault(os.path.basename(name).lower(), digest.lower())
return digests
def _matching_versions(
muse_info: Any,
filename: str,
*,
digests: dict[str, str] | None = None,
sha256: str = "",
) -> list[dict[str, Any]]:
"""Return the ``versions`` entries that publish the wanted model file.
Strategies, in order:
1. **sha256** — the file's content hash, looked up through
:func:`_file_digests`. This is the only strategy that survives the
user renaming the weights, which is common once a model is filed away.
2. **Exact basename** against each version's ``stats.fileList``.
3. **``showName`` inside the file stem**, which absorbs the naming drift
ModelScope sometimes applies to uploaded weights.
A known-but-unmatched hash falls through to the filename strategies
rather than giving up, in case the local file was re-encoded. All matches
are returned so a file re-published across several versions contributes
all of its example images. With no *filename* and no *sha256*, only an
unambiguous single-version repository is used, because a per-file image
must never be attributed to the wrong file.
"""
if not isinstance(muse_info, dict):
@@ -371,6 +420,18 @@ def _matching_versions(muse_info: Any, filename: str) -> list[dict[str, Any]]:
if not entries:
return []
target_hash = (sha256 or "").strip().lower()
if target_hash:
known = digests or {}
by_hash: list[dict[str, Any]] = []
for version in entries:
for path in _version_files(version):
if known.get(os.path.basename(path).lower()) == target_hash:
by_hash.append(version)
break
if by_hash:
return by_hash
if not filename:
return entries if len(entries) == 1 else []