fix(agent): drop site-generated placeholder model cards

A repository whose uploader wrote no README still gets a card.  ModelScope
answers with a placeholder notice ("the contributor provided no further
description"), a block of SDK/git download instructions, and a closing
invitation to complete the card.  None of it describes the model, yet it was
being sent to the LLM and, worse, stored as `modelDescription` — so a Krea 2
LoRA whose only real text was the author's summary showed 841 characters of
`pip install modelscope` scaffolding on its description tab.

Add `_strip_generated_card_boilerplate()` and run it on both paths:
`clean_readme_for_llm()` (the prompt) and `convert_readme_to_html()` (the
stored description).  Markers are matched as substrings because the notices
are prose and because non-Latin scripts are not space-delimited — the notice
continues with a full-width period, so the `title == keyword` matching used
for the English boilerplate headings never fired.

A marker heading takes its whole section with it, which is what removes the
download block hanging off the notice; a stand-alone notice line is dropped
alone.  Content the author added later, under a heading of equal or higher
level, is kept, so a card that was improved after the placeholder is not
thrown away.

Verified on the live repositories: the placeholder card's description went
from 841 characters to the 86-character author summary, while the repo with
a genuinely author-written card is byte-for-byte unchanged.
This commit is contained in:
Will Miao
2026-09-14 21:35:35 +08:00
parent 0f160e157f
commit 942717f0b6
3 changed files with 257 additions and 1 deletions
@@ -392,12 +392,18 @@ def _extract_frontmatter(text: str) -> str:
def convert_readme_to_html(markdown_text: str | None) -> str: def convert_readme_to_html(markdown_text: str | None) -> str:
"""Convert HF README markdown to sanitised HTML.""" """Convert HF README markdown to sanitised HTML.
Site-generated placeholder notices are dropped here too, so a repository
whose author wrote nothing does not store the download instructions as its
model description; the result is an empty string in that case.
"""
if not markdown_text: if not markdown_text:
return "" return ""
text = markdown_text text = markdown_text
text = _strip_frontmatter(text) text = _strip_frontmatter(text)
text = _strip_generated_card_boilerplate(text)
text = _strip_gallery(text) text = _strip_gallery(text)
text = _strip_badge_images(text) text = _strip_badge_images(text)
text = _strip_html_comments(text) text = _strip_html_comments(text)
@@ -444,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150
#: Minimum consecutive enumeration lines to trigger massive-list stripping. #: Minimum consecutive enumeration lines to trigger massive-list stripping.
_MASSIVE_LIST_THRESHOLD = 8 _MASSIVE_LIST_THRESHOLD = 8
#: Substrings identifying text a *site* generated to fill a model card whose
#: author wrote nothing, as opposed to the author's own content. ModelScope
#: renders such a card as a placeholder notice, a block of SDK/git download
#: instructions, and a closing invitation to improve the card.
#:
#: Matched as substrings rather than whole headings because the notices are
#: prose, and because non-Latin scripts are not space-delimited — the notice
#: continues with a full-width period, so the ``title == kw`` style matching
#: used for :data:`_BOILERPLATE_HEADERS` would never fire.
_GENERATED_CARD_MARKERS: tuple[str, ...] = (
"当前模型的贡献者未提供更加详细的模型介绍",
"您可以通过如下",
"如果您是本模型的贡献者",
)
def _strip_generated_card_boilerplate(text: str) -> str:
"""Remove the notices a site generates to fill an empty model card.
A repository whose uploader wrote no README still gets a card: ModelScope
answers with "the contributor provided no further description", the SDK
and git download commands, and an invitation to complete the card. None
of it describes the model, yet it was landing in both the LLM prompt and
the stored description.
A notice that is a heading takes its whole section with it, so the
download block goes too; a stand-alone notice line is dropped on its own.
Content the author added later — under a heading of equal or higher
level — is kept, so an improved card is not thrown away.
"""
lines = text.split("\n")
out: list[str] = []
skip_until_level: int | None = None
for line in lines:
level = _heading_level(line)
if any(marker in line for marker in _GENERATED_CARD_MARKERS):
if level > 0:
skip_until_level = level
continue
if skip_until_level is not None:
if level > 0 and level <= skip_until_level:
skip_until_level = None
else:
continue
out.append(line)
return "\n".join(out)
def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str: def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
"""Clean a HF README for injection into an LLM metadata-extraction prompt. """Clean a HF README for injection into an LLM metadata-extraction prompt.
@@ -453,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
* ``widget:`` YAML block (example prompts + output URLs) * ``widget:`` YAML block (example prompts + output URLs)
* ``<Gallery />`` tags and wrappers * ``<Gallery />`` tags and wrappers
* Site-generated placeholder notices for a card the author never wrote
(see :func:`_strip_generated_card_boilerplate`)
* Fenced code blocks (Python / bash / bibtex / yaml) * Fenced code blocks (Python / bash / bibtex / yaml)
* Standalone ``![...](...)`` image lines and ``<img>`` tags * Standalone ``![...](...)`` image lines and ``<img>`` tags
* Training-parameter tables * Training-parameter tables
@@ -478,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
# Order matters — broader strips first, then finer ones. # Order matters — broader strips first, then finer ones.
text = _strip_gallery(text) text = _strip_gallery(text)
text = _strip_widget_section(text) text = _strip_widget_section(text)
text = _strip_generated_card_boilerplate(text)
text = _strip_fenced_code_blocks(text) text = _strip_fenced_code_blocks(text)
text = _strip_standalone_images(text) text = _strip_standalone_images(text)
text = _strip_training_tables(text) text = _strip_training_tables(text)
+115
View File
@@ -490,3 +490,118 @@ class TestStripFencedCodeBlocks:
def test_pattern(self, R): def test_pattern(self, R):
text = "x\n```yaml\nkey: val\n```\ny" text = "x\n```yaml\nkey: val\n```\ny"
assert "key: val" not in R._strip_fenced_code_blocks(text) assert "key: val" not in R._strip_fenced_code_blocks(text)
# ======================================================================
# Site-generated placeholder cards
# ======================================================================
#: The card ModelScope renders when the uploader wrote no README. Copied from
#: a live repository so the marker strings stay honest.
PLACEHOLDER_CARD = """---
base_model: krea/Krea-2-Turbo
license: Apache License 2.0
tags:
- LoRA
- text-to-image
- \u68a6\u5e7b\u5149\u5f71
---
### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\u6a21\u578b\u6587\u4ef6\u548c\u6743\u91cd\uff0c\u53ef\u6d4f\u89c8\u201c\u6a21\u578b\u6587\u4ef6\u201d\u9875\u9762\u83b7\u53d6\u3002
#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\uff0c\u6216\u8005ModelScope SDK\u6765\u4e0b\u8f7d\u6a21\u578b
SDK\u4e0b\u8f7d
```bash
#\u5b89\u88c5ModelScope
pip install modelscope
```
Git\u4e0b\u8f7d
```
#Git\u6a21\u578b\u4e0b\u8f7d
git clone https://www.modelscope.cn/yan303145427/krea2-CcFQWZ-Portrait.git
```
<p style="color: lightgrey;">\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u6211\u4eec\u9080\u8bf7\u60a8\u6839\u636e<a href="x">\u6a21\u578b\u8d21\u732e\u6587\u6863</a>\uff0c\u53ca\u65f6\u5b8c\u5584\u6a21\u578b\u5361\u7247\u5185\u5bb9\u3002</p>
"""
#: A real, author-written card (ModelScope AIGC training output).
REAL_CARD = """---
base_model: krea/Krea-2-Turbo
---
# krea\u8138\u6a21
## \u6a21\u578b\u4ecb\u7ecd
\u672c\u6a21\u578b\u4f9d\u6258\u9b54\u642d\u793e\u533a\u5b8c\u6210\u8bad\u7ec3\u3002
## \u63a8\u7406\u4ee3\u7801
\u5b89\u88c5 DiffSynth-Studio\uff1a
"""
class TestStripGeneratedCardBoilerplate:
def test_removes_the_whole_placeholder_body(self, R):
stripped = R._strip_generated_card_boilerplate(PLACEHOLDER_CARD)
assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in stripped
assert "SDK\u4e0b\u8f7d" not in stripped
assert "git clone" not in stripped
assert "\u9080\u8bf7\u60a8" not in stripped
# The frontmatter is the only thing that survives.
assert "base_model: krea/Krea-2-Turbo" in stripped
def test_leaves_a_real_card_untouched(self, R):
assert R._strip_generated_card_boilerplate(REAL_CARD) == REAL_CARD
def test_drops_a_standalone_invitation_line(self, R):
text = "real body\n<p>\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u8bf7\u5b8c\u5584</p>\nmore body"
stripped = R._strip_generated_card_boilerplate(text)
assert "real body" in stripped
assert "more body" in stripped
assert "\u8d21\u732e\u8005" not in stripped
def test_keeps_content_added_after_the_placeholder(self, R):
"""An author who later wrote a real section must not lose it."""
text = (
"### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\n"
"#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\u4e0b\u8f7d\u6a21\u578b\n"
"```\ngit clone x\n```\n"
"## \u6211\u7684\u771f\u5b9e\u4ecb\u7ecd\n"
"\u8fd9\u662f\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9\u3002\n"
)
stripped = R._strip_generated_card_boilerplate(text)
assert "\u6211\u7684\u771f\u5b9e\u4ecb\u7ecd" in stripped
assert "\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9" in stripped
assert "git clone" not in stripped
def test_handles_html_headings(self, R):
text = (
"<h3>\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002</h3>\n"
"<p>pip install modelscope</p>\n"
)
assert R._strip_generated_card_boilerplate(text).strip() == ""
class TestCleanReadmeForLlmPlaceholder:
def test_boilerplate_is_gone_but_frontmatter_survives(self, R):
cleaned = R.clean_readme_for_llm(PLACEHOLDER_CARD)
assert "pip install modelscope" not in cleaned
assert "git clone" not in cleaned
assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in cleaned
# Metadata the LLM still needs.
assert "base_model: krea/Krea-2-Turbo" in cleaned
assert "\u68a6\u5e7b\u5149\u5f71" in cleaned
def test_a_real_card_keeps_its_body(self, R):
cleaned = R.clean_readme_for_llm(REAL_CARD)
assert "krea\u8138\u6a21" in cleaned
assert "\u6a21\u578b\u4ecb\u7ecd" in cleaned
class TestConvertReadmeToHtmlPlaceholder:
def test_a_placeholder_card_renders_to_nothing(self, R):
assert R.convert_readme_to_html(PLACEHOLDER_CARD) == ""
def test_a_real_card_still_renders(self, R):
html = R.convert_readme_to_html(REAL_CARD)
assert "<h1>krea\u8138\u6a21</h1>" in html
assert "DiffSynth-Studio" in html
+79
View File
@@ -1042,3 +1042,82 @@ class TestDeterministicFallbacks:
assert "tags" not in applied assert "tags" not in applied
assert "notes" not in applied assert "notes" not in applied
assert "usage_tips" not in applied assert "usage_tips" not in applied
class TestPlaceholderCardDescription:
"""A site-generated placeholder card must not become the description."""
MODELSCOPE_METADATA = {
"from_civitai": False,
"source_platform": "modelscope",
"source_url": "https://modelscope.cn/models/user/repo",
}
PLACEHOLDER_README = """---
base_model: krea/Krea-2-Turbo
---
### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重,可浏览“模型文件”页面获取。
#### 您可以通过如下git clone命令,或者ModelScope SDK来下载模型
SDK下载
```bash
pip install modelscope
```
<p style="color: lightgrey;">如果您是本模型的贡献者,我们邀请您根据文档及时完善模型卡片内容。</p>
"""
LLM_OUTPUT = {
"base_model": "Krea 2",
"trigger_words": [],
"short_description": "一个 Krea 2 人像 LoRA。",
"tags": [],
"recommended_width": 0,
"recommended_height": 0,
"preview_url": "",
"notes": "",
"usage_tips": "{}",
"confidence": "medium",
}
@pytest.mark.asyncio
async def test_description_holds_only_the_author_summary(self, processor):
context = ModelCardContext(description="权重0.5-1.2。")
with (
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
mock.patch("py.metadata_ops.download_preview", return_value=None),
mock.patch("py.metadata_ops.refresh_cache"),
):
await processor.process(
skill_name="enrich_hf_metadata",
model_path="/p.safetensors",
llm_output=self.LLM_OUTPUT,
metadata=dict(self.MODELSCOPE_METADATA),
readme_content=self.PLACEHOLDER_README,
source_context=context,
)
description = mock_apply.call_args[0][1]["modelDescription"]
assert description == "<p>权重0.5-1.2。</p>"
assert "pip install modelscope" not in description
assert "git clone" not in description
assert "贡献者" not in description
@pytest.mark.asyncio
async def test_placeholder_card_alone_writes_no_description(self, processor):
"""Without an author summary there is nothing worth storing."""
with (
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
mock.patch("py.metadata_ops.download_preview", return_value=None),
mock.patch("py.metadata_ops.refresh_cache"),
):
await processor.process(
skill_name="enrich_hf_metadata",
model_path="/p.safetensors",
llm_output=self.LLM_OUTPUT,
metadata=dict(self.MODELSCOPE_METADATA),
readme_content=self.PLACEHOLDER_README,
source_context=ModelCardContext(),
)
assert "modelDescription" not in mock_apply.call_args[0][1]