diff --git a/py/services/agent/skills/enrich_hf_metadata/readme_processor.py b/py/services/agent/skills/enrich_hf_metadata/readme_processor.py
index 452a9542..bcea8856 100644
--- a/py/services/agent/skills/enrich_hf_metadata/readme_processor.py
+++ b/py/services/agent/skills/enrich_hf_metadata/readme_processor.py
@@ -392,12 +392,18 @@ def _extract_frontmatter(text: str) -> str:
def convert_readme_to_html(markdown_text: str | None) -> str:
- """Convert HF README markdown to sanitised HTML."""
+ """Convert HF README markdown to sanitised HTML.
+
+ Site-generated placeholder notices are dropped here too, so a repository
+ whose author wrote nothing does not store the download instructions as its
+ model description; the result is an empty string in that case.
+ """
if not markdown_text:
return ""
text = markdown_text
text = _strip_frontmatter(text)
+ text = _strip_generated_card_boilerplate(text)
text = _strip_gallery(text)
text = _strip_badge_images(text)
text = _strip_html_comments(text)
@@ -444,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150
#: Minimum consecutive enumeration lines to trigger massive-list stripping.
_MASSIVE_LIST_THRESHOLD = 8
+#: Substrings identifying text a *site* generated to fill a model card whose
+#: author wrote nothing, as opposed to the author's own content. ModelScope
+#: renders such a card as a placeholder notice, a block of SDK/git download
+#: instructions, and a closing invitation to improve the card.
+#:
+#: Matched as substrings rather than whole headings because the notices are
+#: prose, and because non-Latin scripts are not space-delimited — the notice
+#: continues with a full-width period, so the ``title == kw`` style matching
+#: used for :data:`_BOILERPLATE_HEADERS` would never fire.
+_GENERATED_CARD_MARKERS: tuple[str, ...] = (
+ "当前模型的贡献者未提供更加详细的模型介绍",
+ "您可以通过如下",
+ "如果您是本模型的贡献者",
+)
+
+
+def _strip_generated_card_boilerplate(text: str) -> str:
+ """Remove the notices a site generates to fill an empty model card.
+
+ A repository whose uploader wrote no README still gets a card: ModelScope
+ answers with "the contributor provided no further description", the SDK
+ and git download commands, and an invitation to complete the card. None
+ of it describes the model, yet it was landing in both the LLM prompt and
+ the stored description.
+
+ A notice that is a heading takes its whole section with it, so the
+ download block goes too; a stand-alone notice line is dropped on its own.
+ Content the author added later — under a heading of equal or higher
+ level — is kept, so an improved card is not thrown away.
+ """
+
+ lines = text.split("\n")
+ out: list[str] = []
+ skip_until_level: int | None = None
+
+ for line in lines:
+ level = _heading_level(line)
+
+ if any(marker in line for marker in _GENERATED_CARD_MARKERS):
+ if level > 0:
+ skip_until_level = level
+ continue
+
+ if skip_until_level is not None:
+ if level > 0 and level <= skip_until_level:
+ skip_until_level = None
+ else:
+ continue
+
+ out.append(line)
+
+ return "\n".join(out)
+
def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
"""Clean a HF README for injection into an LLM metadata-extraction prompt.
@@ -453,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
* ``widget:`` YAML block (example prompts + output URLs)
* ```` tags
* Training-parameter tables
@@ -478,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
# Order matters — broader strips first, then finer ones.
text = _strip_gallery(text)
text = _strip_widget_section(text)
+ text = _strip_generated_card_boilerplate(text)
text = _strip_fenced_code_blocks(text)
text = _strip_standalone_images(text)
text = _strip_training_tables(text)
diff --git a/tests/metadata_ops/test_readme_processor.py b/tests/metadata_ops/test_readme_processor.py
index e695c9aa..e45cbe9c 100644
--- a/tests/metadata_ops/test_readme_processor.py
+++ b/tests/metadata_ops/test_readme_processor.py
@@ -490,3 +490,118 @@ class TestStripFencedCodeBlocks:
def test_pattern(self, R):
text = "x\n```yaml\nkey: val\n```\ny"
assert "key: val" not in R._strip_fenced_code_blocks(text)
+
+
+# ======================================================================
+# Site-generated placeholder cards
+# ======================================================================
+
+#: The card ModelScope renders when the uploader wrote no README. Copied from
+#: a live repository so the marker strings stay honest.
+PLACEHOLDER_CARD = """---
+base_model: krea/Krea-2-Turbo
+license: Apache License 2.0
+tags:
+- LoRA
+- text-to-image
+- \u68a6\u5e7b\u5149\u5f71
+---
+### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\u6a21\u578b\u6587\u4ef6\u548c\u6743\u91cd\uff0c\u53ef\u6d4f\u89c8\u201c\u6a21\u578b\u6587\u4ef6\u201d\u9875\u9762\u83b7\u53d6\u3002
+#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\uff0c\u6216\u8005ModelScope SDK\u6765\u4e0b\u8f7d\u6a21\u578b
+
+SDK\u4e0b\u8f7d
+```bash
+#\u5b89\u88c5ModelScope
+pip install modelscope
+```
+Git\u4e0b\u8f7d
+```
+#Git\u6a21\u578b\u4e0b\u8f7d
+git clone https://www.modelscope.cn/yan303145427/krea2-CcFQWZ-Portrait.git
+```
+
+
\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u6211\u4eec\u9080\u8bf7\u60a8\u6839\u636e\u6a21\u578b\u8d21\u732e\u6587\u6863\uff0c\u53ca\u65f6\u5b8c\u5584\u6a21\u578b\u5361\u7247\u5185\u5bb9\u3002
+""" + +#: A real, author-written card (ModelScope AIGC training output). +REAL_CARD = """--- +base_model: krea/Krea-2-Turbo +--- +# krea\u8138\u6a21 + +## \u6a21\u578b\u4ecb\u7ecd + +\u672c\u6a21\u578b\u4f9d\u6258\u9b54\u642d\u793e\u533a\u5b8c\u6210\u8bad\u7ec3\u3002 + +## \u63a8\u7406\u4ee3\u7801 + +\u5b89\u88c5 DiffSynth-Studio\uff1a +""" + + +class TestStripGeneratedCardBoilerplate: + def test_removes_the_whole_placeholder_body(self, R): + stripped = R._strip_generated_card_boilerplate(PLACEHOLDER_CARD) + assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in stripped + assert "SDK\u4e0b\u8f7d" not in stripped + assert "git clone" not in stripped + assert "\u9080\u8bf7\u60a8" not in stripped + # The frontmatter is the only thing that survives. + assert "base_model: krea/Krea-2-Turbo" in stripped + + def test_leaves_a_real_card_untouched(self, R): + assert R._strip_generated_card_boilerplate(REAL_CARD) == REAL_CARD + + def test_drops_a_standalone_invitation_line(self, R): + text = "real body\n\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u8bf7\u5b8c\u5584
\nmore body" + stripped = R._strip_generated_card_boilerplate(text) + assert "real body" in stripped + assert "more body" in stripped + assert "\u8d21\u732e\u8005" not in stripped + + def test_keeps_content_added_after_the_placeholder(self, R): + """An author who later wrote a real section must not lose it.""" + text = ( + "### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\n" + "#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\u4e0b\u8f7d\u6a21\u578b\n" + "```\ngit clone x\n```\n" + "## \u6211\u7684\u771f\u5b9e\u4ecb\u7ecd\n" + "\u8fd9\u662f\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9\u3002\n" + ) + stripped = R._strip_generated_card_boilerplate(text) + assert "\u6211\u7684\u771f\u5b9e\u4ecb\u7ecd" in stripped + assert "\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9" in stripped + assert "git clone" not in stripped + + def test_handles_html_headings(self, R): + text = ( + "pip install modelscope
\n" + ) + assert R._strip_generated_card_boilerplate(text).strip() == "" + + +class TestCleanReadmeForLlmPlaceholder: + def test_boilerplate_is_gone_but_frontmatter_survives(self, R): + cleaned = R.clean_readme_for_llm(PLACEHOLDER_CARD) + assert "pip install modelscope" not in cleaned + assert "git clone" not in cleaned + assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in cleaned + # Metadata the LLM still needs. + assert "base_model: krea/Krea-2-Turbo" in cleaned + assert "\u68a6\u5e7b\u5149\u5f71" in cleaned + + def test_a_real_card_keeps_its_body(self, R): + cleaned = R.clean_readme_for_llm(REAL_CARD) + assert "krea\u8138\u6a21" in cleaned + assert "\u6a21\u578b\u4ecb\u7ecd" in cleaned + + +class TestConvertReadmeToHtmlPlaceholder: + def test_a_placeholder_card_renders_to_nothing(self, R): + assert R.convert_readme_to_html(PLACEHOLDER_CARD) == "" + + def test_a_real_card_still_renders(self, R): + html = R.convert_readme_to_html(REAL_CARD) + assert "如果您是本模型的贡献者,我们邀请您根据文档及时完善模型卡片内容。
+""" + + LLM_OUTPUT = { + "base_model": "Krea 2", + "trigger_words": [], + "short_description": "一个 Krea 2 人像 LoRA。", + "tags": [], + "recommended_width": 0, + "recommended_height": 0, + "preview_url": "", + "notes": "", + "usage_tips": "{}", + "confidence": "medium", + } + + @pytest.mark.asyncio + async def test_description_holds_only_the_author_summary(self, processor): + context = ModelCardContext(description="权重0.5-1.2。") + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=None), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=self.LLM_OUTPUT, + metadata=dict(self.MODELSCOPE_METADATA), + readme_content=self.PLACEHOLDER_README, + source_context=context, + ) + + description = mock_apply.call_args[0][1]["modelDescription"] + assert description == "权重0.5-1.2。
" + assert "pip install modelscope" not in description + assert "git clone" not in description + assert "贡献者" not in description + + @pytest.mark.asyncio + async def test_placeholder_card_alone_writes_no_description(self, processor): + """Without an author summary there is nothing worth storing.""" + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=None), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=self.LLM_OUTPUT, + metadata=dict(self.MODELSCOPE_METADATA), + readme_content=self.PLACEHOLDER_README, + source_context=ModelCardContext(), + ) + + assert "modelDescription" not in mock_apply.call_args[0][1]