fix(agent): drop site-generated placeholder model cards

A repository whose uploader wrote no README still gets a card. ModelScope answers with a placeholder notice ("the contributor provided no further description"), a block of SDK/git download instructions, and a closing invitation to complete the card. None of it describes the model, yet it was being sent to the LLM and, worse, stored as `modelDescription` — so a Krea 2 LoRA whose only real text was the author's summary showed 841 characters of `pip install modelscope` scaffolding on its description tab. Add `_strip_generated_card_boilerplate()` and run it on both paths: `clean_readme_for_llm()` (the prompt) and `convert_readme_to_html()` (the stored description). Markers are matched as substrings because the notices are prose and because non-Latin scripts are not space-delimited — the notice continues with a full-width period, so the `title == keyword` matching used for the English boilerplate headings never fired. A marker heading takes its whole section with it, which is what removes the download block hanging off the notice; a stand-alone notice line is dropped alone. Content the author added later, under a heading of equal or higher level, is kept, so a card that was improved after the placeholder is not thrown away. Verified on the live repositories: the placeholder card's description went from 841 characters to the 86-character author summary, while the repo with a genuinely author-written card is byte-for-byte unchanged.
2026-09-20 18:51:26 -03:00 · 2026-09-14 21:35:35 +08:00
parent 0f160e157f
commit 942717f0b6
3 changed files with 257 additions and 1 deletions
@@ -392,12 +392,18 @@ def _extract_frontmatter(text: str) -> str:


 def convert_readme_to_html(markdown_text: str | None) -> str:
-    """Convert HF README markdown to sanitised HTML."""
+    """Convert HF README markdown to sanitised HTML.
+
+    Site-generated placeholder notices are dropped here too, so a repository
+    whose author wrote nothing does not store the download instructions as its
+    model description; the result is an empty string in that case.
+    """
    if not markdown_text:
        return ""

    text = markdown_text
    text = _strip_frontmatter(text)
+    text = _strip_generated_card_boilerplate(text)
    text = _strip_gallery(text)
    text = _strip_badge_images(text)
    text = _strip_html_comments(text)
@@ -444,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150
 #: Minimum consecutive enumeration lines to trigger massive-list stripping.
 _MASSIVE_LIST_THRESHOLD = 8

+#: Substrings identifying text a *site* generated to fill a model card whose
+#: author wrote nothing, as opposed to the author's own content.  ModelScope
+#: renders such a card as a placeholder notice, a block of SDK/git download
+#: instructions, and a closing invitation to improve the card.
+#:
+#: Matched as substrings rather than whole headings because the notices are
+#: prose, and because non-Latin scripts are not space-delimited — the notice
+#: continues with a full-width period, so the ``title == kw`` style matching
+#: used for :data:`_BOILERPLATE_HEADERS` would never fire.
+_GENERATED_CARD_MARKERS: tuple[str, ...] = (
+    "当前模型的贡献者未提供更加详细的模型介绍",
+    "您可以通过如下",
+    "如果您是本模型的贡献者",
+)
+
+
+def _strip_generated_card_boilerplate(text: str) -> str:
+    """Remove the notices a site generates to fill an empty model card.
+
+    A repository whose uploader wrote no README still gets a card: ModelScope
+    answers with "the contributor provided no further description", the SDK
+    and git download commands, and an invitation to complete the card.  None
+    of it describes the model, yet it was landing in both the LLM prompt and
+    the stored description.
+
+    A notice that is a heading takes its whole section with it, so the
+    download block goes too; a stand-alone notice line is dropped on its own.
+    Content the author added later — under a heading of equal or higher
+    level — is kept, so an improved card is not thrown away.
+    """
+
+    lines = text.split("\n")
+    out: list[str] = []
+    skip_until_level: int | None = None
+
+    for line in lines:
+        level = _heading_level(line)
+
+        if any(marker in line for marker in _GENERATED_CARD_MARKERS):
+            if level > 0:
+                skip_until_level = level
+            continue
+
+        if skip_until_level is not None:
+            if level > 0 and level <= skip_until_level:
+                skip_until_level = None
+            else:
+                continue
+
+        out.append(line)
+
+    return "\n".join(out)
+

 def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
    """Clean a HF README for injection into an LLM metadata-extraction prompt.
@@ -453,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s

    * ``widget:`` YAML block (example prompts + output URLs)
    * ``<Gallery />`` tags and wrappers
+    * Site-generated placeholder notices for a card the author never wrote
+      (see :func:`_strip_generated_card_boilerplate`)
    * Fenced code blocks (Python / bash / bibtex / yaml)
    * Standalone ``![...](...)`` image lines and ``<img>`` tags
    * Training-parameter tables
@@ -478,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
    # Order matters — broader strips first, then finer ones.
    text = _strip_gallery(text)
    text = _strip_widget_section(text)
+    text = _strip_generated_card_boilerplate(text)
    text = _strip_fenced_code_blocks(text)
    text = _strip_standalone_images(text)
    text = _strip_training_tables(text)
@@ -490,3 +490,118 @@ class TestStripFencedCodeBlocks:
    def test_pattern(self, R):
        text = "x\n```yaml\nkey: val\n```\ny"
        assert "key: val" not in R._strip_fenced_code_blocks(text)
+
+
+# ======================================================================
+# Site-generated placeholder cards
+# ======================================================================
+
+#: The card ModelScope renders when the uploader wrote no README.  Copied from
+#: a live repository so the marker strings stay honest.
+PLACEHOLDER_CARD = """---
+base_model: krea/Krea-2-Turbo
+license: Apache License 2.0
+tags:
+- LoRA
+- text-to-image
+- \u68a6\u5e7b\u5149\u5f71
+---
+### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\u6a21\u578b\u6587\u4ef6\u548c\u6743\u91cd\uff0c\u53ef\u6d4f\u89c8\u201c\u6a21\u578b\u6587\u4ef6\u201d\u9875\u9762\u83b7\u53d6\u3002
+#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\uff0c\u6216\u8005ModelScope SDK\u6765\u4e0b\u8f7d\u6a21\u578b
+
+SDK\u4e0b\u8f7d
+```bash
+#\u5b89\u88c5ModelScope
+pip install modelscope
+```
+Git\u4e0b\u8f7d
+```
+#Git\u6a21\u578b\u4e0b\u8f7d
+git clone https://www.modelscope.cn/yan303145427/krea2-CcFQWZ-Portrait.git
+```
+
+<p style="color: lightgrey;">\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u6211\u4eec\u9080\u8bf7\u60a8\u6839\u636e<a href="x">\u6a21\u578b\u8d21\u732e\u6587\u6863</a>\uff0c\u53ca\u65f6\u5b8c\u5584\u6a21\u578b\u5361\u7247\u5185\u5bb9\u3002</p>
+"""
+
+#: A real, author-written card (ModelScope AIGC training output).
+REAL_CARD = """---
+base_model: krea/Krea-2-Turbo
+---
+# krea\u8138\u6a21
+
+## \u6a21\u578b\u4ecb\u7ecd
+
+\u672c\u6a21\u578b\u4f9d\u6258\u9b54\u642d\u793e\u533a\u5b8c\u6210\u8bad\u7ec3\u3002
+
+## \u63a8\u7406\u4ee3\u7801
+
+\u5b89\u88c5 DiffSynth-Studio\uff1a
+"""
+
+
+class TestStripGeneratedCardBoilerplate:
+    def test_removes_the_whole_placeholder_body(self, R):
+        stripped = R._strip_generated_card_boilerplate(PLACEHOLDER_CARD)
+        assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in stripped
+        assert "SDK\u4e0b\u8f7d" not in stripped
+        assert "git clone" not in stripped
+        assert "\u9080\u8bf7\u60a8" not in stripped
+        # The frontmatter is the only thing that survives.
+        assert "base_model: krea/Krea-2-Turbo" in stripped
+
+    def test_leaves_a_real_card_untouched(self, R):
+        assert R._strip_generated_card_boilerplate(REAL_CARD) == REAL_CARD
+
+    def test_drops_a_standalone_invitation_line(self, R):
+        text = "real body\n<p>\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u8bf7\u5b8c\u5584</p>\nmore body"
+        stripped = R._strip_generated_card_boilerplate(text)
+        assert "real body" in stripped
+        assert "more body" in stripped
+        assert "\u8d21\u732e\u8005" not in stripped
+
+    def test_keeps_content_added_after_the_placeholder(self, R):
+        """An author who later wrote a real section must not lose it."""
+        text = (
+            "### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\n"
+            "#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\u4e0b\u8f7d\u6a21\u578b\n"
+            "```\ngit clone x\n```\n"
+            "## \u6211\u7684\u771f\u5b9e\u4ecb\u7ecd\n"
+            "\u8fd9\u662f\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9\u3002\n"
+        )
+        stripped = R._strip_generated_card_boilerplate(text)
+        assert "\u6211\u7684\u771f\u5b9e\u4ecb\u7ecd" in stripped
+        assert "\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9" in stripped
+        assert "git clone" not in stripped
+
+    def test_handles_html_headings(self, R):
+        text = (
+            "<h3>\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002</h3>\n"
+            "<p>pip install modelscope</p>\n"
+        )
+        assert R._strip_generated_card_boilerplate(text).strip() == ""
+
+
+class TestCleanReadmeForLlmPlaceholder:
+    def test_boilerplate_is_gone_but_frontmatter_survives(self, R):
+        cleaned = R.clean_readme_for_llm(PLACEHOLDER_CARD)
+        assert "pip install modelscope" not in cleaned
+        assert "git clone" not in cleaned
+        assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in cleaned
+        # Metadata the LLM still needs.
+        assert "base_model: krea/Krea-2-Turbo" in cleaned
+        assert "\u68a6\u5e7b\u5149\u5f71" in cleaned
+
+    def test_a_real_card_keeps_its_body(self, R):
+        cleaned = R.clean_readme_for_llm(REAL_CARD)
+        assert "krea\u8138\u6a21" in cleaned
+        assert "\u6a21\u578b\u4ecb\u7ecd" in cleaned
+
+
+class TestConvertReadmeToHtmlPlaceholder:
+    def test_a_placeholder_card_renders_to_nothing(self, R):
+        assert R.convert_readme_to_html(PLACEHOLDER_CARD) == ""
+
+    def test_a_real_card_still_renders(self, R):
+        html = R.convert_readme_to_html(REAL_CARD)
+        assert "<h1>krea\u8138\u6a21</h1>" in html
+        assert "DiffSynth-Studio" in html
@@ -1042,3 +1042,82 @@ class TestDeterministicFallbacks:
        assert "tags" not in applied
        assert "notes" not in applied
        assert "usage_tips" not in applied
+
+
+class TestPlaceholderCardDescription:
+    """A site-generated placeholder card must not become the description."""
+
+    MODELSCOPE_METADATA = {
+        "from_civitai": False,
+        "source_platform": "modelscope",
+        "source_url": "https://modelscope.cn/models/user/repo",
+    }
+
+    PLACEHOLDER_README = """---
+base_model: krea/Krea-2-Turbo
+---
+### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重，可浏览“模型文件”页面获取。
+#### 您可以通过如下git clone命令，或者ModelScope SDK来下载模型
+
+SDK下载
+```bash
+pip install modelscope
+```
+
+<p style="color: lightgrey;">如果您是本模型的贡献者，我们邀请您根据文档及时完善模型卡片内容。</p>
+"""
+
+    LLM_OUTPUT = {
+        "base_model": "Krea 2",
+        "trigger_words": [],
+        "short_description": "一个 Krea 2 人像 LoRA。",
+        "tags": [],
+        "recommended_width": 0,
+        "recommended_height": 0,
+        "preview_url": "",
+        "notes": "",
+        "usage_tips": "{}",
+        "confidence": "medium",
+    }
+
+    @pytest.mark.asyncio
+    async def test_description_holds_only_the_author_summary(self, processor):
+        context = ModelCardContext(description="权重0.5-1.2。")
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.LLM_OUTPUT,
+                metadata=dict(self.MODELSCOPE_METADATA),
+                readme_content=self.PLACEHOLDER_README,
+                source_context=context,
+            )
+
+        description = mock_apply.call_args[0][1]["modelDescription"]
+        assert description == "<p>权重0.5-1.2。</p>"
+        assert "pip install modelscope" not in description
+        assert "git clone" not in description
+        assert "贡献者" not in description
+
+    @pytest.mark.asyncio
+    async def test_placeholder_card_alone_writes_no_description(self, processor):
+        """Without an author summary there is nothing worth storing."""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.LLM_OUTPUT,
+                metadata=dict(self.MODELSCOPE_METADATA),
+                readme_content=self.PLACEHOLDER_README,
+                source_context=ModelCardContext(),
+            )
+
+        assert "modelDescription" not in mock_apply.call_args[0][1]