diff --git a/py/services/agent/skills/enrich_hf_metadata/readme_processor.py b/py/services/agent/skills/enrich_hf_metadata/readme_processor.py index 452a9542..bcea8856 100644 --- a/py/services/agent/skills/enrich_hf_metadata/readme_processor.py +++ b/py/services/agent/skills/enrich_hf_metadata/readme_processor.py @@ -392,12 +392,18 @@ def _extract_frontmatter(text: str) -> str: def convert_readme_to_html(markdown_text: str | None) -> str: - """Convert HF README markdown to sanitised HTML.""" + """Convert HF README markdown to sanitised HTML. + + Site-generated placeholder notices are dropped here too, so a repository + whose author wrote nothing does not store the download instructions as its + model description; the result is an empty string in that case. + """ if not markdown_text: return "" text = markdown_text text = _strip_frontmatter(text) + text = _strip_generated_card_boilerplate(text) text = _strip_gallery(text) text = _strip_badge_images(text) text = _strip_html_comments(text) @@ -444,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150 #: Minimum consecutive enumeration lines to trigger massive-list stripping. _MASSIVE_LIST_THRESHOLD = 8 +#: Substrings identifying text a *site* generated to fill a model card whose +#: author wrote nothing, as opposed to the author's own content. ModelScope +#: renders such a card as a placeholder notice, a block of SDK/git download +#: instructions, and a closing invitation to improve the card. +#: +#: Matched as substrings rather than whole headings because the notices are +#: prose, and because non-Latin scripts are not space-delimited — the notice +#: continues with a full-width period, so the ``title == kw`` style matching +#: used for :data:`_BOILERPLATE_HEADERS` would never fire. +_GENERATED_CARD_MARKERS: tuple[str, ...] = ( + "当前模型的贡献者未提供更加详细的模型介绍", + "您可以通过如下", + "如果您是本模型的贡献者", +) + + +def _strip_generated_card_boilerplate(text: str) -> str: + """Remove the notices a site generates to fill an empty model card. + + A repository whose uploader wrote no README still gets a card: ModelScope + answers with "the contributor provided no further description", the SDK + and git download commands, and an invitation to complete the card. None + of it describes the model, yet it was landing in both the LLM prompt and + the stored description. + + A notice that is a heading takes its whole section with it, so the + download block goes too; a stand-alone notice line is dropped on its own. + Content the author added later — under a heading of equal or higher + level — is kept, so an improved card is not thrown away. + """ + + lines = text.split("\n") + out: list[str] = [] + skip_until_level: int | None = None + + for line in lines: + level = _heading_level(line) + + if any(marker in line for marker in _GENERATED_CARD_MARKERS): + if level > 0: + skip_until_level = level + continue + + if skip_until_level is not None: + if level > 0 and level <= skip_until_level: + skip_until_level = None + else: + continue + + out.append(line) + + return "\n".join(out) + def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str: """Clean a HF README for injection into an LLM metadata-extraction prompt. @@ -453,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s * ``widget:`` YAML block (example prompts + output URLs) * ```` tags and wrappers + * Site-generated placeholder notices for a card the author never wrote + (see :func:`_strip_generated_card_boilerplate`) * Fenced code blocks (Python / bash / bibtex / yaml) * Standalone ``![...](...)`` image lines and ```` tags * Training-parameter tables @@ -478,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s # Order matters — broader strips first, then finer ones. text = _strip_gallery(text) text = _strip_widget_section(text) + text = _strip_generated_card_boilerplate(text) text = _strip_fenced_code_blocks(text) text = _strip_standalone_images(text) text = _strip_training_tables(text) diff --git a/tests/metadata_ops/test_readme_processor.py b/tests/metadata_ops/test_readme_processor.py index e695c9aa..e45cbe9c 100644 --- a/tests/metadata_ops/test_readme_processor.py +++ b/tests/metadata_ops/test_readme_processor.py @@ -490,3 +490,118 @@ class TestStripFencedCodeBlocks: def test_pattern(self, R): text = "x\n```yaml\nkey: val\n```\ny" assert "key: val" not in R._strip_fenced_code_blocks(text) + + +# ====================================================================== +# Site-generated placeholder cards +# ====================================================================== + +#: The card ModelScope renders when the uploader wrote no README. Copied from +#: a live repository so the marker strings stay honest. +PLACEHOLDER_CARD = """--- +base_model: krea/Krea-2-Turbo +license: Apache License 2.0 +tags: +- LoRA +- text-to-image +- \u68a6\u5e7b\u5149\u5f71 +--- +### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\u6a21\u578b\u6587\u4ef6\u548c\u6743\u91cd\uff0c\u53ef\u6d4f\u89c8\u201c\u6a21\u578b\u6587\u4ef6\u201d\u9875\u9762\u83b7\u53d6\u3002 +#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\uff0c\u6216\u8005ModelScope SDK\u6765\u4e0b\u8f7d\u6a21\u578b + +SDK\u4e0b\u8f7d +```bash +#\u5b89\u88c5ModelScope +pip install modelscope +``` +Git\u4e0b\u8f7d +``` +#Git\u6a21\u578b\u4e0b\u8f7d +git clone https://www.modelscope.cn/yan303145427/krea2-CcFQWZ-Portrait.git +``` + +

\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u6211\u4eec\u9080\u8bf7\u60a8\u6839\u636e\u6a21\u578b\u8d21\u732e\u6587\u6863\uff0c\u53ca\u65f6\u5b8c\u5584\u6a21\u578b\u5361\u7247\u5185\u5bb9\u3002

+""" + +#: A real, author-written card (ModelScope AIGC training output). +REAL_CARD = """--- +base_model: krea/Krea-2-Turbo +--- +# krea\u8138\u6a21 + +## \u6a21\u578b\u4ecb\u7ecd + +\u672c\u6a21\u578b\u4f9d\u6258\u9b54\u642d\u793e\u533a\u5b8c\u6210\u8bad\u7ec3\u3002 + +## \u63a8\u7406\u4ee3\u7801 + +\u5b89\u88c5 DiffSynth-Studio\uff1a +""" + + +class TestStripGeneratedCardBoilerplate: + def test_removes_the_whole_placeholder_body(self, R): + stripped = R._strip_generated_card_boilerplate(PLACEHOLDER_CARD) + assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in stripped + assert "SDK\u4e0b\u8f7d" not in stripped + assert "git clone" not in stripped + assert "\u9080\u8bf7\u60a8" not in stripped + # The frontmatter is the only thing that survives. + assert "base_model: krea/Krea-2-Turbo" in stripped + + def test_leaves_a_real_card_untouched(self, R): + assert R._strip_generated_card_boilerplate(REAL_CARD) == REAL_CARD + + def test_drops_a_standalone_invitation_line(self, R): + text = "real body\n

\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u8bf7\u5b8c\u5584

\nmore body" + stripped = R._strip_generated_card_boilerplate(text) + assert "real body" in stripped + assert "more body" in stripped + assert "\u8d21\u732e\u8005" not in stripped + + def test_keeps_content_added_after_the_placeholder(self, R): + """An author who later wrote a real section must not lose it.""" + text = ( + "### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\n" + "#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\u4e0b\u8f7d\u6a21\u578b\n" + "```\ngit clone x\n```\n" + "## \u6211\u7684\u771f\u5b9e\u4ecb\u7ecd\n" + "\u8fd9\u662f\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9\u3002\n" + ) + stripped = R._strip_generated_card_boilerplate(text) + assert "\u6211\u7684\u771f\u5b9e\u4ecb\u7ecd" in stripped + assert "\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9" in stripped + assert "git clone" not in stripped + + def test_handles_html_headings(self, R): + text = ( + "

\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002

\n" + "

pip install modelscope

\n" + ) + assert R._strip_generated_card_boilerplate(text).strip() == "" + + +class TestCleanReadmeForLlmPlaceholder: + def test_boilerplate_is_gone_but_frontmatter_survives(self, R): + cleaned = R.clean_readme_for_llm(PLACEHOLDER_CARD) + assert "pip install modelscope" not in cleaned + assert "git clone" not in cleaned + assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in cleaned + # Metadata the LLM still needs. + assert "base_model: krea/Krea-2-Turbo" in cleaned + assert "\u68a6\u5e7b\u5149\u5f71" in cleaned + + def test_a_real_card_keeps_its_body(self, R): + cleaned = R.clean_readme_for_llm(REAL_CARD) + assert "krea\u8138\u6a21" in cleaned + assert "\u6a21\u578b\u4ecb\u7ecd" in cleaned + + +class TestConvertReadmeToHtmlPlaceholder: + def test_a_placeholder_card_renders_to_nothing(self, R): + assert R.convert_readme_to_html(PLACEHOLDER_CARD) == "" + + def test_a_real_card_still_renders(self, R): + html = R.convert_readme_to_html(REAL_CARD) + assert "

krea\u8138\u6a21

" in html + assert "DiffSynth-Studio" in html diff --git a/tests/services/test_post_processor.py b/tests/services/test_post_processor.py index 2780ab36..feb80e13 100644 --- a/tests/services/test_post_processor.py +++ b/tests/services/test_post_processor.py @@ -1042,3 +1042,82 @@ class TestDeterministicFallbacks: assert "tags" not in applied assert "notes" not in applied assert "usage_tips" not in applied + + +class TestPlaceholderCardDescription: + """A site-generated placeholder card must not become the description.""" + + MODELSCOPE_METADATA = { + "from_civitai": False, + "source_platform": "modelscope", + "source_url": "https://modelscope.cn/models/user/repo", + } + + PLACEHOLDER_README = """--- +base_model: krea/Krea-2-Turbo +--- +### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重,可浏览“模型文件”页面获取。 +#### 您可以通过如下git clone命令,或者ModelScope SDK来下载模型 + +SDK下载 +```bash +pip install modelscope +``` + +

如果您是本模型的贡献者,我们邀请您根据文档及时完善模型卡片内容。

+""" + + LLM_OUTPUT = { + "base_model": "Krea 2", + "trigger_words": [], + "short_description": "一个 Krea 2 人像 LoRA。", + "tags": [], + "recommended_width": 0, + "recommended_height": 0, + "preview_url": "", + "notes": "", + "usage_tips": "{}", + "confidence": "medium", + } + + @pytest.mark.asyncio + async def test_description_holds_only_the_author_summary(self, processor): + context = ModelCardContext(description="权重0.5-1.2。") + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=None), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=self.LLM_OUTPUT, + metadata=dict(self.MODELSCOPE_METADATA), + readme_content=self.PLACEHOLDER_README, + source_context=context, + ) + + description = mock_apply.call_args[0][1]["modelDescription"] + assert description == "

权重0.5-1.2。

" + assert "pip install modelscope" not in description + assert "git clone" not in description + assert "贡献者" not in description + + @pytest.mark.asyncio + async def test_placeholder_card_alone_writes_no_description(self, processor): + """Without an author summary there is nothing worth storing.""" + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=None), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=self.LLM_OUTPUT, + metadata=dict(self.MODELSCOPE_METADATA), + readme_content=self.PLACEHOLDER_README, + source_context=ModelCardContext(), + ) + + assert "modelDescription" not in mock_apply.call_args[0][1]