mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 03:01:27 -03:00
fix(agent): drop site-generated placeholder model cards
A repository whose uploader wrote no README still gets a card. ModelScope
answers with a placeholder notice ("the contributor provided no further
description"), a block of SDK/git download instructions, and a closing
invitation to complete the card. None of it describes the model, yet it was
being sent to the LLM and, worse, stored as `modelDescription` — so a Krea 2
LoRA whose only real text was the author's summary showed 841 characters of
`pip install modelscope` scaffolding on its description tab.
Add `_strip_generated_card_boilerplate()` and run it on both paths:
`clean_readme_for_llm()` (the prompt) and `convert_readme_to_html()` (the
stored description). Markers are matched as substrings because the notices
are prose and because non-Latin scripts are not space-delimited — the notice
continues with a full-width period, so the `title == keyword` matching used
for the English boilerplate headings never fired.
A marker heading takes its whole section with it, which is what removes the
download block hanging off the notice; a stand-alone notice line is dropped
alone. Content the author added later, under a heading of equal or higher
level, is kept, so a card that was improved after the placeholder is not
thrown away.
Verified on the live repositories: the placeholder card's description went
from 841 characters to the 86-character author summary, while the repo with
a genuinely author-written card is byte-for-byte unchanged.
This commit is contained in:
@@ -392,12 +392,18 @@ def _extract_frontmatter(text: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def convert_readme_to_html(markdown_text: str | None) -> str:
|
def convert_readme_to_html(markdown_text: str | None) -> str:
|
||||||
"""Convert HF README markdown to sanitised HTML."""
|
"""Convert HF README markdown to sanitised HTML.
|
||||||
|
|
||||||
|
Site-generated placeholder notices are dropped here too, so a repository
|
||||||
|
whose author wrote nothing does not store the download instructions as its
|
||||||
|
model description; the result is an empty string in that case.
|
||||||
|
"""
|
||||||
if not markdown_text:
|
if not markdown_text:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
text = markdown_text
|
text = markdown_text
|
||||||
text = _strip_frontmatter(text)
|
text = _strip_frontmatter(text)
|
||||||
|
text = _strip_generated_card_boilerplate(text)
|
||||||
text = _strip_gallery(text)
|
text = _strip_gallery(text)
|
||||||
text = _strip_badge_images(text)
|
text = _strip_badge_images(text)
|
||||||
text = _strip_html_comments(text)
|
text = _strip_html_comments(text)
|
||||||
@@ -444,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150
|
|||||||
#: Minimum consecutive enumeration lines to trigger massive-list stripping.
|
#: Minimum consecutive enumeration lines to trigger massive-list stripping.
|
||||||
_MASSIVE_LIST_THRESHOLD = 8
|
_MASSIVE_LIST_THRESHOLD = 8
|
||||||
|
|
||||||
|
#: Substrings identifying text a *site* generated to fill a model card whose
|
||||||
|
#: author wrote nothing, as opposed to the author's own content. ModelScope
|
||||||
|
#: renders such a card as a placeholder notice, a block of SDK/git download
|
||||||
|
#: instructions, and a closing invitation to improve the card.
|
||||||
|
#:
|
||||||
|
#: Matched as substrings rather than whole headings because the notices are
|
||||||
|
#: prose, and because non-Latin scripts are not space-delimited — the notice
|
||||||
|
#: continues with a full-width period, so the ``title == kw`` style matching
|
||||||
|
#: used for :data:`_BOILERPLATE_HEADERS` would never fire.
|
||||||
|
_GENERATED_CARD_MARKERS: tuple[str, ...] = (
|
||||||
|
"当前模型的贡献者未提供更加详细的模型介绍",
|
||||||
|
"您可以通过如下",
|
||||||
|
"如果您是本模型的贡献者",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_generated_card_boilerplate(text: str) -> str:
|
||||||
|
"""Remove the notices a site generates to fill an empty model card.
|
||||||
|
|
||||||
|
A repository whose uploader wrote no README still gets a card: ModelScope
|
||||||
|
answers with "the contributor provided no further description", the SDK
|
||||||
|
and git download commands, and an invitation to complete the card. None
|
||||||
|
of it describes the model, yet it was landing in both the LLM prompt and
|
||||||
|
the stored description.
|
||||||
|
|
||||||
|
A notice that is a heading takes its whole section with it, so the
|
||||||
|
download block goes too; a stand-alone notice line is dropped on its own.
|
||||||
|
Content the author added later — under a heading of equal or higher
|
||||||
|
level — is kept, so an improved card is not thrown away.
|
||||||
|
"""
|
||||||
|
|
||||||
|
lines = text.split("\n")
|
||||||
|
out: list[str] = []
|
||||||
|
skip_until_level: int | None = None
|
||||||
|
|
||||||
|
for line in lines:
|
||||||
|
level = _heading_level(line)
|
||||||
|
|
||||||
|
if any(marker in line for marker in _GENERATED_CARD_MARKERS):
|
||||||
|
if level > 0:
|
||||||
|
skip_until_level = level
|
||||||
|
continue
|
||||||
|
|
||||||
|
if skip_until_level is not None:
|
||||||
|
if level > 0 and level <= skip_until_level:
|
||||||
|
skip_until_level = None
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
|
out.append(line)
|
||||||
|
|
||||||
|
return "\n".join(out)
|
||||||
|
|
||||||
|
|
||||||
def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
|
def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
|
||||||
"""Clean a HF README for injection into an LLM metadata-extraction prompt.
|
"""Clean a HF README for injection into an LLM metadata-extraction prompt.
|
||||||
@@ -453,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
|
|||||||
|
|
||||||
* ``widget:`` YAML block (example prompts + output URLs)
|
* ``widget:`` YAML block (example prompts + output URLs)
|
||||||
* ``<Gallery />`` tags and wrappers
|
* ``<Gallery />`` tags and wrappers
|
||||||
|
* Site-generated placeholder notices for a card the author never wrote
|
||||||
|
(see :func:`_strip_generated_card_boilerplate`)
|
||||||
* Fenced code blocks (Python / bash / bibtex / yaml)
|
* Fenced code blocks (Python / bash / bibtex / yaml)
|
||||||
* Standalone ```` image lines and ``<img>`` tags
|
* Standalone ```` image lines and ``<img>`` tags
|
||||||
* Training-parameter tables
|
* Training-parameter tables
|
||||||
@@ -478,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
|
|||||||
# Order matters — broader strips first, then finer ones.
|
# Order matters — broader strips first, then finer ones.
|
||||||
text = _strip_gallery(text)
|
text = _strip_gallery(text)
|
||||||
text = _strip_widget_section(text)
|
text = _strip_widget_section(text)
|
||||||
|
text = _strip_generated_card_boilerplate(text)
|
||||||
text = _strip_fenced_code_blocks(text)
|
text = _strip_fenced_code_blocks(text)
|
||||||
text = _strip_standalone_images(text)
|
text = _strip_standalone_images(text)
|
||||||
text = _strip_training_tables(text)
|
text = _strip_training_tables(text)
|
||||||
|
|||||||
@@ -490,3 +490,118 @@ class TestStripFencedCodeBlocks:
|
|||||||
def test_pattern(self, R):
|
def test_pattern(self, R):
|
||||||
text = "x\n```yaml\nkey: val\n```\ny"
|
text = "x\n```yaml\nkey: val\n```\ny"
|
||||||
assert "key: val" not in R._strip_fenced_code_blocks(text)
|
assert "key: val" not in R._strip_fenced_code_blocks(text)
|
||||||
|
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# Site-generated placeholder cards
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
#: The card ModelScope renders when the uploader wrote no README. Copied from
|
||||||
|
#: a live repository so the marker strings stay honest.
|
||||||
|
PLACEHOLDER_CARD = """---
|
||||||
|
base_model: krea/Krea-2-Turbo
|
||||||
|
license: Apache License 2.0
|
||||||
|
tags:
|
||||||
|
- LoRA
|
||||||
|
- text-to-image
|
||||||
|
- \u68a6\u5e7b\u5149\u5f71
|
||||||
|
---
|
||||||
|
### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\u6a21\u578b\u6587\u4ef6\u548c\u6743\u91cd\uff0c\u53ef\u6d4f\u89c8\u201c\u6a21\u578b\u6587\u4ef6\u201d\u9875\u9762\u83b7\u53d6\u3002
|
||||||
|
#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\uff0c\u6216\u8005ModelScope SDK\u6765\u4e0b\u8f7d\u6a21\u578b
|
||||||
|
|
||||||
|
SDK\u4e0b\u8f7d
|
||||||
|
```bash
|
||||||
|
#\u5b89\u88c5ModelScope
|
||||||
|
pip install modelscope
|
||||||
|
```
|
||||||
|
Git\u4e0b\u8f7d
|
||||||
|
```
|
||||||
|
#Git\u6a21\u578b\u4e0b\u8f7d
|
||||||
|
git clone https://www.modelscope.cn/yan303145427/krea2-CcFQWZ-Portrait.git
|
||||||
|
```
|
||||||
|
|
||||||
|
<p style="color: lightgrey;">\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u6211\u4eec\u9080\u8bf7\u60a8\u6839\u636e<a href="x">\u6a21\u578b\u8d21\u732e\u6587\u6863</a>\uff0c\u53ca\u65f6\u5b8c\u5584\u6a21\u578b\u5361\u7247\u5185\u5bb9\u3002</p>
|
||||||
|
"""
|
||||||
|
|
||||||
|
#: A real, author-written card (ModelScope AIGC training output).
|
||||||
|
REAL_CARD = """---
|
||||||
|
base_model: krea/Krea-2-Turbo
|
||||||
|
---
|
||||||
|
# krea\u8138\u6a21
|
||||||
|
|
||||||
|
## \u6a21\u578b\u4ecb\u7ecd
|
||||||
|
|
||||||
|
\u672c\u6a21\u578b\u4f9d\u6258\u9b54\u642d\u793e\u533a\u5b8c\u6210\u8bad\u7ec3\u3002
|
||||||
|
|
||||||
|
## \u63a8\u7406\u4ee3\u7801
|
||||||
|
|
||||||
|
\u5b89\u88c5 DiffSynth-Studio\uff1a
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestStripGeneratedCardBoilerplate:
|
||||||
|
def test_removes_the_whole_placeholder_body(self, R):
|
||||||
|
stripped = R._strip_generated_card_boilerplate(PLACEHOLDER_CARD)
|
||||||
|
assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in stripped
|
||||||
|
assert "SDK\u4e0b\u8f7d" not in stripped
|
||||||
|
assert "git clone" not in stripped
|
||||||
|
assert "\u9080\u8bf7\u60a8" not in stripped
|
||||||
|
# The frontmatter is the only thing that survives.
|
||||||
|
assert "base_model: krea/Krea-2-Turbo" in stripped
|
||||||
|
|
||||||
|
def test_leaves_a_real_card_untouched(self, R):
|
||||||
|
assert R._strip_generated_card_boilerplate(REAL_CARD) == REAL_CARD
|
||||||
|
|
||||||
|
def test_drops_a_standalone_invitation_line(self, R):
|
||||||
|
text = "real body\n<p>\u5982\u679c\u60a8\u662f\u672c\u6a21\u578b\u7684\u8d21\u732e\u8005\uff0c\u8bf7\u5b8c\u5584</p>\nmore body"
|
||||||
|
stripped = R._strip_generated_card_boilerplate(text)
|
||||||
|
assert "real body" in stripped
|
||||||
|
assert "more body" in stripped
|
||||||
|
assert "\u8d21\u732e\u8005" not in stripped
|
||||||
|
|
||||||
|
def test_keeps_content_added_after_the_placeholder(self, R):
|
||||||
|
"""An author who later wrote a real section must not lose it."""
|
||||||
|
text = (
|
||||||
|
"### \u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002\n"
|
||||||
|
"#### \u60a8\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0bgit clone\u547d\u4ee4\u4e0b\u8f7d\u6a21\u578b\n"
|
||||||
|
"```\ngit clone x\n```\n"
|
||||||
|
"## \u6211\u7684\u771f\u5b9e\u4ecb\u7ecd\n"
|
||||||
|
"\u8fd9\u662f\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9\u3002\n"
|
||||||
|
)
|
||||||
|
stripped = R._strip_generated_card_boilerplate(text)
|
||||||
|
assert "\u6211\u7684\u771f\u5b9e\u4ecb\u7ecd" in stripped
|
||||||
|
assert "\u4f5c\u8005\u540e\u6765\u8865\u5199\u7684\u5185\u5bb9" in stripped
|
||||||
|
assert "git clone" not in stripped
|
||||||
|
|
||||||
|
def test_handles_html_headings(self, R):
|
||||||
|
text = (
|
||||||
|
"<h3>\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005\u672a\u63d0\u4f9b\u66f4\u52a0\u8be6\u7ec6\u7684\u6a21\u578b\u4ecb\u7ecd\u3002</h3>\n"
|
||||||
|
"<p>pip install modelscope</p>\n"
|
||||||
|
)
|
||||||
|
assert R._strip_generated_card_boilerplate(text).strip() == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestCleanReadmeForLlmPlaceholder:
|
||||||
|
def test_boilerplate_is_gone_but_frontmatter_survives(self, R):
|
||||||
|
cleaned = R.clean_readme_for_llm(PLACEHOLDER_CARD)
|
||||||
|
assert "pip install modelscope" not in cleaned
|
||||||
|
assert "git clone" not in cleaned
|
||||||
|
assert "\u5f53\u524d\u6a21\u578b\u7684\u8d21\u732e\u8005" not in cleaned
|
||||||
|
# Metadata the LLM still needs.
|
||||||
|
assert "base_model: krea/Krea-2-Turbo" in cleaned
|
||||||
|
assert "\u68a6\u5e7b\u5149\u5f71" in cleaned
|
||||||
|
|
||||||
|
def test_a_real_card_keeps_its_body(self, R):
|
||||||
|
cleaned = R.clean_readme_for_llm(REAL_CARD)
|
||||||
|
assert "krea\u8138\u6a21" in cleaned
|
||||||
|
assert "\u6a21\u578b\u4ecb\u7ecd" in cleaned
|
||||||
|
|
||||||
|
|
||||||
|
class TestConvertReadmeToHtmlPlaceholder:
|
||||||
|
def test_a_placeholder_card_renders_to_nothing(self, R):
|
||||||
|
assert R.convert_readme_to_html(PLACEHOLDER_CARD) == ""
|
||||||
|
|
||||||
|
def test_a_real_card_still_renders(self, R):
|
||||||
|
html = R.convert_readme_to_html(REAL_CARD)
|
||||||
|
assert "<h1>krea\u8138\u6a21</h1>" in html
|
||||||
|
assert "DiffSynth-Studio" in html
|
||||||
|
|||||||
@@ -1042,3 +1042,82 @@ class TestDeterministicFallbacks:
|
|||||||
assert "tags" not in applied
|
assert "tags" not in applied
|
||||||
assert "notes" not in applied
|
assert "notes" not in applied
|
||||||
assert "usage_tips" not in applied
|
assert "usage_tips" not in applied
|
||||||
|
|
||||||
|
|
||||||
|
class TestPlaceholderCardDescription:
|
||||||
|
"""A site-generated placeholder card must not become the description."""
|
||||||
|
|
||||||
|
MODELSCOPE_METADATA = {
|
||||||
|
"from_civitai": False,
|
||||||
|
"source_platform": "modelscope",
|
||||||
|
"source_url": "https://modelscope.cn/models/user/repo",
|
||||||
|
}
|
||||||
|
|
||||||
|
PLACEHOLDER_README = """---
|
||||||
|
base_model: krea/Krea-2-Turbo
|
||||||
|
---
|
||||||
|
### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重,可浏览“模型文件”页面获取。
|
||||||
|
#### 您可以通过如下git clone命令,或者ModelScope SDK来下载模型
|
||||||
|
|
||||||
|
SDK下载
|
||||||
|
```bash
|
||||||
|
pip install modelscope
|
||||||
|
```
|
||||||
|
|
||||||
|
<p style="color: lightgrey;">如果您是本模型的贡献者,我们邀请您根据文档及时完善模型卡片内容。</p>
|
||||||
|
"""
|
||||||
|
|
||||||
|
LLM_OUTPUT = {
|
||||||
|
"base_model": "Krea 2",
|
||||||
|
"trigger_words": [],
|
||||||
|
"short_description": "一个 Krea 2 人像 LoRA。",
|
||||||
|
"tags": [],
|
||||||
|
"recommended_width": 0,
|
||||||
|
"recommended_height": 0,
|
||||||
|
"preview_url": "",
|
||||||
|
"notes": "",
|
||||||
|
"usage_tips": "{}",
|
||||||
|
"confidence": "medium",
|
||||||
|
}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_description_holds_only_the_author_summary(self, processor):
|
||||||
|
context = ModelCardContext(description="权重0.5-1.2。")
|
||||||
|
with (
|
||||||
|
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
|
||||||
|
mock.patch("py.metadata_ops.download_preview", return_value=None),
|
||||||
|
mock.patch("py.metadata_ops.refresh_cache"),
|
||||||
|
):
|
||||||
|
await processor.process(
|
||||||
|
skill_name="enrich_hf_metadata",
|
||||||
|
model_path="/p.safetensors",
|
||||||
|
llm_output=self.LLM_OUTPUT,
|
||||||
|
metadata=dict(self.MODELSCOPE_METADATA),
|
||||||
|
readme_content=self.PLACEHOLDER_README,
|
||||||
|
source_context=context,
|
||||||
|
)
|
||||||
|
|
||||||
|
description = mock_apply.call_args[0][1]["modelDescription"]
|
||||||
|
assert description == "<p>权重0.5-1.2。</p>"
|
||||||
|
assert "pip install modelscope" not in description
|
||||||
|
assert "git clone" not in description
|
||||||
|
assert "贡献者" not in description
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_placeholder_card_alone_writes_no_description(self, processor):
|
||||||
|
"""Without an author summary there is nothing worth storing."""
|
||||||
|
with (
|
||||||
|
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
|
||||||
|
mock.patch("py.metadata_ops.download_preview", return_value=None),
|
||||||
|
mock.patch("py.metadata_ops.refresh_cache"),
|
||||||
|
):
|
||||||
|
await processor.process(
|
||||||
|
skill_name="enrich_hf_metadata",
|
||||||
|
model_path="/p.safetensors",
|
||||||
|
llm_output=self.LLM_OUTPUT,
|
||||||
|
metadata=dict(self.MODELSCOPE_METADATA),
|
||||||
|
readme_content=self.PLACEHOLDER_README,
|
||||||
|
source_context=ModelCardContext(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "modelDescription" not in mock_apply.call_args[0][1]
|
||||||
|
|||||||
Reference in New Issue
Block a user