feat(agent): fix extract_relevant_section false positives, add validation pipeline audit

- extract_relevant_section: raise token threshold >3, verify anchor
  sections contain basename, require 2+ heading token overlaps, skip
  TOC-style headings (markdown links), verify heading section size
- metadata_constructor: parse repo_id,model_name.safetensors format
  so model_path basename matches real filename
- config: replace hardcoded SUPPORTED_BASE_MODELS with dynamic
  init_supported_base_models() using production list_base_models()
- preprocessing_auditor: new Phase 1.5 audit module — fetches each
  README, runs extract_relevant_section + clean_readme_for_llm,
  records stats and flags, saves raw READMEs for cross-reference
- run_validation: integrate audit phase, add --audit-only mode,
  add LLM config consistency check, add ComfyUI root to sys.path
- report_generator: add Preprocessing Audit and Config Warnings
  sections to both markdown and JSON reports
This commit is contained in:
Will Miao
2026-07-05 11:18:48 +08:00
parent dd3aa97d0a
commit 8fb00998a7
6 changed files with 891 additions and 86 deletions

View File

@@ -675,8 +675,10 @@ def extract_relevant_section(
lines = readme_content.split("\n")
n = len(lines)
basename_lower = model_basename.lower()
# Tokens from the basename split on common separators
tokens = {t for t in re.split(r"[_\-.\s]+", basename_lower) if len(t) > 2}
# Tokens from the basename split on common separators.
# Exclude tokens of length ≤ 3 — 2-3 char tokens (e.g. "cry", "myjs")
# are too short to discriminate between different models in collection repos.
tokens = {t for t in re.split(r"[_\-.\s]+", basename_lower) if len(t) > 3}
# ------------------------------------------------------------------
# Strategy 1: Find a download link containing the basename
@@ -695,7 +697,13 @@ def extract_relevant_section(
if m:
aid = m.group(1).lower()
if any(token in aid for token in tokens):
return _extract_section(lines, idx, context_lines)
section = _extract_section(lines, idx, context_lines)
# Verify the extracted section actually mentions the model —
# short anchor IDs can coincidentally match tokens from
# unrelated models (e.g. "myjs" matching a different LoRA).
if basename_lower in section.lower():
return section
# False positive — continue searching
# ------------------------------------------------------------------
# Strategy 3: Find an HTML or markdown heading with overlapping tokens
@@ -712,10 +720,28 @@ def extract_relevant_section(
if mm:
heading_text = mm.group(1)
if heading_text:
# Skip TOC-style entries where the heading text is a markdown
# link or bullet list item (e.g. "### - [model_name](url)").
# These are table-of-contents entries, not real section headers.
stripped = heading_text.strip()
if stripped.startswith("- [") or re.match(r"^\[.+?\]\(.+?\)", stripped):
continue
heading_lower = heading_text.lower()
# Check if any token appears in the heading
if any(token in heading_lower for token in tokens):
return _extract_section(lines, idx, context_lines)
# Require at least 2 token overlaps, or the full basename as a
# substring of the heading. A single 4-5 char token match is
# too weak — e.g. "devil" matching "dante_devil_may_cry" when
# the actual model is "vergil_devil_may_cry", or "image" matching
# a table-of-contents heading.
matching = [t for t in tokens if t in heading_lower]
if len(matching) >= 2 or basename_lower in heading_lower:
section = _extract_section(lines, idx, context_lines)
# Verify the section contains the model name — headings in
# TOC areas can match tokens but produce a tiny irrelevant
# section (e.g. "### - [z_image_turbo](url)" matched by
# tokens "lora" and "turbo").
if basename_lower in section.lower() or len(section) > max(500, context_lines * 20):
return section
# ------------------------------------------------------------------
# Fallback: return FULL readme