feat(agent): fix extract_relevant_section false positives, add validation pipeline audit

- extract_relevant_section: raise token threshold >3, verify anchor
  sections contain basename, require 2+ heading token overlaps, skip
  TOC-style headings (markdown links), verify heading section size
- metadata_constructor: parse repo_id,model_name.safetensors format
  so model_path basename matches real filename
- config: replace hardcoded SUPPORTED_BASE_MODELS with dynamic
  init_supported_base_models() using production list_base_models()
- preprocessing_auditor: new Phase 1.5 audit module — fetches each
  README, runs extract_relevant_section + clean_readme_for_llm,
  records stats and flags, saves raw READMEs for cross-reference
- run_validation: integrate audit phase, add --audit-only mode,
  add LLM config consistency check, add ComfyUI root to sys.path
- report_generator: add Preprocessing Audit and Config Warnings
  sections to both markdown and JSON reports

tests/enrich_hf_validation/run_validation.py

@@ -3,7 +3,7 @@
 
 Usage::
 
-    # Full run (100 models, serial, ~1-2 h)
+    # Full run (44 models, serial, ~1-2 h)
     python -m tests.enrich_hf_validation.run_validation \\
         --output /tmp/hf_enrich_validation
 
@@ -13,6 +13,9 @@ Usage::
     # Resume from a previous partial run
     python -m tests.enrich_hf_validation.run_validation --resume
 
+    # Audit preprocessing only (no LLM calls, fast)
+    python -m tests.enrich_hf_validation.run_validation --audit-only
+
     # Custom settings file
     python -m tests.enrich_hf_validation.run_validation \\
         --settings /custom/path/settings.json
@@ -27,7 +30,7 @@ import logging
 import os
 import sys
 import time
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
 
 # Ensure the project root is on sys.path so that ``from py import ...`` works.
 _PROJECT_ROOT = os.path.normpath(
@@ -36,8 +39,18 @@ _PROJECT_ROOT = os.path.normpath(
 if _PROJECT_ROOT not in sys.path:
     sys.path.insert(0, _PROJECT_ROOT)
 
-from tests.enrich_hf_validation.config import load_settings
+# Add ComfyUI root to sys.path so ``folder_paths`` can be imported.
+# Project layout: ComfyUI/custom_nodes/ComfyUI-Lora-Manager/
+_COMFYUI_ROOT = os.path.normpath(os.path.join(_PROJECT_ROOT, "..", ".."))
+if _COMFYUI_ROOT not in sys.path:
+    sys.path.insert(0, _COMFYUI_ROOT)
+
+from tests.enrich_hf_validation.config import (
+    init_supported_base_models,
+    load_settings,
+)
 from tests.enrich_hf_validation.metadata_constructor import (
+    RepoEntry,
     create_all_initial_metadata,
     load_repo_ids,
 )
@@ -46,6 +59,10 @@ from tests.enrich_hf_validation.evaluation_engine import (
     aggregate_scores,
     evaluate_batch,
 )
+from tests.enrich_hf_validation.preprocessing_auditor import (
+    audit_records_to_serializable,
+    run_audit,
+)
 from tests.enrich_hf_validation.report_generator import (
     generate_markdown_report,
     save_json_report,
@@ -70,8 +87,8 @@ def _parse_args(argv: List[str]) -> argparse.Namespace:
     )
     parser.add_argument(
         "--models",
-        default="~/Documents/hf_lora_models.txt",
-        help="Path to the HF repo ID list (one per line)",
+        default="~/Documents/hf_lora_models_with_safetensors.txt",
+        help="Path to the HF repo entries file (format: repo_id, model_name.safetensors per line)",
     )
     parser.add_argument(
         "--settings",
@@ -99,6 +116,11 @@ def _parse_args(argv: List[str]) -> argparse.Namespace:
         action="store_true",
         help="Skip enrichment phase (evaluate existing metadata only)",
     )
+    parser.add_argument(
+        "--audit-only",
+        action="store_true",
+        help="Run preprocessing audit only (no enrichment, no evaluation)",
+    )
     parser.add_argument(
         "--timeout",
         type=int,
@@ -125,6 +147,117 @@ def _phase_header(label: str) -> None:
     print(sep, file=sys.stderr)
 
 
+# ---------------------------------------------------------------------------
+# Read back LLM config after enrichment (for consistency reporting)
+# ---------------------------------------------------------------------------
+
+
+def _get_actual_llm_config() -> Dict[str, str]:
+    """Read what LLMService is actually using, if initialized.
+
+    Only meaningful when called AFTER enrichment has started (i.e. after
+    ``AgentService.get_instance()`` has been called).
+    """
+    try:
+        from py.services.llm_service import LLMService
+
+        instance = LLMService._instance
+        if instance is None:
+            return {"status": "not initialized"}
+        cfg = instance._get_config()
+        return {
+            "provider": cfg.get("provider", ""),
+            "model": cfg.get("model", ""),
+            "api_base": cfg.get("api_base", ""),
+        }
+    except Exception as exc:
+        return {"status": f"error: {exc}"}
+
+
+def _compare_llm_config(
+    pipeline_cfg: Dict[str, Any],
+    actual_cfg: Dict[str, str],
+) -> List[str]:
+    """Compare pipeline-loaded vs LLMService-used config.
+
+    Returns warning messages if they differ.
+    """
+    warnings: List[str] = []
+    if not actual_cfg or actual_cfg.get("status", "") == "not initialized":
+        warnings.append(
+            "LLMService was not initialized during this run — cannot verify "
+            "config consistency."
+        )
+        return warnings
+
+    field_map = [
+        ("llm_provider", "provider"),
+        ("llm_model", "model"),
+        ("llm_api_base", "api_base"),
+    ]
+    for pipeline_key, llm_key in field_map:
+        pv = (pipeline_cfg.get(pipeline_key) or "").strip()
+        lv = (actual_cfg.get(llm_key) or "").strip()
+        if pv and lv and pv != lv:
+            warnings.append(
+                f"LLM config mismatch: --settings has '{pv}' for {pipeline_key}, "
+                f"but LLMService uses '{lv}'. "
+                f"The pipeline's --settings path ({pipeline_cfg.get('settings_path', '?')}) "
+                "may differ from where SettingsManager reads."
+            )
+    if not warnings and actual_cfg:
+        warnings.append(
+            "✅ LLM config matches between pipeline --settings and LLMService."
+        )
+    return warnings
+
+
+# ---------------------------------------------------------------------------
+# Phase 1.5: preprocessing audit
+# ---------------------------------------------------------------------------
+
+
+async def _run_preprocessing_audit(
+    entries: List[RepoEntry],
+    output_dir: str,
+) -> Dict[str, Any]:
+    """Execute the preprocessing audit and save results."""
+    _phase_header("Preprocessing audit")
+    print(f"  Auditing {len(entries)} repos ...", file=sys.stderr)
+
+    readmes_dir = os.path.join(output_dir, "readmes")
+    t0 = time.perf_counter()
+    records, summary = await run_audit(entries, readmes_dir=readmes_dir)
+    elapsed = time.perf_counter() - t0
+
+    # Save audit data
+    audit_path = os.path.join(output_dir, "preprocessing_audit.json")
+    with open(audit_path, "w", encoding="utf-8") as fh:
+        json.dump(
+            {
+                "summary": summary,
+                "records": audit_records_to_serializable(records),
+            },
+            fh,
+            indent=2,
+            ensure_ascii=False,
+        )
+
+    print(f"  Audit complete: {len(records)} repos in {elapsed:.0f}s", file=sys.stderr)
+    print(f"  Section extraction activated: {summary.get('section_extraction_pct', 0)}%", file=sys.stderr)
+    print(f"  Basename in extracted section: {summary.get('basename_in_section_pct', 0)}%", file=sys.stderr)
+    print(f"  Avg compression: {summary.get('avg_compression_pct', 0)}%", file=sys.stderr)
+    print(f"  Avg cleaned length: {summary.get('avg_cleaned_length', 0)} chars", file=sys.stderr)
+    print(f"  Audit data: {audit_path}", file=sys.stderr)
+
+    if summary.get("top_flags"):
+        print("  Top flags:", file=sys.stderr)
+        for flag, count in summary["top_flags"][:5]:
+            print(f"    - {flag}: {count}x", file=sys.stderr)
+
+    return summary
+
+
 async def _run_enrichment(
     model_paths: List[str],
     repos: List[str],
@@ -170,7 +303,7 @@ def _collect_enriched_metadata(
     errors, metadata.
     """
     enriched: List[Dict[str, Any]] = []
-    # Build a lookup from repo_id → enrichment result
+    # Build a lookup from repo_id to enrichment result
     result_lookup: Dict[str, Dict[str, Any]] = {}
     for r in results:
         result_lookup[r["repo_id"]] = r
@@ -214,29 +347,43 @@ async def main(argv: List[str]) -> int:
     output_dir = os.path.abspath(os.path.expanduser(args.output))
     os.makedirs(output_dir, exist_ok=True)
 
+    # ---- Phase 0: Initialise shared state ----
+    _phase_header("Initialise")
     settings = load_settings(args.settings)
     logger.info(
-        "LLM config: provider=%s model=%s api_base=%s",
+        "LLM config from --settings: provider=%s model=%s api_base=%s",
         settings["llm_provider"],
         settings["llm_model"],
         settings["llm_api_base"],
     )
+    # Load the production base model list (replaces the old hardcoded list)
+    await init_supported_base_models()
 
-    # ---- Phase 1: Load repo IDs & construct initial metadata ----
-    _phase_header("Load repo IDs & construct initial metadata")
-    repos = load_repo_ids(args.models, max_models=args.sample if args.sample > 0 else None)
-    model_paths = create_all_initial_metadata(
-        repos, output_dir, skip_existing=True,
+    # ---- Load entries ----
+    _phase_header("Load repo entries & construct initial metadata")
+    entries = load_repo_ids(args.models, max_models=args.sample if args.sample > 0 else None)
+    model_paths, repo_ids = create_all_initial_metadata(
+        entries, output_dir, skip_existing=True,
     )
     print(f"  {len(model_paths)} repos ready", file=sys.stderr)
 
+    # ---- Phase 1.5: Preprocessing audit ----
+    audit_summary: Dict[str, Any] = {}
+    t_start = time.perf_counter()
+    audit_summary = await _run_preprocessing_audit(entries, output_dir)
+
+    if args.audit_only:
+        total_wall = time.perf_counter() - t_start
+        print(f"\n  Audit-only done in {total_wall:.0f}s", file=sys.stderr)
+        print(f"  Audit data: {output_dir}/preprocessing_audit.json", file=sys.stderr)
+        return 0
+
     # ---- Phase 2: Enrichment ----
     enrichment_results: List[Dict[str, Any]] = []
-    t_start = time.perf_counter()
     if not args.no_enrich:
         _phase_header("Enrich metadata via LLM")
         enrichment_out = await _run_enrichment(
-            model_paths, repos, output_dir, args.timeout, args.verbose,
+            model_paths, repo_ids, output_dir, args.timeout, args.verbose,
         )
         enrichment_results = enrichment_out["results"]
     else:
@@ -246,7 +393,7 @@ async def main(argv: List[str]) -> int:
 
     # ---- Phase 3: Evaluation ----
     _phase_header("Evaluate enriched metadata")
-    enriched = _collect_enriched_metadata(model_paths, repos, enrichment_results)
+    enriched = _collect_enriched_metadata(model_paths, repo_ids, enrichment_results)
     scores = evaluate_batch(enriched)
     agg = aggregate_scores(scores)
     print(
@@ -274,8 +421,18 @@ async def main(argv: List[str]) -> int:
                 "max_s": round(max(durations), 1),
             }
 
-    save_json_report(agg, scores, enrichment_results, output_dir, duration_summary)
-    generate_markdown_report(agg, scores, output_dir, duration_summary)
+    # Check LLM config consistency after enrichment (LLMService is now initialized)
+    actual_llm_cfg = _get_actual_llm_config()
+    config_warnings = _compare_llm_config(settings, actual_llm_cfg)
+
+    save_json_report(
+        agg, scores, enrichment_results, output_dir, duration_summary,
+        audit_summary=audit_summary, config_warnings=config_warnings,
+    )
+    generate_markdown_report(
+        agg, scores, output_dir, duration_summary,
+        audit_summary=audit_summary, config_warnings=config_warnings,
+    )
 
     # ---- Final summary ----
     total_wall = time.perf_counter() - t_start