Merge pull request #1013 from willmiao/agent

Hugging Face model metadata AI enrichment

tests/__init__.py

@@ -0,0 +1 @@
+# Test suite package.

tests/enrich_hf_validation/__init__.py

@@ -0,0 +1 @@
+# HF Metadata Enrichment validation suite.

tests/enrich_hf_validation/config.py

@@ -0,0 +1,133 @@
+"""Configuration for the HF metadata enrichment validation suite.
+
+Loads user settings, defines paths, and pulls constants from the main
+codebase (``py.utils.constants``).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Default paths
+# ---------------------------------------------------------------------------
+
+_DEFAULT_MODELS_FILE = os.path.join(
+    os.path.dirname(__file__), "test_data", "hf_lora_models_with_safetensors.txt"
+)
+_DEFAULT_SETTINGS_PATH = os.path.expanduser(
+    "~/.config/ComfyUI-LoRA-Manager/settings.json"
+)
+_DEFAULT_OUTPUT_DIR = "/tmp/hf_enrich_validation"
+
+# ---------------------------------------------------------------------------
+# Constants from the main codebase (copied at import time)
+# ---------------------------------------------------------------------------
+
+# Priority tags used in the LLM prompt for tag selection guidance.
+CIVITAI_MODEL_TAGS: List[str] = [
+    "character", "concept", "clothing", "realistic", "anime", "toon",
+    "furry", "style", "poses", "background", "tool", "vehicle",
+    "buildings", "objects", "assets", "animal", "action",
+]
+
+# ---------------------------------------------------------------------------
+# Base model resolution — dynamically fetched from production code
+# ---------------------------------------------------------------------------
+
+# Module-level cache — populated by init_supported_base_models().
+# Falls back to a comprehensive hardcoded list when the live fetch fails.
+SUPPORTED_BASE_MODELS: List[str] = []
+
+# Fallback base models when the production list_base_models() is unavailable.
+_FALLBACK_BASE_MODELS: List[str] = [
+    "SD 1.4", "SD 1.5", "SD 1.5 LCM", "SD 1.5 Hyper",
+    "SD 2.0", "SD 2.1",
+    "SD 3", "SD 3.5", "SD 3.5 Medium", "SD 3.5 Large", "SD 3.5 Large Turbo",
+    "SDXL 1.0", "SDXL Lightning", "SDXL Hyper",
+    "Flux.1 D", "Flux.1 S", "Flux.1 Krea", "Flux.1 Kontext",
+    "Flux.2 D", "Flux.2 Klein 9B", "Flux.2 Klein 9B-base",
+    "Flux.2 Klein 4B", "Flux.2 Klein 4B-base",
+    "AuraFlow", "Chroma", "PixArt a", "PixArt E",
+    "Hunyuan 1", "Lumina", "Kolors",
+    "NoobAI", "Illustrious", "Pony", "Pony V7",
+    "HiDream", "Qwen", "ZImageTurbo", "ZImageBase",
+    "SVD", "LTXV", "LTXV2", "LTXV 2.3",
+    "CogVideoX", "Mochi",
+    "Wan Video", "Wan Video 1.3B t2v", "Wan Video 14B t2v",
+    "Wan Video 14B i2v 480p", "Wan Video 14B i2v 720p",
+    "Wan Video 2.2 TI2V-5B", "Wan Video 2.2 T2V-A14B",
+    "Wan Video 2.2 I2V-A14B",
+    "Wan Video 2.5 T2V", "Wan Video 2.5 I2V",
+    "Hunyuan Video", "Anima", "Ernie", "Ernie Turbo",
+    "Nucleus", "Krea 2",
+]
+
+
+async def init_supported_base_models() -> None:
+    """Populate ``SUPPORTED_BASE_MODELS`` from the production codebase.
+
+    Calls ``py.metadata_ops.list_base_models()`` which merges a hardcoded
+    fallback with models fetched from the CivitAI API.  When the call
+    fails (e.g. offline, API error), falls back to ``_FALLBACK_BASE_MODELS``.
+
+    Must be called from within an async event loop (i.e. during
+    ``run_validation.main()``, not at module level).
+    """
+    try:
+        from py.metadata_ops import list_base_models
+
+        models = await list_base_models()
+        if models:
+            SUPPORTED_BASE_MODELS[:] = models
+            logger.info("Loaded %d base models from production code", len(models))
+            return
+        logger.warning("list_base_models returned empty list, using fallback")
+    except Exception as exc:
+        logger.warning("Failed to load base models from production: %s", exc)
+
+    SUPPORTED_BASE_MODELS[:] = _FALLBACK_BASE_MODELS
+    logger.info("Using fallback base model list (%d entries)", len(SUPPORTED_BASE_MODELS))
+
+
+# Placeholder values the LLM sometimes emits that should count as "empty".
+PLACEHOLDER_VALUES = frozenset({
+    "none", "null", "n/a", "unknown", "not available",
+    "not specified", "no trigger words", "no trigger word",
+})
+
+
+# ---------------------------------------------------------------------------
+# User settings loader
+# ---------------------------------------------------------------------------
+
+
+def load_settings(settings_path: str) -> Dict[str, Any]:
+    """Load LoRA Manager settings from *settings_path*.
+
+    Returns a flat dict with the LLM configuration fields that the
+    enrichment pipeline depends on.
+    """
+    path = os.path.expanduser(settings_path)
+    if not os.path.exists(path):
+        raise FileNotFoundError(
+            f"Settings file not found: {path}\n"
+            "Please provide a valid --settings path."
+        )
+
+    with open(path, "r", encoding="utf-8") as fh:
+        raw: Dict[str, Any] = json.load(fh)
+
+    # Extract LLM-relevant config
+    return {
+        "llm_provider": raw.get("llm_provider", "ollama"),
+        "llm_model": raw.get("llm_model", "qwen3.5:9b"),
+        "llm_api_base": raw.get("llm_api_base", "http://localhost:11434/v1"),
+        "llm_api_key": raw.get("llm_api_key", ""),
+        "settings_path": path,
+    }

tests/enrich_hf_validation/enrichment_runner.py

@@ -0,0 +1,208 @@
+"""Execute the ``enrich_hf_metadata`` skill serially over a list of models.
+
+Design decisions (local Ollama, no rate limits):
+
+- Sequential execution: one model at a time.  100 models at ~30-90 s/call
+  → roughly 1-2 h total.
+- Progress persisted to a JSON checkpoint file so the run can be resumed
+  with ``--resume``.
+- Per-model timeout guards against a stuck Ollama inference.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_SKILL_NAME = "enrich_hf_metadata"
+
+# How long to wait for a single LLM call before marking it timed-out.
+_PER_MODEL_TIMEOUT = 240  # seconds
+
+# ---------------------------------------------------------------------------
+# Progress checkpoint helpers
+# ---------------------------------------------------------------------------
+
+_PROGRESS_FILE = "progress.json"
+
+
+def _load_progress(output_dir: str) -> Dict[str, Any]:
+    path = os.path.join(output_dir, _PROGRESS_FILE)
+    if os.path.exists(path):
+        with open(path, "r") as fh:
+            return json.load(fh)
+    return {"completed": [], "failed": [], "timed_out": []}
+
+
+def _save_progress(output_dir: str, progress: Dict[str, Any]) -> None:
+    path = os.path.join(output_dir, _PROGRESS_FILE)
+    with open(path, "w") as fh:
+        json.dump(progress, fh, indent=2)
+
+
+# ---------------------------------------------------------------------------
+# Core runner
+# ---------------------------------------------------------------------------
+
+
+class EnrichmentRunner:
+    """Serial enrichment runner with checkpoint resume."""
+
+    def __init__(
+        self,
+        output_dir: str,
+        *,
+        per_model_timeout: int = _PER_MODEL_TIMEOUT,
+    ) -> None:
+        self._output_dir = output_dir
+        self._per_model_timeout = per_model_timeout
+        self._agent_service: Optional[Any] = None
+
+    async def _ensure_agent_service(self) -> Any:
+        """Lazy-init AgentService (expensive — needs LLMService init)."""
+        if self._agent_service is not None:
+            return self._agent_service
+        from py.services.agent.agent_service import AgentService
+
+        self._agent_service = await AgentService.get_instance()
+        return self._agent_service
+
+    async def run(
+        self,
+        model_paths: List[str],
+        repos: List[str],
+    ) -> Dict[str, Any]:
+        """Run enrichment over *model_paths* (one-by-one).
+
+        Args:
+            model_paths: model paths in the same order as *repos*.
+            repos: HF repo IDs (for display / checkpoint labelling).
+
+        Returns:
+            A dict with keys ``results``, ``progress``, ``durations``.
+        """
+        assert len(model_paths) == len(repos)
+
+        progress = _load_progress(self._output_dir)
+        completed_set = set(progress["completed"])
+        failed_set = set(progress["failed"])
+        timed_out_set = set(progress.get("timed_out", []))
+
+        agent = await self._ensure_agent_service()
+        results: List[Dict[str, Any]] = []
+        durations: Dict[str, float] = {}
+
+        total = len(model_paths)
+        processed_before = len(completed_set | failed_set | timed_out_set)
+
+        logger.info(
+            "Enrichment runner: %d models total, %d already processed",
+            total,
+            processed_before,
+        )
+
+        for idx, (model_path, repo_id) in enumerate(zip(model_paths, repos)):
+            if repo_id in completed_set:
+                logger.info("[%d/%d] SKIP (already done): %s", idx + 1, total, repo_id)
+                continue
+            if repo_id in failed_set or repo_id in timed_out_set:
+                logger.info(
+                    "[%d/%d] SKIP (previously failed/timeout): %s",
+                    idx + 1, total, repo_id,
+                )
+                continue
+
+            logger.info(
+                "[%d/%d] Enriching %s ...", idx + 1, total, repo_id,
+            )
+            t0 = time.perf_counter()
+
+            try:
+                result = await asyncio.wait_for(
+                    agent.execute_skill(
+                        skill_name=_SKILL_NAME,
+                        input_data={"model_paths": [model_path]},
+                        progress_callback=None,
+                    ),
+                    timeout=self._per_model_timeout,
+                )
+
+                elapsed = time.perf_counter() - t0
+                durations[repo_id] = round(elapsed, 2)
+
+                if result.success:
+                    completed_set.add(repo_id)
+                    progress["completed"].append(repo_id)
+                    logger.info(
+                        "  ✓ %s  (%.1f s) — %s",
+                        repo_id, elapsed, result.summary,
+                    )
+                else:
+                    failed_set.add(repo_id)
+                    progress["failed"].append(repo_id)
+                    logger.warning(
+                        "  ✗ %s  (%.1f s) — %s",
+                        repo_id, elapsed,
+                        "; ".join(result.errors) if result.errors else result.summary,
+                    )
+
+                results.append({
+                    "repo_id": repo_id,
+                    "model_path": model_path,
+                    "success": result.success,
+                    "updated_fields": result.updated_models,
+                    "errors": result.errors,
+                    "summary": result.summary,
+                    "duration_s": round(elapsed, 2),
+                })
+
+            except asyncio.TimeoutError:
+                elapsed = time.perf_counter() - t0
+                durations[repo_id] = round(elapsed, 2)
+                timed_out_set.add(repo_id)
+                progress.setdefault("timed_out", []).append(repo_id)
+                logger.warning(
+                    "  ⏱ TIMEOUT %s  (%.1f s, limit=%ds)",
+                    repo_id, elapsed, self._per_model_timeout,
+                )
+                results.append({
+                    "repo_id": repo_id,
+                    "model_path": model_path,
+                    "success": False,
+                    "errors": [f"Timeout after {self._per_model_timeout}s"],
+                    "summary": "LLM call timed out",
+                    "duration_s": round(elapsed, 2),
+                })
+
+            except Exception as exc:
+                elapsed = time.perf_counter() - t0
+                durations[repo_id] = round(elapsed, 2)
+                failed_set.add(repo_id)
+                progress["failed"].append(repo_id)
+                logger.error(
+                    "  ✗ %s  (%.1f s) — %s",
+                    repo_id, elapsed, exc,
+                )
+                results.append({
+                    "repo_id": repo_id,
+                    "model_path": model_path,
+                    "success": False,
+                    "errors": [str(exc)],
+                    "summary": f"Exception: {exc}",
+                    "duration_s": round(elapsed, 2),
+                })
+
+            # Checkpoint after each model
+            _save_progress(self._output_dir, progress)
+
+        return {
+            "results": results,
+            "progress": progress,
+            "durations": durations,
+        }

tests/enrich_hf_validation/evaluation_engine.py

@@ -0,0 +1,352 @@
+"""Evaluate enriched ``.metadata.json`` quality across multiple dimensions.
+
+Scoring rubric (per field):
+
+- **Completeness**: Is the field populated with meaningful content?
+- **Validity**: Does the value conform to expected constraints (controlled
+  vocab, non-placeholder, parsable JSON)?
+- **Accuracy**: (sub-sample only — requires manual verification against
+  the HF README).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Dict, List, Optional, Set
+
+from .config import (
+    CIVITAI_MODEL_TAGS,
+    PLACEHOLDER_VALUES,
+    SUPPORTED_BASE_MODELS,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Scoring helpers
+# ---------------------------------------------------------------------------
+
+_MIN_TAGS = 1
+_MAX_TAGS = 8
+_MIN_DESC_LENGTH = 20
+_MIN_NOTES_LENGTH = 30
+
+# Tags that the LLM sometimes emits but which are not meaningful content tags.
+_TECH_TAGS = frozenset({
+    "lora", "dreambooth", "text-to-image", "diffusers", "flux",
+    "sdxl", "checkpoint", "pytorch", "safetensors", "fine-tuning",
+    "stable-diffusion", "training", "stablediffusion",
+})
+
+
+def _is_placeholder(val: str) -> bool:
+    return val.strip().lower() in PLACEHOLDER_VALUES
+
+
+def _is_valid_trigger_words(words: List[str]) -> bool:
+    """Return True if *words* is a non-empty list of real trigger words."""
+    if not words:
+        return False
+    cleaned = [w.strip() for w in words if w.strip()]
+    if not cleaned:
+        return False
+    # Reject if ALL entries are placeholders
+    non_placeholder = [w for w in cleaned if not _is_placeholder(w)]
+    return len(non_placeholder) > 0
+
+
+def _is_valid_tags(tags: List[str]) -> bool:
+    """Return True if *tags* is a reasonable list of content tags."""
+    if not tags:
+        return False
+    cleaned = [t.strip().lower() for t in tags if t.strip()]
+    if not cleaned:
+        return False
+    # At least one tag that isn't a technical keyword
+    meaningful = [t for t in cleaned if t not in _TECH_TAGS]
+    return len(meaningful) >= _MIN_TAGS
+
+
+def _tag_priority_coverage(tags: List[str]) -> float:
+    """Fraction of tags that align with the user's priority tag vocabulary."""
+    if not tags:
+        return 0.0
+    priority_lower = {t.lower() for t in CIVITAI_MODEL_TAGS}
+    matched = sum(1 for t in tags if t.strip().lower() in priority_lower)
+    return matched / len(tags)
+
+
+# ---------------------------------------------------------------------------
+# Per-model evaluation
+# ---------------------------------------------------------------------------
+
+# Type alias for a score record
+ScoreRecord = Dict[str, Any]
+
+
+def evaluate_model(
+    metadata: Dict[str, Any],
+    model_path: str,
+    repo_id: str,
+    *,
+    enrichment_success: bool,
+    enrichment_errors: List[str],
+) -> ScoreRecord:
+    """Score a single enriched model's metadata.
+
+    Returns a dict with per-field scores, a total score, and a list of
+    flagged issues.
+    """
+    civitai = metadata.get("civitai") or {}
+    trained_words: List[str] = civitai.get("trainedWords") or metadata.get("trainedWords") or []
+    short_desc: str = civitai.get("description") or ""
+    tags: List[str] = metadata.get("tags") or []
+    notes: str = metadata.get("notes") or ""
+    usage_tips_raw: str = metadata.get("usage_tips") or "{}"
+    model_description: str = metadata.get("modelDescription") or ""
+    base_model: str = metadata.get("base_model") or ""
+    preview_url: str = metadata.get("preview_url") or ""
+    confidence: str = metadata.get("_llm_confidence") or ""
+
+    # --- base_model ---
+    base_model_valid = base_model in SUPPORTED_BASE_MODELS
+    base_model_filled = bool(base_model) and base_model != "Unknown"
+
+    # --- trigger_words (trainedWords) ---
+    triggers_valid = _is_valid_trigger_words(trained_words)
+
+    # --- short_description (civitai.description) ---
+    desc_filled = len(short_desc.strip()) >= _MIN_DESC_LENGTH
+
+    # --- tags ---
+    tags_valid = _is_valid_tags(tags)
+    tags_priority_coverage = _tag_priority_coverage(tags)
+    tags_no_technical = (
+        sum(1 for t in tags if t.strip().lower() not in _TECH_TAGS) >= _MIN_TAGS
+        if tags else False
+    )
+
+    # --- notes ---
+    notes_filled = len(notes.strip()) >= _MIN_NOTES_LENGTH
+
+    # --- usage_tips ---
+    usage_tips_valid = False
+    if usage_tips_raw.strip() and usage_tips_raw.strip() != "{}":
+        try:
+            parsed = json.loads(usage_tips_raw)
+            if isinstance(parsed, dict) and len(parsed) > 0:
+                usage_tips_valid = True
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+    # --- modelDescription (README → HTML) ---
+    desc_html_filled = len(model_description.strip()) > 100
+
+    # --- preview_url ---
+    preview_filled = bool(preview_url) and os.path.exists(preview_url)
+
+    # ------------------------------------------------------------------
+    # Composite score (0-100)
+    # ------------------------------------------------------------------
+
+    field_scores = {
+        "base_model": _score_bool(base_model_filled and base_model_valid, weight=15),
+        "trigger_words": _score_bool(triggers_valid, weight=15),
+        "short_description": _score_bool(desc_filled, weight=10),
+        "tags": _score_bool(tags_valid, weight=15),
+        "tags_priority_coverage": _score_continuous(tags_priority_coverage, weight=5),
+        "notes": _score_bool(notes_filled, weight=5),
+        "usage_tips": _score_bool(usage_tips_valid, weight=5),
+        "modelDescription_html": _score_bool(desc_html_filled, weight=10),
+        "preview_downloaded": _score_bool(preview_filled, weight=10),
+    }
+
+    # Deduct points for enrichment-level failures
+    penalty = 0
+    if enrichment_errors:
+        penalty += 10
+    if not enrichment_success:
+        penalty += 20
+
+    total_raw = sum(field_scores.values())
+    total = max(0, min(100, total_raw - penalty))
+
+    # ------------------------------------------------------------------
+    # Flagged issues
+    # ------------------------------------------------------------------
+
+    issues: List[str] = []
+    if not base_model_filled:
+        issues.append("base_model is empty or 'Unknown'")
+    elif not base_model_valid:
+        issues.append(f"base_model '{base_model}' not in SUPPORTED_BASE_MODELS")
+    if not triggers_valid:
+        issues.append("trigger_words are missing or contain only placeholders")
+    if not desc_filled:
+        issues.append("short_description is too short or empty")
+    if not tags_valid:
+        issues.append("tags are missing, too few, or purely technical")
+    if tags_valid and tags_priority_coverage < 0.5:
+        issues.append("tags have low overlap with priority_tags (< 50%)")
+    if not notes_filled:
+        issues.append("notes are too short or empty")
+    if not usage_tips_valid:
+        issues.append("usage_tips is empty or invalid JSON")
+    if not desc_html_filled:
+        issues.append("modelDescription is too short (README may not have been converted)")
+    if not preview_filled:
+        issues.append("preview image not downloaded (URL missing or download failed)")
+
+    return {
+        "repo_id": repo_id,
+        "model_path": model_path,
+        "enrichment_success": enrichment_success,
+        "total_score": total,
+        "field_scores": field_scores,
+        "issues": issues,
+        "confidence_from_llm": confidence,
+        "raw_values": {
+            "base_model": base_model,
+            "trigger_words": trained_words,
+            "short_description": short_desc,
+            "tags": tags,
+            "notes": notes,
+            "usage_tips": usage_tips_raw,
+            "preview_url": preview_url,
+            "has_modelDescription": len(model_description) > 0,
+        },
+    }
+
+
+def _score_bool(condition: bool, weight: int = 10) -> int:
+    return weight if condition else 0
+
+
+def _score_continuous(value: float, weight: int = 10) -> int:
+    """Linear interpolation: value 0.0 → 0, value 1.0 → *weight*."""
+    return int(round(value * weight))
+
+
+# ---------------------------------------------------------------------------
+# Batch evaluation
+# ---------------------------------------------------------------------------
+
+
+def evaluate_batch(
+    enriched: List[Dict[str, Any]],
+) -> List[ScoreRecord]:
+    """Evaluate a list of enrichment results.
+
+    Each entry in *enriched* should have keys:
+      ``repo_id``, ``model_path``, ``metadata`` (the enriched dict),
+      ``success``, ``errors``.
+    """
+    scores: List[ScoreRecord] = []
+    for entry in enriched:
+        record = evaluate_model(
+            metadata=entry.get("metadata", {}),
+            model_path=entry.get("model_path", ""),
+            repo_id=entry.get("repo_id", ""),
+            enrichment_success=entry.get("success", False),
+            enrichment_errors=entry.get("errors", []),
+        )
+        scores.append(record)
+    return scores
+
+
+# ---------------------------------------------------------------------------
+# Aggregate statistics
+# ---------------------------------------------------------------------------
+
+
+def aggregate_scores(scores: List[ScoreRecord]) -> Dict[str, Any]:
+    """Compute aggregate stats across all scored models."""
+    n = len(scores)
+    if n == 0:
+        return {"error": "no scores to aggregate"}
+
+    field_names = [
+        "base_model", "trigger_words", "short_description", "tags",
+        "tags_priority_coverage", "notes", "usage_tips",
+        "modelDescription_html", "preview_downloaded",
+    ]
+    possible = {f: 15 if f == "base_model" or f == "trigger_words" or f == "tags" else
+                   10 if f == "short_description" or f == "modelDescription_html" or f == "preview_downloaded" else
+                   5
+                for f in field_names}
+
+    # Per-field aggregate
+    field_agg: Dict[str, Any] = {}
+    for fn in field_names:
+        vals = [s["field_scores"].get(fn, 0) for s in scores]
+        max_per_field = possible[fn]
+        field_agg[fn] = {
+            "mean": round(sum(vals) / n, 1) if n else 0,
+            "fill_rate_pct": round(
+                sum(1 for v in vals if v >= max_per_field) / n * 100, 1
+            ) if n else 0.0,
+            "partial_rate_pct": round(
+                sum(1 for v in vals if 0 < v < max_per_field) / n * 100, 1
+            ) if n else 0.0,
+            "empty_rate_pct": round(
+                sum(1 for v in vals if v == 0) / n * 100, 1
+            ) if n else 0.0,
+        }
+
+    # Total score distribution
+    total_scores = [s["total_score"] for s in scores]
+    total_agg = {
+        "mean": round(sum(total_scores) / n, 1) if n else 0,
+        "median": _median(total_scores),
+        "min": min(total_scores) if total_scores else 0,
+        "max": max(total_scores) if total_scores else 0,
+        "bins": {
+            "excellent_80+": sum(1 for s in total_scores if s >= 80),
+            "good_60_79": sum(1 for s in total_scores if 60 <= s < 80),
+            "fair_40_59": sum(1 for s in total_scores if 40 <= s < 60),
+            "poor_20_39": sum(1 for s in total_scores if 20 <= s < 40),
+            "bad_0_19": sum(1 for s in total_scores if s < 20),
+        },
+    }
+
+    # Issue frequency
+    issue_counter: Dict[str, int] = {}
+    for s in scores:
+        for issue in s["issues"]:
+            issue_counter[issue] = issue_counter.get(issue, 0) + 1
+    top_issues = sorted(issue_counter.items(), key=lambda x: -x[1])
+
+    # Confidence distribution
+    conf_counter: Dict[str, int] = {"high": 0, "medium": 0, "low": 0, "": 0}
+    for s in scores:
+        c = (s.get("confidence_from_llm") or "").strip().lower()
+        if c in conf_counter:
+            conf_counter[c] += 1
+        else:
+            conf_counter[""] += 1
+
+    # Success / timeout / failure stats
+    success_count = sum(1 for s in scores if s["enrichment_success"])
+    fail_count = n - success_count
+
+    return {
+        "model_count": n,
+        "success_count": success_count,
+        "fail_count": fail_count,
+        "total_score": total_agg,
+        "field_aggregates": field_agg,
+        "top_issues": top_issues[:15],
+        "confidence_distribution": conf_counter,
+    }
+
+
+def _median(values: List[float]) -> float:
+    if not values:
+        return 0.0
+    sorted_v = sorted(values)
+    m = len(sorted_v) // 2
+    if len(sorted_v) % 2 == 0:
+        return round((sorted_v[m - 1] + sorted_v[m]) / 2, 1)
+    return round(sorted_v[m], 1)

tests/enrich_hf_validation/metadata_constructor.py

@@ -0,0 +1,202 @@
+"""Construct initial ``.metadata.json`` sidecars for HF model repos.
+
+Each HF repo + safetensors pair gets a minimal metadata file — no real model
+file is needed.  The enrichment pipeline reads only the sidecar.
+
+Data format (one line per entry)::
+
+    repo_id, model_name.safetensors
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Dict, List, Tuple
+
+from .config import CIVITAI_MODEL_TAGS
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+# A validated entry parsed from the models file:
+#   (repo_id, safetensors_name)
+RepoEntry = Tuple[str, str]
+
+
+def load_repo_ids(path: str, max_models: int | None = None) -> List[RepoEntry]:
+    """Read ``repo_id, safetensors_name`` pairs from *path*.
+
+    Format (one per line, blanks and ``#`` comments ignored)::
+
+        user/repo-name, lora_zimage_turbo_myjs_alpha01.safetensors
+
+    Returns a list of ``(repo_id, safetensors_name)`` tuples.
+    """
+    path = os.path.expanduser(path)
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Models file not found: {path}")
+
+    entries: List[RepoEntry] = []
+    with open(path, "r", encoding="utf-8") as fh:
+        for raw_line in fh:
+            line = raw_line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            # Split on the first comma
+            if "," not in line:
+                logger.warning("Skipping malformed line (no comma): %s", raw_line.rstrip())
+                continue
+
+            repo_id, safetensors_name = [part.strip() for part in line.split(",", 1)]
+            if not repo_id or not safetensors_name:
+                logger.warning("Skipping malformed line (empty fields): %s", raw_line.rstrip())
+                continue
+            if not safetensors_name.lower().endswith(".safetensors"):
+                logger.warning(
+                    "Skipping line — safetensors_name doesn't end with .safetensors: %s",
+                    raw_line.rstrip(),
+                )
+                continue
+
+            entries.append((repo_id, safetensors_name))
+
+    if max_models is not None and max_models > 0:
+        entries = entries[:max_models]
+
+    logger.info("Loaded %d HF repo entries from %s", len(entries), path)
+    return entries
+
+
+def sanitize_repo_id(repo_id: str) -> str:
+    """Turn ``user/repo-name`` into a safe directory name."""
+    return repo_id.replace("/", "__").replace(".", "_")
+
+
+def build_model_dir(output_dir: str, repo_id: str) -> str:
+    """Return the per-model working directory."""
+    return os.path.join(output_dir, "models", sanitize_repo_id(repo_id))
+
+
+def build_model_path(model_dir: str, safetensors_name: str) -> str:
+    """Return the model file path using the real safetensors filename."""
+    return os.path.join(model_dir, safetensors_name)
+
+
+def build_metadata_path(model_path: str) -> str:
+    """Return the sidecar path for a model file.
+
+    This MUST match the convention used by ``MetadataManager`` /
+    ``apply_metadata_updates``, which derives the sidecar path via
+    ``os.path.splitext(model_path)[0] + '.metadata.json'``.
+    For a model file ``lora_x.safetensors`` the sidecar is
+    ``lora_x.metadata.json`` — *not* ``lora_x.safetensors.metadata.json``.
+    """
+    return f"{os.path.splitext(model_path)[0]}.metadata.json"
+
+
+def create_initial_metadata(
+    output_dir: str,
+    repo_id: str,
+    safetensors_name: str,
+) -> str:
+    """Write a minimal ``.metadata.json`` for *repo_id* + *safetensors_name*.
+
+    Args:
+        output_dir: Root output directory.
+        repo_id: HuggingFace repo identifier (``user/repo``).
+        safetensors_name: The specific model file name (e.g.
+            ``lora_zimage_turbo_myjs_alpha01.safetensors``).
+
+    Returns the **model path** (the ``.safetensors`` path whose sidecar was
+    written).  The caller passes this path to ``AgentService.execute_skill``.
+    The basename (filename without extension) will match the real model file,
+    so ``extract_relevant_section`` can reliably match against the README.
+    """
+    model_dir = build_model_dir(output_dir, repo_id)
+    os.makedirs(model_dir, exist_ok=True)
+    model_path = build_model_path(model_dir, safetensors_name)
+    metadata_path = build_metadata_path(model_path)
+
+    hf_url = f"https://huggingface.co/{repo_id}"
+    file_name = safetensors_name
+
+    metadata: Dict[str, Any] = {
+        "file_name": file_name,
+        "model_name": safetensors_name,
+        "file_path": model_path.replace(os.sep, "/"),
+        "size": 0,
+        "modified": 0,
+        "sha256": "",
+        "base_model": "Unknown",
+        "preview_url": "",
+        "preview_nsfw_level": 0,
+        "notes": "",
+        "from_civitai": False,
+        "civitai": {},
+        "tags": [],
+        "modelDescription": "",
+        "civitai_deleted": False,
+        "favorite": False,
+        "exclude": False,
+        "db_checked": False,
+        "skip_metadata_refresh": False,
+        "metadata_source": "",
+        "last_checked_at": 0,
+        "hash_status": "completed",
+        "trainedWords": [],
+        "hf_url": hf_url,
+        "usage_tips": "{}",
+    }
+
+    with open(metadata_path, "w", encoding="utf-8") as fh:
+        json.dump(metadata, fh, indent=2, ensure_ascii=False)
+
+    logger.debug("Created initial metadata for %s -> %s", repo_id, metadata_path)
+    return model_path
+
+
+def create_all_initial_metadata(
+    entries: List[RepoEntry],
+    output_dir: str,
+    *,
+    skip_existing: bool = True,
+) -> Tuple[List[str], List[str]]:
+    """Create initial metadata for every repo entry.
+
+    Args:
+        entries: List of ``(repo_id, safetensors_name)`` tuples.
+        output_dir: Root output directory.
+        skip_existing: If True, skip repos whose metadata already exists.
+
+    Returns:
+        A tuple ``(model_paths, repo_ids)`` — two parallel lists in the same
+        order as *entries*.  This keeps downstream code (enrichment runner,
+        evaluation engine) unchanged.
+    """
+    model_paths: List[str] = []
+    repo_ids: List[str] = []
+    for repo_id, safetensors_name in entries:
+        model_dir = build_model_dir(output_dir, repo_id)
+        model_path = build_model_path(model_dir, safetensors_name)
+        metadata_path = build_metadata_path(model_path)
+
+        if skip_existing and os.path.exists(metadata_path):
+            model_paths.append(model_path)
+            repo_ids.append(repo_id)
+            continue
+
+        model_paths.append(create_initial_metadata(output_dir, repo_id, safetensors_name))
+        repo_ids.append(repo_id)
+
+    logger.info(
+        "Constructed initial metadata for %d/%d repos",
+        len(model_paths),
+        len(entries),
+    )
+    return model_paths, repo_ids

tests/enrich_hf_validation/preprocessing_auditor.py

@@ -0,0 +1,467 @@
+"""Preprocessing audit for the HF metadata enrichment validation pipeline.
+
+Phase 1.5 — runs between Phase 1 (metadata creation) and Phase 2 (enrichment).
+
+Audits the README preprocessing pipeline (section extraction + cleaning)
+for each repo in the dataset, capturing intermediate outputs so we can
+distinguish between:
+
+    (A) Preprocessing failed → LLM never saw the right content
+    (B) Preprocessing succeeded → LLM/prompt needs improvement
+
+This prevents wasted effort optimizing prompts when the actual problem is
+that ``extract_relevant_section`` or ``clean_readme_for_llm`` removed or
+misaligned the content the LLM needed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import re
+import time
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, List, Tuple
+
+import aiohttp
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Audit record
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class AuditRecord:
+    """Preprocessing audit for a single repo entry."""
+
+    # Identity
+    repo_id: str
+    safetensors_name: str
+    basename: str  # filename without .safetensors
+
+    # Raw README stats
+    raw_readme_length: int
+    raw_readme_line_count: int
+    has_yaml_frontmatter: bool
+    yaml_has_base_model: bool
+    yaml_has_tags: bool
+
+    # Section extraction
+    section_extraction_activated: bool  # output < 95% of input length
+    section_length: int
+    section_line_count: int
+    basename_in_section: bool  # basename appears in extracted section text
+
+    # Cleaning
+    cleaned_length: int
+    cleaned_line_count: int
+    compression_pct: float  # (1 - cleaned/raw) * 100
+
+    # Widget section (stripped by _strip_widget_section)
+    widget_section_found: bool
+    widget_section_length: int
+
+    # Flags (list of anomaly descriptions)
+    flags: List[str] = field(default_factory=list)
+
+    # Local file path to the saved raw README (for cross-reference)
+    readme_file: str = ""
+
+    # Staged intermediate output for report detail
+    raw_readme_preview: str = ""  # first 200 chars
+    section_preview: str = ""  # first 300 chars
+
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_HF_RAW_URL = "https://huggingface.co/{repo_id}/raw/main/README.md"
+
+# Thresholds for flagging
+_SECTION_ACTIVATION_RATIO = 0.95
+_MIN_CLEANED_LENGTH = 100
+_MAX_COMPRESSION_PCT = 99.0
+_MIN_SECTION_LINES = 3
+
+
+# ---------------------------------------------------------------------------
+# Module loader — bypasses parent-package __init__ that imports ComfyUI
+# ---------------------------------------------------------------------------
+
+_readme_processor_module = None
+
+
+def _load_readme_processor():
+    """Import ``readme_processor`` without triggering ``folder_paths`` import.
+
+    The normal import path (``py.services.agent.skills.enrich_hf_metadata.
+    readme_processor``) triggers ``py.services.agent.__init__`` which
+    imports ``agent_service.py`` → ``py/config.py`` → ComfyUI's
+    ``folder_paths``, which is not available in standalone mode.
+    """
+    global _readme_processor_module
+    if _readme_processor_module is not None:
+        return _readme_processor_module
+
+    import importlib.util
+
+    _RP_PATH = os.path.join(
+        os.path.dirname(__file__),  # tests/enrich_hf_validation/
+        "..", "..",
+        "py", "services", "agent", "skills", "enrich_hf_metadata",
+        "readme_processor.py",
+    )
+    rp_path = os.path.normpath(_RP_PATH)
+    if not os.path.exists(rp_path):
+        logger.error("readme_processor.py not found at %s", rp_path)
+        return None
+
+    spec = importlib.util.spec_from_file_location(
+        "readme_processor", rp_path,
+    )
+    if spec is None or spec.loader is None:
+        logger.error("Could not create spec for readme_processor.py")
+        return None
+
+    mod = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(mod)
+    except Exception as exc:
+        logger.error("Failed to load readme_processor.py: %s", exc)
+        return None
+
+    _readme_processor_module = mod
+    return mod
+
+
+# ---------------------------------------------------------------------------
+# HF README fetcher
+# ---------------------------------------------------------------------------
+
+
+async def _fetch_readme(repo_id: str, session: aiohttp.ClientSession) -> str:
+    """Fetch the raw README.md from HuggingFace."""
+    url = _HF_RAW_URL.format(repo_id=repo_id)
+    try:
+        async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
+            if resp.status == 200:
+                return await resp.text()
+            logger.warning("Failed to fetch README for %s: HTTP %d", repo_id, resp.status)
+            return ""
+    except (asyncio.TimeoutError, aiohttp.ClientError) as exc:
+        logger.warning("Failed to fetch README for %s: %s", repo_id, exc)
+        return ""
+
+
+# ---------------------------------------------------------------------------
+# Analysis helpers
+# ---------------------------------------------------------------------------
+
+
+def _has_yaml_frontmatter(text: str) -> bool:
+    return bool(text.strip().startswith("---"))
+
+
+def _extract_yaml_field(text: str, field: str) -> bool:
+    """Check if the given YAML field exists in the frontmatter."""
+    lines = text.split("\n")
+    if not lines or not lines[0].strip().startswith("---"):
+        return False
+    end = 1
+    while end < len(lines):
+        if lines[end].strip().startswith("---"):
+            break
+        end += 1
+    if end >= len(lines):
+        return False
+    frontmatter = "\n".join(lines[1:end])
+    pattern = rf"^{field}:"
+    return bool(re.search(pattern, frontmatter, re.MULTILINE))
+
+
+def _find_widget_section_length(text: str) -> int:
+    """Find the ``widget:`` YAML section and return its length (0 if none)."""
+    if not _has_yaml_frontmatter(text):
+        return 0
+    frontmatter_end = text.find("---", 3)
+    if frontmatter_end == -1:
+        return 0
+    frontmatter = text[3:frontmatter_end]
+
+    # Match widget: through to the next top-level key or frontmatter end
+    m = re.search(r"\nwidget:", frontmatter)
+    if not m:
+        return 0
+    # Length from widget: to end of frontmatter (the next \n\w+: or \n---)
+    return len(frontmatter[m.start():])
+
+
+# ---------------------------------------------------------------------------
+# Core auditor
+# ---------------------------------------------------------------------------
+
+
+async def run_audit(
+    entries: List[Tuple[str, str]],
+    *,
+    concurrency: int = 10,
+    readmes_dir: str | None = None,
+) -> Tuple[List[AuditRecord], Dict[str, Any]]:
+    """Run the preprocessing audit over all repo entries.
+
+    Args:
+        entries: List of ``(repo_id, safetensors_name)``.
+        concurrency: Max parallel fetches to HuggingFace.
+        readmes_dir: If set, saves each fetched README as
+            ``{sanitized_repo_id}.md`` in this directory for offline
+            cross-reference against audit results.
+
+    Returns:
+        Tuple of ``(records, summary)`` where *summary* is a dict with
+        aggregate statistics.
+    """
+    semaphore = asyncio.Semaphore(concurrency)
+    records: List[AuditRecord] = []
+    flag_counter: Dict[str, int] = {}
+
+    if readmes_dir:
+        os.makedirs(readmes_dir, exist_ok=True)
+
+    connector = aiohttp.TCPConnector(limit=concurrency)
+    async with aiohttp.ClientSession(connector=connector) as session:
+        tasks = [_audit_one(entry, session, semaphore, readmes_dir=readmes_dir) for entry in entries]
+        gathered = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for entry, result in zip(entries, gathered):
+            if isinstance(result, Exception):
+                logger.error("Audit failed for %s: %s", entry[0], result)
+                records.append(
+                    AuditRecord(
+                        repo_id=entry[0],
+                        safetensors_name=entry[1],
+                        basename=os.path.splitext(entry[1])[0],
+                        raw_readme_length=0,
+                        raw_readme_line_count=0,
+                        has_yaml_frontmatter=False,
+                        yaml_has_base_model=False,
+                        yaml_has_tags=False,
+                        section_extraction_activated=False,
+                        section_length=0,
+                        section_line_count=0,
+                        basename_in_section=False,
+                        cleaned_length=0,
+                        cleaned_line_count=0,
+                        compression_pct=0.0,
+                        widget_section_found=False,
+                        widget_section_length=0,
+                        readme_file="",
+                        flags=[f"Audit exception: {result}"],
+                    )
+                )
+                continue
+
+            # The continue above ensures result is AuditRecord here
+            assert isinstance(result, AuditRecord)
+            records.append(result)
+            for flag in result.flags:
+                flag_counter[flag] = flag_counter.get(flag, 0) + 1
+
+    summary = _build_summary(records, flag_counter)
+    return records, summary
+
+
+def _sanitize_repo_id(repo_id: str) -> str:
+    """Turn ``user/repo-name`` into a safe filename."""
+    return repo_id.replace("/", "__").replace(".", "_")
+
+
+async def _audit_one(
+    entry: Tuple[str, str],
+    session: aiohttp.ClientSession,
+    semaphore: asyncio.Semaphore,
+    *,
+    readmes_dir: str | None = None,
+) -> AuditRecord:
+    """Audit a single repo entry."""
+    repo_id, safetensors_name = entry
+    basename = os.path.splitext(safetensors_name)[0]
+
+    async with semaphore:
+        # Import production preprocessing functions.
+        # Use importlib to bypass py.services.agent.__init__ which triggers
+        # ComfyUI's folder_paths module (not available in standalone mode).
+        _rp = _load_readme_processor()
+        if _rp is None:
+            return AuditRecord(
+                repo_id=repo_id,
+                safetensors_name=safetensors_name,
+                basename=basename,
+                raw_readme_length=0, raw_readme_line_count=0,
+                has_yaml_frontmatter=False, yaml_has_base_model=False, yaml_has_tags=False,
+                readme_file="",
+                section_extraction_activated=False, section_length=0, section_line_count=0,
+                basename_in_section=False, cleaned_length=0, cleaned_line_count=0,
+                compression_pct=0.0, widget_section_found=False, widget_section_length=0,
+                flags=["IMPORT_FAILED"],
+            )
+        clean_readme_for_llm = _rp.clean_readme_for_llm
+        extract_relevant_section = _rp.extract_relevant_section
+
+        # Step 1: Fetch the raw README
+        raw_text = await _fetch_readme(repo_id, session)
+        if not raw_text:
+            return AuditRecord(
+                repo_id=repo_id,
+                safetensors_name=safetensors_name,
+                basename=basename,
+                raw_readme_length=0,
+                raw_readme_line_count=0,
+                has_yaml_frontmatter=False,
+                yaml_has_base_model=False,
+                yaml_has_tags=False,
+                section_extraction_activated=False,
+                section_length=0,
+                section_line_count=0,
+                basename_in_section=False,
+                readme_file="",
+                cleaned_length=0,
+                cleaned_line_count=0,
+                compression_pct=0.0,
+                widget_section_found=False,
+                widget_section_length=0,
+                flags=["README_FETCH_FAILED"],
+            )
+
+        # Save the raw README to disk for offline cross-reference
+        readme_path = ""
+        if readmes_dir:
+            safe_name = _sanitize_repo_id(repo_id)
+            readme_path = os.path.join(readmes_dir, f"{safe_name}.md")
+            try:
+                with open(readme_path, "w", encoding="utf-8") as fh:
+                    fh.write(raw_text)
+            except OSError as exc:
+                logger.warning("Failed to save README for %s: %s", repo_id, exc)
+                readme_path = ""
+
+        raw_lines = raw_text.split("\n")
+        raw_len = len(raw_text)
+        raw_line_count = len(raw_lines)
+
+        # Step 2: Analyze raw README
+        yaml_fm = _has_yaml_frontmatter(raw_text)
+        yaml_has_bm = _extract_yaml_field(raw_text, "base_model") if yaml_fm else False
+        yaml_has_tg = _extract_yaml_field(raw_text, "tags") if yaml_fm else False
+        widget_len = _find_widget_section_length(raw_text)
+
+        # Step 3: Section extraction
+        section = extract_relevant_section(raw_text, basename)
+        section_len = len(section)
+        section_line_count = len(section.split("\n"))
+        section_activated = section_len < raw_len * _SECTION_ACTIVATION_RATIO
+        basename_in_sec = basename.lower() in section.lower()
+
+        # Step 4: Cleaning for LLM
+        cleaned = clean_readme_for_llm(section)
+        cleaned_len = len(cleaned)
+        cleaned_line_count = len(cleaned.split("\n"))
+        compression_pct = round((1 - cleaned_len / raw_len) * 100, 1) if raw_len else 0.0
+
+        # Step 5: Flag anomalies
+        flags: List[str] = []
+        if not raw_text.strip():
+            flags.append("README_EMPTY")
+        if not yaml_fm:
+            flags.append("NO_YAML_FRONTMATTER")
+        if not section_activated:
+            # Check if basename is extremely short/generic (likely synthetic)
+            if len(basename) <= 5:
+                flags.append("BASENAME_TOO_SHORT_SECTION_NOT_EXPECTED")
+            else:
+                flags.append("SECTION_EXTRACTION_NOT_ACTIVATED")
+        elif not basename_in_sec:
+            flags.append("BASENAME_NOT_IN_EXTRACTED_SECTION")
+        if widget_len == 0:
+            # Not necessarily a problem — many repos lack a widget section
+            pass
+        if cleaned_len < _MIN_CLEANED_LENGTH:
+            flags.append("CLEANED_README_TOO_SHORT")
+        if compression_pct > _MAX_COMPRESSION_PCT:
+            flags.append("EXTREME_COMPRESSION")
+        if section_activated and section_line_count < _MIN_SECTION_LINES:
+            flags.append("SECTION_TOO_SMALL")
+
+        return AuditRecord(
+            repo_id=repo_id,
+            safetensors_name=safetensors_name,
+            basename=basename,
+            raw_readme_length=raw_len,
+            raw_readme_line_count=raw_line_count,
+            has_yaml_frontmatter=yaml_fm,
+            yaml_has_base_model=yaml_has_bm,
+            yaml_has_tags=yaml_has_tg,
+            section_extraction_activated=section_activated,
+            section_length=section_len,
+            section_line_count=section_line_count,
+            basename_in_section=basename_in_sec,
+            cleaned_length=cleaned_len,
+            cleaned_line_count=cleaned_line_count,
+            compression_pct=compression_pct,
+            widget_section_found=widget_len > 0,
+            widget_section_length=widget_len,
+            readme_file=readme_path,
+            flags=flags,
+            raw_readme_preview=raw_text[:200],
+            section_preview=section[:300],
+        )
+
+
+def _build_summary(
+    records: List[AuditRecord],
+    flag_counter: Dict[str, int],
+) -> Dict[str, Any]:
+    """Aggregate audit statistics."""
+    n = len(records)
+    if n == 0:
+        return {"error": "no records", "model_count": 0}
+
+    activated = sum(1 for r in records if r.section_extraction_activated)
+    basename_hit = sum(1 for r in records if r.basename_in_section)
+    with_yaml = sum(1 for r in records if r.has_yaml_frontmatter)
+    with_widget = sum(1 for r in records if r.widget_section_found)
+    fetch_failed = sum(1 for r in records if "README_FETCH_FAILED" in r.flags)
+
+    avg_compression = round(
+        sum(r.compression_pct for r in records if r.raw_readme_length > 0) / max(n - fetch_failed, 1),
+        1,
+    )
+    avg_cleaned = round(
+        sum(r.cleaned_length for r in records if r.raw_readme_length > 0) / max(n - fetch_failed, 1),
+    )
+
+    top_flags = sorted(flag_counter.items(), key=lambda x: -x[1])[:10]
+
+    return {
+        "model_count": n,
+        "fetch_failed_count": fetch_failed,
+        "section_extraction_activated": activated,
+        "section_extraction_pct": round(activated / max(n - fetch_failed, 1) * 100, 1),
+        "basename_in_section": basename_hit,
+        "basename_in_section_pct": round(basename_hit / max(n - fetch_failed, 1) * 100, 1),
+        "with_yaml_frontmatter": with_yaml,
+        "with_yaml_frontmatter_pct": round(with_yaml / max(n - fetch_failed, 1) * 100, 1),
+        "with_widget_section": with_widget,
+        "avg_compression_pct": avg_compression,
+        "avg_cleaned_length": avg_cleaned,
+        "top_flags": top_flags,
+    }
+
+
+def audit_records_to_serializable(records: List[AuditRecord]) -> List[Dict[str, Any]]:
+    """Convert AuditRecord dataclasses to plain dicts for JSON serialization."""
+    return [asdict(r) for r in records]

tests/enrich_hf_validation/report_generator.py

@@ -0,0 +1,391 @@
+"""Generate structured reports from evaluation results.
+
+Produces:
+
+1. A JSON data dump (``report.json``) with all scores and aggregations.
+2. A human-readable Markdown report (``report.md``) with summary stats,
+   issue patterns, and actionable optimisation suggestions.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime
+from typing import Any, Dict, List
+
+from .config import SUPPORTED_BASE_MODELS
+from .evaluation_engine import ScoreRecord
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Markdown report
+# ---------------------------------------------------------------------------
+
+
+def _fmt_pct(value: float) -> str:
+    return f"{value:.1f}%"
+
+
+def _bar(value: float, width: int = 20) -> str:
+    filled = int(round(value / 100 * width))
+    return "█" * filled + "░" * (width - filled)
+
+
+def generate_optimisation_suggestions(
+    agg: Dict[str, Any],
+    scores: List[ScoreRecord],
+) -> List[str]:
+    """Analyse evaluation results and produce concrete suggestions."""
+    suggestions: List[str] = []
+    fa = agg.get("field_aggregates", {})
+
+    # --- base_model ---
+    bm = fa.get("base_model", {})
+    if bm and bm.get("empty_rate_pct", 0) > 30:
+        suggestions.append(
+            "- **base_model 空置率高 ({:.0f}%)**: 多数 HF 模型卡片未在 YAML frontmatter 中声明 "
+            "`base_model:` 字段，LLM 无法推断。可考虑在 prompt 中增加 \"look at the model file name "
+            "for clues\" 的引导，或在后处理中增加基于文件名规则的 fallback 猜测。".format(
+                bm.get("empty_rate_pct", 0)
+            )
+        )
+    bm_invalid = sum(
+        1
+        for s in scores
+        if s["raw_values"]["base_model"]
+        and s["raw_values"]["base_model"] != "Unknown"
+        and s["raw_values"]["base_model"] not in set(SUPPORTED_BASE_MODELS)
+    )
+    if bm_invalid > 5:
+        suggestions.append(
+            "- **base_model 含非标准值 ({} 个)**: LLM 输出了未在当前生产系统的 base model 列表 "
+            "中的名称。建议在 prompt 中强调 \"Use EXACTLY one name from the list\" 并在 "
+            "`PostProcessor` 中加一层验证过滤，非标准值直接丢弃。".format(bm_invalid)
+        )
+
+    # --- trigger_words ---
+    tw = fa.get("trigger_words", {})
+    if tw and tw.get("empty_rate_pct", 0) > 40:
+        suggestions.append(
+            "- **trigger_words 空置率高 ({:.0f}%)**: 大量 HF 模型卡没有明确的 "
+            "`instance_prompt:` 或 trigger word 说明。当前 prompt 已覆盖常见模式。若确认这些模型确实"
+            "没有 trigger words（例如 style lora），空数组是正确结果，不需优化。".format(
+                tw.get("empty_rate_pct", 0)
+            )
+        )
+
+    # --- tags ---
+    tag = fa.get("tags", {})
+    if tag and tag.get("empty_rate_pct", 0) > 30:
+        suggestions.append(
+            "- **tags 空置率高 ({:.0f}%)**: 当前 prompt 要求 tags 必须与 "
+            "`priority_tags`（CIVITAI_MODEL_TAGS）对齐。HF 模型的标签体系与 Civitai 不同，"
+            "很多 model card 使用细粒度标签（如 `pokemon`、`watercolor`）而不在 priority list 中。"
+            "建议: 扩大 priority_tags 范围，或允许 LLM 自由生成 tags 后只做去重不做严格过滤。".format(
+                tag.get("empty_rate_pct", 0)
+            )
+        )
+
+    # --- tags priority coverage ---
+    low_coverage = sum(
+        1
+        for s in scores
+        if s["field_scores"].get("tags_priority_coverage", 5) < 3  # < 60% of max
+        and s["field_scores"].get("tags", 0) > 0
+    )
+    if low_coverage > 10:
+        suggestions.append(
+            "- **{} 个模型的 tags 与 priority_tags 匹配度低于 60%**: "
+            "LLM 生成了有意义但不属于 CIVITAI_MODEL_TAGS 的标签。这说明 priority_tags "
+            "的覆盖范围对 HF 模型不足，建议按 HF 模型的实际分布补充新类别。".format(low_coverage)
+        )
+
+    # --- preview ---
+    prev = fa.get("preview_downloaded", {})
+    if prev and prev.get("empty_rate_pct", 0) > 50:
+        suggestions.append(
+            "- **预览图下载成功率低 ({:.0f}%)**: 很多 HF 模型卡没有 embed 图片（仅使用 YAML widget "
+            "或 external link）。当前 `readme_processor.py` 的 `extract_gallery_images` 和 "
+            "`extract_gallery_table_images` 已覆盖了多数场景。若预览图不重要，可降低此字段权重。".format(
+                prev.get("empty_rate_pct", 0)
+            )
+        )
+
+    # --- usage_tips ---
+    ut = fa.get("usage_tips", {})
+    if ut and ut.get("empty_rate_pct", 0) > 70:
+        suggestions.append(
+            "- **usage_tips 空置率极高 ({:.0f}%)**: 这是预期行为。HF 模型卡通常不包含 LoRA "
+            "强度/CLIP skip 等结构化参数。当前提取策略已合理。若需要可用数据，"
+            "可以考虑使用模型类型的通用默认值。".format(
+                ut.get("empty_rate_pct", 0)
+            )
+        )
+
+    # --- short_description ---
+    sd = fa.get("short_description", {})
+    if sd and sd.get("empty_rate_pct", 0) > 40:
+        suggestions.append(
+            "- **short_description 空置率 ({:.0f}%)**: 部分 HF 模型卡 README 内容极少（仅含标签和训练参数）。".format(
+                sd.get("empty_rate_pct", 0)
+            )
+        )
+
+    if not suggestions:
+        suggestions.append("- 未发现明显问题模式，各字段填充率均在可接受范围。")
+
+    return suggestions
+
+
+def generate_markdown_report(
+    agg: Dict[str, Any],
+    scores: List[ScoreRecord],
+    output_dir: str,
+    duration_summary: Dict[str, Any] | None = None,
+    *,
+    audit_summary: Dict[str, Any] | None = None,
+    config_warnings: List[str] | None = None,
+) -> str:
+    """Write ``report.md`` and return its content.
+
+    Args:
+        agg: Aggregate evaluation scores.
+        scores: Per-model evaluation records.
+        output_dir: Output directory for the report file.
+        duration_summary: Optional timing statistics.
+        audit_summary: Optional preprocessing audit summary (Phase 1.5).
+        config_warnings: Optional LLM config consistency warnings.
+    """
+    lines: List[str] = []
+    def wl(text: str = "") -> None:
+        lines.append(text)
+
+    wl("# HF Metadata Enrichment Validation Report")
+    wl()
+    wl(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    wl(f"Models evaluated: **{agg.get('model_count', 0)}**")
+    wl(f"Successful enrichments: **{agg.get('success_count', 0)}**")
+    wl(f"Failures: **{agg.get('fail_count', 0)}**")
+    wl()
+
+    # ---- Preprocessing Audit Section ----
+    if audit_summary and audit_summary.get("model_count", 0) > 0:
+        wl("## Preprocessing Audit")
+        wl()
+        wl(f"| Metric | Value |")
+        wl(f"|--------|-------|")
+        wl(f"| Models audited | {audit_summary.get('model_count', 0)} |")
+        wl(f"| README fetch failed | {audit_summary.get('fetch_failed_count', 0)} |")
+        wl(f"| Section extraction activated | {_fmt_pct(audit_summary.get('section_extraction_pct', 0))} |")
+        wl(f"| Basename found in section | {_fmt_pct(audit_summary.get('basename_in_section_pct', 0))} |")
+        wl(f"| Has YAML frontmatter | {_fmt_pct(audit_summary.get('with_yaml_frontmatter_pct', 0))} |")
+        wl(f"| Has YAML widget section | {_fmt_pct(audit_summary.get('with_widget_section', 0))} |")
+        wl(f"| Avg README compression | {audit_summary.get('avg_compression_pct', 0)}% |")
+        wl(f"| Avg cleaned length | {audit_summary.get('avg_cleaned_length', 0)} chars |")
+        wl()
+
+        if audit_summary.get("top_flags"):
+            wl("### Audit Flags (most frequent)")
+            wl()
+            for flag, count in audit_summary["top_flags"]:
+                wl(f"- **{flag}**: {count}x")
+            wl()
+
+        wl("**Interpretation:**")
+        wl()
+        act_pct = audit_summary.get("section_extraction_pct", 0)
+        if act_pct < 50:
+            wl(
+                "- ⚠️ Section extraction activated for fewer than 50% of repos. "
+                "This may indicate the basename doesn't match README content, or the "
+                "repos are mostly single-model (where full README is expected)."
+            )
+        else:
+            wl(
+                "- ✅ Section extraction is working for most repos — the LLM is "
+                "receiving focused README sections."
+            )
+
+        if audit_summary.get("basename_in_section_pct", 100) < 80:
+            wl(
+                "- ⚠️ The safetensors basename was NOT found in the extracted section "
+                "for many repos. This could mean the section extraction matched the wrong "
+                "section, or the README doesn't explicitly reference the filename."
+            )
+        wl()
+
+    # ---- Config warnings ----
+    if config_warnings:
+        wl("## ⚠️ Configuration Warnings")
+        wl()
+        for w in config_warnings:
+            wl(f"- {w}")
+        wl()
+
+    # ---- Duration ----
+    if duration_summary:
+        wl("## Timing")
+        wl()
+        wl(f"- Total wall time: **{duration_summary.get('total_wall_s', 0):.0f} s** ")
+        wl(f"  ({duration_summary.get('total_wall_s', 0) / 60:.1f} min)")
+        wl(f"- Mean per model: **{duration_summary.get('mean_s', 0):.1f} s**")
+        wl(f"- Median per model: **{duration_summary.get('median_s', 0):.1f} s**")
+        wl(f"- Fastest: **{duration_summary.get('min_s', 0):.1f} s**")
+        wl(f"- Slowest: **{duration_summary.get('max_s', 0):.1f} s**")
+        wl()
+
+    # ---- Overall score ----
+    ts = agg.get("total_score", {})
+    wl("## Overall Score Distribution  (0–100)")
+    wl()
+    wl(f"| Metric | Value |")
+    wl(f"|--------|-------|")
+    wl(f"| Mean   | {ts.get('mean', 'N/A')} |")
+    wl(f"| Median | {ts.get('median', 'N/A')} |")
+    wl(f"| Min    | {ts.get('min', 'N/A')} |")
+    wl(f"| Max    | {ts.get('max', 'N/A')} |")
+    wl()
+    for label, key in [
+        ("Excellent (≥80)", "excellent_80+"),
+        ("Good (60–79)", "good_60_79"),
+        ("Fair (40–59)", "fair_40_59"),
+        ("Poor (20–39)", "poor_20_39"),
+        ("Bad (<20)", "bad_0_19"),
+    ]:
+        count = ts.get("bins", {}).get(key, 0)
+        pct = count / agg["model_count"] * 100 if agg["model_count"] else 0
+        wl(f"- **{label}**: {count} models ({_fmt_pct(pct)})")
+    wl()
+
+    # ---- Per-field aggregates ----
+    wl("## Per-Field Completeness")
+    wl()
+    wl("| Field | Mean Score | Fill Rate | Empty Rate |")
+    wl("|-------|-----------:|----------:|-----------:|")
+    fa = agg.get("field_aggregates", {})
+    for fn in [
+        "base_model", "trigger_words", "short_description", "tags",
+        "tags_priority_coverage", "notes", "usage_tips",
+        "modelDescription_html", "preview_downloaded",
+    ]:
+        f = fa.get(fn, {})
+        if not f:
+            continue
+        wl(
+            f"| {fn} "
+            f"| {f.get('mean', 'N/A')} "
+            f"| {_fmt_pct(f.get('fill_rate_pct', 0))} "
+            f"| {_fmt_pct(f.get('empty_rate_pct', 0))} |"
+        )
+    wl()
+
+    # ---- Confidence distribution ----
+    wl("## LLM Confidence Distribution")
+    wl()
+    cd = agg.get("confidence_distribution", {})
+    total_conf = sum(cd.values()) or 1
+    for level in ["high", "medium", "low", ""]:
+        count = cd.get(level, 0)
+        label = level if level else "(not reported)"
+        pct = count / total_conf * 100
+        bar = _bar(pct)
+        wl(f"- **{label}**: {count}  {bar}  {_fmt_pct(pct)}")
+    wl()
+
+    # ---- Top issues ----
+    wl("## Most Frequent Issues")
+    wl()
+    for issue, count in agg.get("top_issues", []):
+        pct = count / agg["model_count"] * 100 if agg["model_count"] else 0
+        wl(f"- **{issue}** — {count}/{agg['model_count']} ({_fmt_pct(pct)})")
+    wl()
+
+    # ---- Optimisation suggestions ----
+    wl("## Optimisation Suggestions")
+    wl()
+    suggestions = generate_optimisation_suggestions(agg, scores)
+    for s in suggestions:
+        wl(s)
+    wl()
+
+    # ---- Per-model detail ----
+    wl("## Per-Model Detail")
+    wl()
+    wl("<details>")
+    wl("<summary>Click to expand</summary>")
+    wl()
+    wl("| # | Repo ID | Score | Issues | Confidence |")
+    wl("|---|---------|------:|--------|------------|")
+    for i, s in enumerate(scores, 1):
+        issue_count = len(s["issues"])
+        issue_str = (
+            f"{issue_count} issue(s)" if issue_count else "✓ ok"
+        )
+        wl(
+            f"| {i} "
+            f"| {s['repo_id']} "
+            f"| {s['total_score']} "
+            f"| {issue_str} "
+            f"| {s.get('confidence_from_llm', '') or '-'} |"
+        )
+    wl()
+    wl("</details>")
+    wl()
+
+    content = "\n".join(lines)
+    report_path = os.path.join(output_dir, "report.md")
+    with open(report_path, "w", encoding="utf-8") as fh:
+        fh.write(content)
+    logger.info("Markdown report written to %s", report_path)
+    return content
+
+
+# ---------------------------------------------------------------------------
+# JSON dump
+# ---------------------------------------------------------------------------
+
+
+def save_json_report(
+    agg: Dict[str, Any],
+    scores: List[ScoreRecord],
+    enrichment_results: List[Dict[str, Any]],
+    output_dir: str,
+    duration_summary: Dict[str, Any] | None = None,
+    *,
+    audit_summary: Dict[str, Any] | None = None,
+    config_warnings: List[str] | None = None,
+) -> str:
+    """Write ``report.json`` and return the path.
+
+    Args:
+        agg: Aggregate evaluation scores.
+        scores: Per-model evaluation records.
+        enrichment_results: Raw enrichment phase results.
+        output_dir: Output directory.
+        duration_summary: Optional timing statistics.
+        audit_summary: Optional preprocessing audit summary.
+        config_warnings: Optional LLM config consistency warnings.
+    """
+    report: Dict[str, Any] = {
+        "metadata": {
+            "generated_at": datetime.now().isoformat(),
+            "model_count": agg.get("model_count", 0),
+        },
+        "aggregate": agg,
+        "timing": duration_summary or {},
+        "per_model_scores": scores,
+        "enrichment_results": enrichment_results,
+    }
+    if audit_summary:
+        report["preprocessing_audit"] = audit_summary
+    if config_warnings:
+        report["config_warnings"] = config_warnings
+
+    path = os.path.join(output_dir, "report.json")
+    with open(path, "w", encoding="utf-8") as fh:
+        json.dump(report, fh, indent=2, ensure_ascii=False)
+    logger.info("JSON report written to %s", path)
+    return path

tests/enrich_hf_validation/run_validation.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+"""CLI entry point for the HF metadata enrichment validation suite.
+
+Usage::
+
+    # Full run (44 models, serial, ~1-2 h)
+    python -m tests.enrich_hf_validation.run_validation \\
+        --output /tmp/hf_enrich_validation
+
+    # Quick smoke test with 2 models
+    python -m tests.enrich_hf_validation.run_validation --sample 2
+
+    # Resume from a previous partial run
+    python -m tests.enrich_hf_validation.run_validation --resume
+
+    # Audit preprocessing only (no LLM calls, fast)
+    python -m tests.enrich_hf_validation.run_validation --audit-only
+
+    # Custom settings file
+    python -m tests.enrich_hf_validation.run_validation \\
+        --settings /custom/path/settings.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+import time
+from typing import Any, Dict, List, Tuple
+
+# Ensure the project root is on sys.path so that ``from py import ...`` works.
+_PROJECT_ROOT = os.path.normpath(
+    os.path.join(os.path.dirname(__file__), "..", "..")
+)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+# Add ComfyUI root to sys.path so ``folder_paths`` can be imported.
+# Project layout: ComfyUI/custom_nodes/ComfyUI-Lora-Manager/
+_COMFYUI_ROOT = os.path.normpath(os.path.join(_PROJECT_ROOT, "..", ".."))
+if _COMFYUI_ROOT not in sys.path:
+    sys.path.insert(0, _COMFYUI_ROOT)
+
+from tests.enrich_hf_validation.config import (
+    init_supported_base_models,
+    load_settings,
+)
+from tests.enrich_hf_validation.metadata_constructor import (
+    RepoEntry,
+    create_all_initial_metadata,
+    load_repo_ids,
+)
+from tests.enrich_hf_validation.enrichment_runner import EnrichmentRunner
+from tests.enrich_hf_validation.evaluation_engine import (
+    aggregate_scores,
+    evaluate_batch,
+)
+from tests.enrich_hf_validation.preprocessing_auditor import (
+    audit_records_to_serializable,
+    run_audit,
+)
+from tests.enrich_hf_validation.report_generator import (
+    generate_markdown_report,
+    save_json_report,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _setup_logging(verbose: bool) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    fmt = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+    logging.basicConfig(level=level, format=fmt, stream=sys.stderr)
+
+    # Quiet noisy third-party loggers
+    for name in ("aiohttp", "asyncio", "urllib3"):
+        logging.getLogger(name).setLevel(logging.WARNING)
+
+
+def _parse_args(argv: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Validate and optimise HF metadata enrichment via LLM.",
+    )
+    parser.add_argument(
+        "--models",
+        default=os.path.join(os.path.dirname(__file__), "test_data", "hf_lora_models_with_safetensors.txt"),
+        help="Path to the HF repo entries file (format: repo_id, model_name.safetensors per line)",
+    )
+    parser.add_argument(
+        "--settings",
+        default="~/.config/ComfyUI-LoRA-Manager/settings.json",
+        help="Path to LoRA Manager settings.json",
+    )
+    parser.add_argument(
+        "--output",
+        default="/tmp/hf_enrich_validation",
+        help="Output directory for reports and intermediate data",
+    )
+    parser.add_argument(
+        "--sample",
+        type=int,
+        default=0,
+        help="Process only the first N models (for quick smoke tests)",
+    )
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="Resume from previous partial run (uses progress.json)",
+    )
+    parser.add_argument(
+        "--no-enrich",
+        action="store_true",
+        help="Skip enrichment phase (evaluate existing metadata only)",
+    )
+    parser.add_argument(
+        "--audit-only",
+        action="store_true",
+        help="Run preprocessing audit only (no enrichment, no evaluation)",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=240,
+        help="Per-model LLM timeout in seconds (default: 240)",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Enable debug logging",
+    )
+    return parser.parse_args(argv)
+
+
+# ---------------------------------------------------------------------------
+# Phase helpers
+# ---------------------------------------------------------------------------
+
+
+def _phase_header(label: str) -> None:
+    sep = "=" * 60
+    print(f"\n{sep}", file=sys.stderr)
+    print(f"  PHASE: {label}", file=sys.stderr)
+    print(sep, file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Read back LLM config after enrichment (for consistency reporting)
+# ---------------------------------------------------------------------------
+
+
+def _get_actual_llm_config() -> Dict[str, str]:
+    """Read what LLMService is actually using, if initialized.
+
+    Only meaningful when called AFTER enrichment has started (i.e. after
+    ``AgentService.get_instance()`` has been called).
+    """
+    try:
+        from py.services.llm_service import LLMService
+
+        instance = LLMService._instance
+        if instance is None:
+            return {"status": "not initialized"}
+        cfg = instance._get_config()
+        return {
+            "provider": cfg.get("provider", ""),
+            "model": cfg.get("model", ""),
+            "api_base": cfg.get("api_base", ""),
+        }
+    except Exception as exc:
+        return {"status": f"error: {exc}"}
+
+
+def _compare_llm_config(
+    pipeline_cfg: Dict[str, Any],
+    actual_cfg: Dict[str, str],
+) -> List[str]:
+    """Compare pipeline-loaded vs LLMService-used config.
+
+    Returns warning messages if they differ.
+    """
+    warnings: List[str] = []
+    if not actual_cfg or actual_cfg.get("status", "") == "not initialized":
+        warnings.append(
+            "LLMService was not initialized during this run — cannot verify "
+            "config consistency."
+        )
+        return warnings
+
+    field_map = [
+        ("llm_provider", "provider"),
+        ("llm_model", "model"),
+        ("llm_api_base", "api_base"),
+    ]
+    for pipeline_key, llm_key in field_map:
+        pv = (pipeline_cfg.get(pipeline_key) or "").strip()
+        lv = (actual_cfg.get(llm_key) or "").strip()
+        if pv and lv and pv != lv:
+            warnings.append(
+                f"LLM config mismatch: --settings has '{pv}' for {pipeline_key}, "
+                f"but LLMService uses '{lv}'. "
+                f"The pipeline's --settings path ({pipeline_cfg.get('settings_path', '?')}) "
+                "may differ from where SettingsManager reads."
+            )
+    if not warnings and actual_cfg:
+        warnings.append(
+            "✅ LLM config matches between pipeline --settings and LLMService."
+        )
+    return warnings
+
+
+# ---------------------------------------------------------------------------
+# Phase 1.5: preprocessing audit
+# ---------------------------------------------------------------------------
+
+
+async def _run_preprocessing_audit(
+    entries: List[RepoEntry],
+    output_dir: str,
+) -> Dict[str, Any]:
+    """Execute the preprocessing audit and save results."""
+    _phase_header("Preprocessing audit")
+    print(f"  Auditing {len(entries)} repos ...", file=sys.stderr)
+
+    readmes_dir = os.path.join(output_dir, "readmes")
+    t0 = time.perf_counter()
+    records, summary = await run_audit(entries, readmes_dir=readmes_dir)
+    elapsed = time.perf_counter() - t0
+
+    # Save audit data
+    audit_path = os.path.join(output_dir, "preprocessing_audit.json")
+    with open(audit_path, "w", encoding="utf-8") as fh:
+        json.dump(
+            {
+                "summary": summary,
+                "records": audit_records_to_serializable(records),
+            },
+            fh,
+            indent=2,
+            ensure_ascii=False,
+        )
+
+    print(f"  Audit complete: {len(records)} repos in {elapsed:.0f}s", file=sys.stderr)
+    print(f"  Section extraction activated: {summary.get('section_extraction_pct', 0)}%", file=sys.stderr)
+    print(f"  Basename in extracted section: {summary.get('basename_in_section_pct', 0)}%", file=sys.stderr)
+    print(f"  Avg compression: {summary.get('avg_compression_pct', 0)}%", file=sys.stderr)
+    print(f"  Avg cleaned length: {summary.get('avg_cleaned_length', 0)} chars", file=sys.stderr)
+    print(f"  Audit data: {audit_path}", file=sys.stderr)
+
+    if summary.get("top_flags"):
+        print("  Top flags:", file=sys.stderr)
+        for flag, count in summary["top_flags"][:5]:
+            print(f"    - {flag}: {count}x", file=sys.stderr)
+
+    return summary
+
+
+async def _run_enrichment(
+    model_paths: List[str],
+    repos: List[str],
+    output_dir: str,
+    timeout: int,
+    verbose: bool,
+) -> Dict[str, Any]:
+    """Execute the enrichment phase."""
+    runner = EnrichmentRunner(
+        output_dir=output_dir,
+        per_model_timeout=timeout,
+    )
+    result = await runner.run(model_paths, repos)
+
+    # Print quick summary
+    progress = result["progress"]
+    total_done = (
+        len(progress.get("completed", []))
+        + len(progress.get("failed", []))
+        + len(progress.get("timed_out", []))
+    )
+    print(
+        f"\n  Enrichment complete: {total_done} processed "
+        f"({len(progress.get('completed', []))} ok, "
+        f"{len(progress.get('failed', []))} failed, "
+        f"{len(progress.get('timed_out', []))} timed out)",
+        file=sys.stderr,
+    )
+    return result
+
+
+def _collect_enriched_metadata(
+    model_paths: List[str],
+    repos: List[str],
+    results: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    """Read enriched .metadata.json for each model.
+
+    Uses the same path convention as the rest of the codebase:
+    ``os.path.splitext(model_path)[0] + '.metadata.json'``.
+
+    Returns a list of dicts with keys: repo_id, model_path, success,
+    errors, metadata.
+    """
+    enriched: List[Dict[str, Any]] = []
+    # Build a lookup from repo_id to enrichment result
+    result_lookup: Dict[str, Dict[str, Any]] = {}
+    for r in results:
+        result_lookup[r["repo_id"]] = r
+
+    for model_path, repo_id in zip(model_paths, repos):
+        res = result_lookup.get(repo_id, {})
+        metadata_path = f"{os.path.splitext(model_path)[0]}.metadata.json"
+        metadata: Dict[str, Any] = {}
+        if os.path.exists(metadata_path):
+            try:
+                with open(metadata_path, "r", encoding="utf-8") as fh:
+                    metadata = json.load(fh)
+            except (json.JSONDecodeError, OSError) as exc:
+                logger.warning("Failed to read %s: %s", metadata_path, exc)
+        else:
+            logger.warning(
+                "Metadata file not found for %s (expected: %s)",
+                repo_id, metadata_path,
+            )
+
+        enriched.append({
+            "repo_id": repo_id,
+            "model_path": model_path,
+            "success": res.get("success", False),
+            "errors": res.get("errors", []),
+            "metadata": metadata,
+        })
+
+    return enriched
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+async def main(argv: List[str]) -> int:
+    args = _parse_args(argv)
+    _setup_logging(args.verbose)
+
+    output_dir = os.path.abspath(os.path.expanduser(args.output))
+    os.makedirs(output_dir, exist_ok=True)
+
+    # ---- Phase 0: Initialise shared state ----
+    _phase_header("Initialise")
+    settings = load_settings(args.settings)
+    logger.info(
+        "LLM config from --settings: provider=%s model=%s api_base=%s",
+        settings["llm_provider"],
+        settings["llm_model"],
+        settings["llm_api_base"],
+    )
+    # Load the production base model list (replaces the old hardcoded list)
+    await init_supported_base_models()
+
+    # ---- Load entries ----
+    _phase_header("Load repo entries & construct initial metadata")
+    entries = load_repo_ids(args.models, max_models=args.sample if args.sample > 0 else None)
+    model_paths, repo_ids = create_all_initial_metadata(
+        entries, output_dir, skip_existing=True,
+    )
+    print(f"  {len(model_paths)} repos ready", file=sys.stderr)
+
+    # ---- Phase 1.5: Preprocessing audit ----
+    audit_summary: Dict[str, Any] = {}
+    t_start = time.perf_counter()
+    audit_summary = await _run_preprocessing_audit(entries, output_dir)
+
+    if args.audit_only:
+        total_wall = time.perf_counter() - t_start
+        print(f"\n  Audit-only done in {total_wall:.0f}s", file=sys.stderr)
+        print(f"  Audit data: {output_dir}/preprocessing_audit.json", file=sys.stderr)
+        return 0
+
+    # ---- Phase 2: Enrichment ----
+    enrichment_results: List[Dict[str, Any]] = []
+    if not args.no_enrich:
+        _phase_header("Enrich metadata via LLM")
+        enrichment_out = await _run_enrichment(
+            model_paths, repo_ids, output_dir, args.timeout, args.verbose,
+        )
+        enrichment_results = enrichment_out["results"]
+    else:
+        print("  Enrichment skipped (--no-enrich)", file=sys.stderr)
+
+    t_enrich = time.perf_counter()
+
+    # ---- Phase 3: Evaluation ----
+    _phase_header("Evaluate enriched metadata")
+    enriched = _collect_enriched_metadata(model_paths, repo_ids, enrichment_results)
+    scores = evaluate_batch(enriched)
+    agg = aggregate_scores(scores)
+    print(
+        f"  Mean total score: {agg.get('total_score', {}).get('mean', 'N/A')} / 100",
+        file=sys.stderr,
+    )
+    print(
+        f"  Models scored: {agg.get('model_count', 0)}",
+        file=sys.stderr,
+    )
+
+    # ---- Phase 4: Report generation ----
+    _phase_header("Generate reports")
+    duration_summary: Dict[str, Any] | None = None
+    if enrichment_results:
+        durations = [r.get("duration_s", 0) for r in enrichment_results if r.get("duration_s")]
+        if durations:
+            sorted_d = sorted(durations)
+            m = len(sorted_d) // 2
+            duration_summary = {
+                "total_wall_s": round(t_enrich - t_start, 1),
+                "mean_s": round(sum(durations) / len(durations), 1),
+                "median_s": round(sorted_d[m] if len(sorted_d) % 2 else (sorted_d[m - 1] + sorted_d[m]) / 2, 1),
+                "min_s": round(min(durations), 1),
+                "max_s": round(max(durations), 1),
+            }
+
+    # Check LLM config consistency after enrichment (LLMService is now initialized)
+    actual_llm_cfg = _get_actual_llm_config()
+    config_warnings = _compare_llm_config(settings, actual_llm_cfg)
+
+    save_json_report(
+        agg, scores, enrichment_results, output_dir, duration_summary,
+        audit_summary=audit_summary, config_warnings=config_warnings,
+    )
+    generate_markdown_report(
+        agg, scores, output_dir, duration_summary,
+        audit_summary=audit_summary, config_warnings=config_warnings,
+    )
+
+    # ---- Final summary ----
+    total_wall = time.perf_counter() - t_start
+    print(f"\n  Done in {total_wall:.0f}s ({total_wall / 60:.1f} min)", file=sys.stderr)
+    print(f"  Reports: {output_dir}/report.md, {output_dir}/report.json", file=sys.stderr)
+    print(file=sys.stderr)
+
+    return 0 if agg.get("success_count", 0) > 0 else 1
+
+
+def entry_point() -> int:
+    return asyncio.run(main(sys.argv[1:]))
+
+
+if __name__ == "__main__":
+    sys.exit(entry_point())

tests/enrich_hf_validation/test_data/base_model_ground_truth.json

@@ -0,0 +1,376 @@
+{
+  "description": "Ground truth base_model mapping for HF LoRA enrichment test data",
+  "generated_at": "2026-07-05T18:20:00+08:00",
+  "inference_method": "Manual analysis of YAML base_model field + README content + filename clues",
+  "canonical_list_source": "Fallback list in config.py + CivitAI production API (73 models total)",
+  "entries": [
+    {
+      "repo_id": "k2styles/krea-2-cobalt-sky-anime-lora",
+      "safetensors_name": "cobalt-sky-anime.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "k2styles/krea-2-azure-gouache-daylight-lora",
+      "safetensors_name": "azure-gouache-daylight.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "TheDivergentAI/krea2-turbo-distill-lora",
+      "safetensors_name": "krea2_turbo_distill_r128.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Raw",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "DeverStyle/Krea2-Loras",
+      "safetensors_name": "n0t_f4l_000001000.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "Komorebi1995/krea2-raw-jpaf-celpaint-lora",
+      "safetensors_name": "krea2_raw_jpaf_celpaint_full_v1.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "Filename contains 'krea2'"
+    },
+    {
+      "repo_id": "artificialguybr/pixelartredmond-1-5v-pixel-art-loras-for-sd-1-5",
+      "safetensors_name": "PixelArtRedmond15V-PixelArt-PIXARFK.safetensors",
+      "yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "Shakker-Labs/FLUX.1-dev-LoRA-Logo-Design",
+      "safetensors_name": "FLUX-dev-lora-Logo-Design.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.1-dev → dev → D"
+    },
+    {
+      "repo_id": "glif-loradex-trainer/bingbangboom_flux_surf",
+      "safetensors_name": "flux_surf_000001500.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "prithivMLmods/Ton618-Epic-Realism-Flux-LoRA",
+      "safetensors_name": "Epic-Realism-Unpruned.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "prithivMLmods/Fashion-Hut-Modeling-LoRA",
+      "safetensors_name": "Fashion-Modeling.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "prithivMLmods/Retro-Pixel-Flux-LoRA",
+      "safetensors_name": "Retro-Pixel.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "D1-3105/HiDream-E1-Full_lora",
+      "safetensors_name": "HiDream-E1-Full.safetensors",
+      "yaml_base_model_raw": "HiDream-ai/HiDream-E1-Full",
+      "correct_base_model": "HiDream",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field; filename contains 'HiDream'"
+    },
+    {
+      "repo_id": "renderartist/Classic-Painting-Z-Image-Turbo-LoRA",
+      "safetensors_name": "Classic_Painting_Z_Image_Turbo_v1_renderartist_1750.safetensors",
+      "yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
+      "correct_base_model": "ZImageTurbo",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field; filename contains 'Z-Image-Turbo'"
+    },
+    {
+      "repo_id": "DeverStyle/Z-Image-loras",
+      "safetensors_name": "z_image_archer_style.safetensors",
+      "yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
+      "correct_base_model": "ZImageTurbo",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "deadman44/Z-Image_LoRA",
+      "safetensors_name": "lora_zimage_turbo_myjs_alpha01.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "ZImageTurbo",
+      "confidence": "high",
+      "evidence": "Filename contains 'zimage_turbo'"
+    },
+    {
+      "repo_id": "zyuzuguldu/vton-lora-linen",
+      "safetensors_name": "pytorch_lora_weights.safetensors",
+      "yaml_base_model_raw": "stabilityai/stable-diffusion-xl-base-1.0",
+      "correct_base_model": "SDXL 1.0",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "svntax-dev/pixel_spritesheet_4walk_small_lora_v1",
+      "safetensors_name": "pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-4B",
+      "correct_base_model": "Flux.2 Klein 4B-base",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.2-klein-base-4B"
+    },
+    {
+      "repo_id": "Haruka041/z-image-anime-lora",
+      "safetensors_name": "sk_anime_style_v1.0.safetensors",
+      "yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
+      "correct_base_model": "ZImageTurbo",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "systms/SYSTMS-INFL8-LoRA-Wan22",
+      "safetensors_name": "SYSTMS_INFL8_LORA_WAN22_low_noise.safetensors",
+      "yaml_base_model_raw": "Wan-AI/Wan2.2-I2V-A14B",
+      "correct_base_model": "Wan Video 2.2 I2V-A14B",
+      "confidence": "high",
+      "evidence": "YAML base_model: Wan2.2-I2V-A14B"
+    },
+    {
+      "repo_id": "crafiq/flux-2-klein-9b-360-panorama-lora",
+      "safetensors_name": "flux-2-klein-9b-360-panorama-lora.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-9B",
+      "correct_base_model": "Flux.2 Klein 9B-base",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.2-klein-base-9B; filename contains 'flux-2-klein-9b'"
+    },
+    {
+      "repo_id": "Leon1000/pixel_spritesheet_4walk_small_lora_v1",
+      "safetensors_name": "pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-4B",
+      "correct_base_model": "Flux.2 Klein 4B-base",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.2-klein-base-4B; filename contains 'flux2_klein_base_4b'"
+    },
+    {
+      "repo_id": "Muapi/pov-missionary-legs-together-lora",
+      "safetensors_name": "pov-missionary-legs-together-lora.safetensors",
+      "yaml_base_model_raw": "OnomaAIResearch/Illustrious-xl-early-release-v0",
+      "correct_base_model": "Illustrious",
+      "confidence": "high",
+      "evidence": "YAML base_model: OnomaAIResearch/Illustrious-*"
+    },
+    {
+      "repo_id": "ostris/ideogram_4_unconditional_lora",
+      "safetensors_name": "ideogram_4_unconditional_lora_r16.safetensors",
+      "yaml_base_model_raw": "ideogram-ai/ideogram-4-fp8",
+      "correct_base_model": "Ideogram 4.0",
+      "confidence": "high",
+      "evidence": "YAML base_model: ideogram-ai/ideogram-4 → Ideogram 4.0; filename contains 'ideogram_4'"
+    },
+    {
+      "repo_id": "ilkerzgi/krea-2-bleached-surreal-uncanny-lora",
+      "safetensors_name": "bleached-surreal-uncanny-comfy.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "ilkerzgi/krea-2-azure-surreal-collage-lora",
+      "safetensors_name": "azure-surreal-collage-comfy.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "ilkerzgi/krea-2-airy-gouache-minimalist-lora",
+      "safetensors_name": "airy-gouache-minimalist-comfy.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "k2styles/krea-2-airy-watercolor-chibi-lora",
+      "safetensors_name": "airy-watercolor-chibi.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Turbo",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "TakeAswing/sdxl-lora-lofi",
+      "safetensors_name": "pytorch_lora_weights.safetensors",
+      "yaml_base_model_raw": "stabilityai/stable-diffusion-xl-base-1.0",
+      "correct_base_model": "SDXL 1.0",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field; repo name contains 'sdxl'"
+    },
+    {
+      "repo_id": "heville/anna-lora-krea2",
+      "safetensors_name": "pytorch_lora_weights.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Raw",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field; repo name contains 'krea2'"
+    },
+    {
+      "repo_id": "Brioch/krea2_loras",
+      "safetensors_name": "mashap_ohwx_woman_krea2.safetensors",
+      "yaml_base_model_raw": "krea/Krea-2-Raw",
+      "correct_base_model": "Krea 2",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "hr16/Miwano-Rag-LoRA",
+      "safetensors_name": "Miwano-Rag-epoch10.lora.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "README: base model is Kanianime (SD 1.5 fine-tune)"
+    },
+    {
+      "repo_id": "ikuseiso/Personal_Lora_collections",
+      "safetensors_name": "vergil_devil_may_cry.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "Sample prompt shows Model: AbyssOrangeMix (SD 1.5), 512x768"
+    },
+    {
+      "repo_id": "Tanger/LoraByTanger",
+      "safetensors_name": "(v4)layila-000005.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "README: trained on anything4.5 (SD 1.5) and nai (SD 1.5); test images on AbyssOrangeMix2_hard"
+    },
+    {
+      "repo_id": "DS-Archive/ds-LoRA",
+      "safetensors_name": "dsharu-v2_lc.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "README explicitly states 'Stable Diffusion 1.5'"
+    },
+    {
+      "repo_id": "soknife/loras",
+      "safetensors_name": "irys-regular-subject-more.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "README mentions SD 1.5 fine-tune models (PastelMix, AbyssOrangeMix, Anything)"
+    },
+    {
+      "repo_id": "prompthero/openjourney-lora",
+      "safetensors_name": "openjourneyLora.safetensors",
+      "yaml_base_model_raw": "stabilityai/stable-diffusion-2-1-base",
+      "correct_base_model": "SD 2.1",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "Banano/banchan-lora",
+      "safetensors_name": "Bananochan-PonySDXL-v2.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "Pony",
+      "confidence": "medium",
+      "evidence": "Filename contains 'PonySDXL-v2' → Pony base model"
+    },
+    {
+      "repo_id": "Maisman/No-Game-NoLife-LoRAs",
+      "safetensors_name": "ShiroNGNL2_Lora.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "Sample prompts show Model: abyssorangemix2_Hardcore (SD 1.5), 512x768"
+    },
+    {
+      "repo_id": "EarthnDusk/Gambit_Xmen_Anime_Lora_V1.1",
+      "safetensors_name": "RemyLebeau.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "Trained Feb 2023 via Kohya LoRA (pre-SDXL era), SD 1.5 lineage"
+    },
+    {
+      "repo_id": "EarthnDusk/DuskfallArt_LoRa",
+      "safetensors_name": "DuskfallArt.safetensors",
+      "yaml_base_model_raw": "stable-diffusion-v1-5/stable-diffusion-v1-5",
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "gaoxiao/pokemon-lora",
+      "safetensors_name": "pytorch_lora_weights.safetensors",
+      "yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "wtcherr/sd-unsplash_10k_canny-model-control-lora",
+      "safetensors_name": "diffusion_pytorch_model.safetensors",
+      "yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "wtcherr/sd-unsplash_10k_blur_rand_KS-model-control-lora",
+      "safetensors_name": "diffusion_pytorch_model.safetensors",
+      "yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
+      "correct_base_model": "SD 1.5",
+      "confidence": "high",
+      "evidence": "YAML frontmatter base_model field"
+    },
+    {
+      "repo_id": "samurai-architects/lora-starbucks",
+      "safetensors_name": "starbucks_interior.safetensors",
+      "yaml_base_model_raw": null,
+      "correct_base_model": null,
+      "confidence": "none",
+      "evidence": "README too minimal, no base_model in YAML, cannot determine"
+    },
+    {
+      "repo_id": "prithivMLmods/Flux-Long-Toon-LoRA",
+      "safetensors_name": "Long-Toon.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
+      "correct_base_model": "Flux.1 D",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.1-dev (dev → D)"
+    },
+    {
+      "repo_id": "Limbicnation/pixel-art-lora",
+      "safetensors_name": "pytorch_lora_weights.comfyui.safetensors",
+      "yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-4B",
+      "correct_base_model": "Flux.2 Klein 4B",
+      "confidence": "high",
+      "evidence": "YAML base_model: FLUX.2-klein-4B; README explicitly states base model"
+    }
+  ]
+}

tests/enrich_hf_validation/test_data/hf_lora_models_with_safetensors.txt

@@ -0,0 +1,46 @@
+k2styles/krea-2-cobalt-sky-anime-lora, cobalt-sky-anime.safetensors
+k2styles/krea-2-azure-gouache-daylight-lora, azure-gouache-daylight.safetensors
+TheDivergentAI/krea2-turbo-distill-lora, krea2_turbo_distill_r128.safetensors
+DeverStyle/Krea2-Loras, n0t_f4l_000001000.safetensors
+Komorebi1995/krea2-raw-jpaf-celpaint-lora, krea2_raw_jpaf_celpaint_full_v1.safetensors
+artificialguybr/pixelartredmond-1-5v-pixel-art-loras-for-sd-1-5, PixelArtRedmond15V-PixelArt-PIXARFK.safetensors
+Shakker-Labs/FLUX.1-dev-LoRA-Logo-Design, FLUX-dev-lora-Logo-Design.safetensors
+glif-loradex-trainer/bingbangboom_flux_surf, flux_surf_000001500.safetensors
+prithivMLmods/Ton618-Epic-Realism-Flux-LoRA, Epic-Realism-Unpruned.safetensors
+prithivMLmods/Fashion-Hut-Modeling-LoRA, Fashion-Modeling.safetensors
+prithivMLmods/Retro-Pixel-Flux-LoRA, Retro-Pixel.safetensors
+D1-3105/HiDream-E1-Full_lora, HiDream-E1-Full.safetensors
+renderartist/Classic-Painting-Z-Image-Turbo-LoRA, Classic_Painting_Z_Image_Turbo_v1_renderartist_1750.safetensors
+DeverStyle/Z-Image-loras, z_image_archer_style.safetensors
+deadman44/Z-Image_LoRA, lora_zimage_turbo_myjs_alpha01.safetensors
+zyuzuguldu/vton-lora-linen, pytorch_lora_weights.safetensors
+svntax-dev/pixel_spritesheet_4walk_small_lora_v1, pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors
+Haruka041/z-image-anime-lora, sk_anime_style_v1.0.safetensors
+systms/SYSTMS-INFL8-LoRA-Wan22, SYSTMS_INFL8_LORA_WAN22_low_noise.safetensors
+crafiq/flux-2-klein-9b-360-panorama-lora, flux-2-klein-9b-360-panorama-lora.safetensors
+Leon1000/pixel_spritesheet_4walk_small_lora_v1, pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors
+Muapi/pov-missionary-legs-together-lora, pov-missionary-legs-together-lora.safetensors
+ostris/ideogram_4_unconditional_lora, ideogram_4_unconditional_lora_r16.safetensors
+ilkerzgi/krea-2-bleached-surreal-uncanny-lora, bleached-surreal-uncanny-comfy.safetensors
+ilkerzgi/krea-2-azure-surreal-collage-lora, azure-surreal-collage-comfy.safetensors
+ilkerzgi/krea-2-airy-gouache-minimalist-lora, airy-gouache-minimalist-comfy.safetensors
+k2styles/krea-2-airy-watercolor-chibi-lora, airy-watercolor-chibi.safetensors
+TakeAswing/sdxl-lora-lofi, pytorch_lora_weights.safetensors
+heville/anna-lora-krea2, pytorch_lora_weights.safetensors
+Brioch/krea2_loras, mashap_ohwx_woman_krea2.safetensors
+hr16/Miwano-Rag-LoRA, Miwano-Rag-epoch10.lora.safetensors
+ikuseiso/Personal_Lora_collections, vergil_devil_may_cry.safetensors
+Tanger/LoraByTanger, (v4)layila-000005.safetensors
+DS-Archive/ds-LoRA, dsharu-v2_lc.safetensors
+soknife/loras, irys-regular-subject-more.safetensors
+prompthero/openjourney-lora, openjourneyLora.safetensors
+Banano/banchan-lora, Bananochan-PonySDXL-v2.safetensors
+Maisman/No-Game-NoLife-LoRAs, ShiroNGNL2_Lora.safetensors
+EarthnDusk/Gambit_Xmen_Anime_Lora_V1.1, RemyLebeau.safetensors
+EarthnDusk/DuskfallArt_LoRa, DuskfallArt.safetensors
+gaoxiao/pokemon-lora, pytorch_lora_weights.safetensors
+wtcherr/sd-unsplash_10k_canny-model-control-lora, diffusion_pytorch_model.safetensors
+wtcherr/sd-unsplash_10k_blur_rand_KS-model-control-lora, diffusion_pytorch_model.safetensors
+samurai-architects/lora-starbucks, starbucks_interior.safetensors
+prithivMLmods/Flux-Long-Toon-LoRA, Long-Toon.safetensors
+Limbicnation/pixel-art-lora, pytorch_lora_weights.comfyui.safetensors

tests/metadata_ops/test_readme_processor.py

@@ -0,0 +1,490 @@
+"""Tests for ``readme_processor.py`` — HF README processing for enrich_hf_metadata.
+
+Import via ``importlib`` to avoid the ``folder_paths`` dependency in
+``py.services.agent.__init__``.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+from pathlib import Path
+
+import pytest
+
+_MODULE_PATH = Path(__file__).parents[2] / "py" / "services" / "agent" / "skills" / "enrich_hf_metadata" / "readme_processor.py"
+
+
+@pytest.fixture(scope="session")
+def R():
+    """Load the ``readme_processor`` module once per session."""
+    spec = importlib.util.spec_from_file_location("readme_processor", str(_MODULE_PATH))
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+# ======================================================================
+# extract_gallery_images
+# ======================================================================
+
+
+class TestExtractGalleryImages:
+    def test_empty(self, R):
+        assert R.extract_gallery_images("", "repo") == []
+        assert R.extract_gallery_images("no frontmatter", "repo") == []
+
+    def test_no_widget(self, R):
+        readme = "---\ntags: [test]\n---\nbody"
+        assert R.extract_gallery_images(readme, "repo") == []
+
+    def test_widget_simple_text(self, R):
+        """YAML ``text: 'plain'`` → extracted as-is."""
+        readme = """---
+widget:
+- text: 'a cute cat'
+  output:
+    url: images/cat.png
+---"""
+        imgs = R.extract_gallery_images(readme, "user/repo")
+        assert len(imgs) == 1
+        assert imgs[0]["meta"]["prompt"] == "a cute cat"
+        assert "images/cat.png" in imgs[0]["url"]
+
+    def test_widget_unquoted_text(self, R):
+        """YAML ``text: plain value`` without quotes."""
+        readme = """---
+widget:
+- text: simple text
+  output:
+    url: img.png
+---"""
+        imgs = R.extract_gallery_images(readme, "user/repo")
+        assert len(imgs) == 1
+        assert imgs[0]["meta"]["prompt"] == "simple text"
+
+    def test_widget_block_scalar(self, R):
+        """YAML ``text: >-`` folded block scalar — extract actual content."""
+        readme = """---
+widget:
+- text: >-
+    Long toons, a close-up of a cartoon characters face is featured in a
+    vibrant red backdrop.
+  output:
+    url: images/LT4.png
+---"""
+        imgs = R.extract_gallery_images(readme, "user/repo")
+        assert len(imgs) == 1
+        prompt = imgs[0]["meta"]["prompt"]
+        assert "Long toons" in prompt
+        assert "vibrant red backdrop" in prompt
+        assert prompt != ">-"
+
+    def test_widget_dash_prefix_output(self, R):
+        """YAML ``- output:`` (dash prefix) — regression for widget parsing."""
+        readme = """---
+widget:
+- output:
+    url: images/test.png
+  text: dash test
+---"""
+        imgs = R.extract_gallery_images(readme, "user/repo")
+        assert len(imgs) == 1
+        assert imgs[0]["meta"]["prompt"] == "dash test"
+        assert "images/test.png" in imgs[0]["url"]
+
+    def test_widget_mixed_entries(self, R):
+        """Multiple widget entries with different text styles."""
+        readme = """---
+widget:
+- text: >-
+    First entry description.
+  output:
+    url: img1.png
+- text: second entry
+  output:
+    url: img2.png
+- text: 'third entry'
+  output:
+    url: img3.png
+---"""
+        imgs = R.extract_gallery_images(readme, "user/repo")
+        assert len(imgs) == 3
+        assert imgs[0]["meta"]["prompt"] == "First entry description."
+        assert imgs[1]["meta"]["prompt"] == "second entry"
+        assert imgs[2]["meta"]["prompt"] == "third entry"
+
+
+# ======================================================================
+# extract_simple_markdown_images
+# ======================================================================
+
+
+class TestExtractSimpleMarkdownImages:
+    def test_empty(self, R):
+        assert R.extract_simple_markdown_images("", "repo") == []
+
+    def test_basic_markdown_image(self, R):
+        """``![alt](./img.png)`` → absolute URL."""
+        imgs = R.extract_simple_markdown_images("![test](./image_0.png)", "u/r")
+        assert len(imgs) == 1
+        assert "image_0.png" in imgs[0]["url"]
+        assert imgs[0]["meta"]["prompt"] == "test"
+
+    def test_absolute_url(self, R):
+        """``![alt](https://...)`` → keep as-is."""
+        imgs = R.extract_simple_markdown_images(
+            "![img](https://example.com/img.png)", "u/r"
+        )
+        assert len(imgs) == 1
+        assert imgs[0]["url"] == "https://example.com/img.png"
+
+    def test_skips_code_fences(self, R):
+        """Inside ``` blocks should be ignored."""
+        text = """outside
+```
+![inside](./img.png)
+```
+outside again
+![valid](./valid.png)"""
+        imgs = R.extract_simple_markdown_images(text, "u/r")
+        assert len(imgs) == 1
+        assert "valid.png" in imgs[0]["url"]
+
+    def test_deduplicates(self, R):
+        text = "![a](./img.png)\n![b](./img.png)"
+        imgs = R.extract_simple_markdown_images(text, "u/r")
+        assert len(imgs) == 1  # deduplicated
+
+
+# ======================================================================
+# extract_html_img_tags
+# ======================================================================
+
+
+class TestExtractHtmlImgTags:
+    def test_double_quoted_src(self, R):
+        imgs = R.extract_html_img_tags('<img src="./img.png">', "u/r")
+        assert len(imgs) == 1
+        assert "img.png" in imgs[0]["url"]
+
+    def test_single_quoted_src(self, R):
+        imgs = R.extract_html_img_tags("<img src='./img.png'>", "u/r")
+        assert len(imgs) == 1
+        assert "img.png" in imgs[0]["url"]
+
+    def test_absolute_url(self, R):
+        imgs = R.extract_html_img_tags(
+            '<img src="https://cdn.example.com/img.png">', "u/r"
+        )
+        assert len(imgs) == 1
+        assert imgs[0]["url"] == "https://cdn.example.com/img.png"
+
+    def test_deduplicates_across_formats(self, R):
+        text = '<img src="./img.png">\n<img src=\'./img.png\'>'
+        imgs = R.extract_html_img_tags(text, "u/r")
+        assert len(imgs) == 1
+
+
+# ======================================================================
+# extract_gallery_table_images
+# ======================================================================
+
+
+class TestExtractGalleryTableImages:
+    def test_gallery_table(self, R):
+        text = """| Preview | Prompt |
+|--------|--------|
+| ![img](./a.png) | a cat |
+| ![img](./b.png) | a dog |"""
+        imgs = R.extract_gallery_table_images(text, "u/r")
+        assert len(imgs) == 2
+        assert imgs[0]["meta"]["prompt"] == "a cat"
+        assert "a.png" in imgs[0]["url"]
+        assert imgs[1]["meta"]["prompt"] == "a dog"
+
+    def test_skips_non_gallery_table(self, R):
+        text = """| Parameter | Value |
+|----------|-------|
+| Steps    | 4     |"""
+        imgs = R.extract_gallery_table_images(text, "u/r")
+        assert len(imgs) == 0
+
+
+# ======================================================================
+# clean_readme_for_llm  +  strip helpers
+# ======================================================================
+
+
+class TestCleanReadmeForLlm:
+    def test_preserves_plain_code_block(self, R):
+        """`` ``` `` without language tag → preserved (trigger words)."""
+        text = """Before
+```
+pixel art sprite, game asset
+```
+After"""
+        cleaned = R.clean_readme_for_llm(text)
+        assert "pixel art sprite" in cleaned
+        assert "game asset" in cleaned
+
+    def test_strips_fenced_code_with_lang(self, R):
+        """`` ```python `` → stripped."""
+        text = "before\n```python\nimport torch\n```\nafter"
+        cleaned = R.clean_readme_for_llm(text)
+        assert "import torch" not in cleaned
+        assert "before" in cleaned
+        assert "after" in cleaned
+
+    def test_preserves_markdown_image_url(self, R):
+        """``![alt](url)`` → URL kept for LLM preview extraction."""
+        text = "![sample](./preview.png)"
+        cleaned = R.clean_readme_for_llm(text)
+        assert "./preview.png" in cleaned
+
+    def test_converts_html_img_tag_to_markdown_image(self, R):
+        """``<img src="...">`` → ``![](src)`` preserving URL for LLM."""
+        text = 'before\n<img src="logo.png">\nafter'
+        cleaned = R.clean_readme_for_llm(text)
+        assert "![](logo.png)" in cleaned
+        assert "logo.png" in cleaned  # URL preserved for LLM extraction
+
+    def test_widget_stripped_frontmatter_preserved(self, R):
+        """Widget YAML stripped but ``base_model:`` kept."""
+        text = """---
+tags: [test]
+widget:
+- text: >-
+    long description here
+  output:
+    url: img.png
+base_model: black-forest-labs/FLUX.1-dev
+instance_prompt: test
+---"""
+        cleaned = R.clean_readme_for_llm(text)
+        assert "widget:" not in cleaned
+        assert "black-forest-labs/FLUX.1-dev" in cleaned
+        assert "instance_prompt: test" in cleaned
+
+    def test_training_table_stripped(self, R):
+        """Training-parameter table → stripped."""
+        text = """before
+| LR Scheduler | constant |
+|--------------|---------|
+| Optimizer    | AdamW   |
+after"""
+        cleaned = R.clean_readme_for_llm(text)
+        assert "LR Scheduler" not in cleaned
+        assert "Optimizer" not in cleaned
+        assert "before" in cleaned
+        assert "after" in cleaned
+
+    def test_best_dimensions_table_kept(self, R):
+        """Non-training table (Best Dimensions) → kept."""
+        text = """## Best Dimensions
+- 768 x 1024 (Best)
+- 1024 x 1024 (Default)"""
+        cleaned = R.clean_readme_for_llm(text)
+        assert "768 x 1024" in cleaned
+
+    def test_boilerplate_section_stripped(self, R):
+        text = """stuff
+## Download model
+[link](url)
+## Next section
+content"""
+        cleaned = R.clean_readme_for_llm(text)
+        assert "Download model" not in cleaned
+        assert "Next section" in cleaned
+        assert "content" in cleaned
+
+    def test_returns_empty_for_none(self, R):
+        assert R.clean_readme_for_llm(None) == ""
+
+    def test_returns_empty_for_empty(self, R):
+        assert R.clean_readme_for_llm("") == ""
+
+
+# ======================================================================
+# _is_heading  /  _heading_level
+# ======================================================================
+
+
+class TestHeadingDetection:
+    @pytest.mark.parametrize(
+        "line,expected",
+        [
+            ("# Title", 1),
+            ("## Sub", 2),
+            ("### Subsub", 3),
+            ("#### Subsubsub", 4),
+            ("<h1>Title</h1>", 1),
+            ("<h2>Sub</h2>", 2),
+            ("<h3 class='x'>Sub</h3>", 3),
+            ("<h4 id='y'>Sub</h4>", 4),
+            ("not a heading", 0),
+            ("###", 0),  # no text after ###
+            ("</h2>", 0),  # closing tag, not a heading
+            ("", 0),
+        ],
+    )
+    def test_heading_level(self, R, line, expected):
+        assert R._heading_level(line) == expected
+
+    @pytest.mark.parametrize(
+        "line,expected",
+        [
+            ("# Title", True),
+            ("<h2>Sub</h2>", True),
+            ("</h2>", False),  # closing tag
+            ("not heading", False),
+        ],
+    )
+    def test_is_heading(self, R, line, expected):
+        assert R._is_heading(line) == expected
+
+
+# ======================================================================
+# extract_relevant_section
+# ======================================================================
+
+
+class TestExtractRelevantSection:
+    def test_fallback_full_readme(self, R):
+        """No match → full README returned."""
+        readme = "# Title\n\nsome content"
+        assert R.extract_relevant_section(readme, "nonexistent") == readme
+
+    def test_empty_basename_returns_full(self, R):
+        readme = "# Title"
+        assert R.extract_relevant_section(readme, "") == readme
+
+    def test_match_heading_includes_yaml(self, R):
+        """Matching heading should still include YAML frontmatter."""
+        readme = """---
+base_model: foo
+---
+# My-Model-Title
+
+content
+## Subsection
+more"""
+        section = R.extract_relevant_section(readme, "My-Model")
+        assert "base_model: foo" in section
+        assert "content" in section
+        assert "Subsection" in section
+
+    def test_match_heading_includes_subheadings(self, R):
+        """``# Title`` match includes all ``##`` children."""
+        readme = """# Main Title
+
+## Child A
+content A
+## Child B
+content B
+## Child C
+content C"""
+        section = R.extract_relevant_section(readme, "Main Title")
+        assert "Child A" in section
+        assert "Child B" in section
+        assert "Child C" in section
+
+    def test_match_download_link(self, R):
+        """Download link containing basename → section extracted."""
+        readme = """# Collection
+## Model A
+[Download](./model_a.safetensors)
+## MyModel
+[Download](./mymodel.safetensors)
+content here
+## Model B
+other"""
+        section = R.extract_relevant_section(readme, "mymodel")
+        assert "content here" in section
+        assert "Model A" not in section  # should not include sibling
+
+    def test_heading_closing_tag_not_boundary(self, R):
+        """``</h2>`` should NOT be treated as a section boundary."""
+        readme = """# Title
+<p>some text</p>
+</h2>
+## Real Section
+content"""
+        section = R.extract_relevant_section(readme, "Title")
+        assert "Real Section" in section  # forward walk should not stop at </h2>
+        assert "content" in section
+
+
+# ======================================================================
+# _extract_frontmatter
+# ======================================================================
+
+
+class TestExtractFrontmatter:
+    def test_basic(self, R):
+        assert R._extract_frontmatter("---\ntags: [a]\n---\nbody") == "\ntags: [a]\n"
+
+    def test_no_frontmatter(self, R):
+        assert R._extract_frontmatter("no dashes") == ""
+
+    def test_empty_string(self, R):
+        assert R._extract_frontmatter("") == ""
+
+
+# ======================================================================
+# _strip_widget_section
+# ======================================================================
+
+
+class TestStripWidgetSection:
+    def test_strip_widget_keep_base_model(self, R):
+        """Widget stripped but ``base_model:`` preserved."""
+        text = """---
+tags: [test]
+widget:
+- text: >-
+    long text
+  output:
+    url: img.png
+base_model: black-forest-labs/FLUX.1-dev
+---"""
+        result = R._strip_widget_section(text)
+        assert "widget:" not in result
+        assert "black-forest-labs/FLUX.1-dev" in result
+
+    def test_no_widget_no_change(self, R):
+        text = "---\ntags: [a]\n---"
+        assert R._strip_widget_section(text) == text
+
+    def test_widget_at_end_of_frontmatter(self, R):
+        """Widget is the last YAML key before closing ---."""
+        text = """---
+base_model: a
+widget:
+- text: x
+  output:
+    url: y.png
+---"""
+        result = R._strip_widget_section(text)
+        assert "widget:" not in result
+        assert "base_model: a" in result
+
+
+# ======================================================================
+# _strip_fenced_code_blocks
+# ======================================================================
+
+
+class TestStripFencedCodeBlocks:
+    def test_strips_with_language(self, R):
+        text = "a\n```python\ncode\n```\nb"
+        assert R._strip_fenced_code_blocks(text) == "a\nb"
+
+    def test_keeps_plain_fence(self, R):
+        """`` ``` `` without language → preserved."""
+        text = "a\n```\ntrigger words\n```\nb"
+        assert "trigger words" in R._strip_fenced_code_blocks(text)
+
+    def test_pattern(self, R):
+        text = "x\n```yaml\nkey: val\n```\ny"
+        assert "key: val" not in R._strip_fenced_code_blocks(text)

tests/routes/__snapshots__/test_api_snapshots.ambr

@@ -28,6 +28,7 @@
     'settings': dict({
       'civitai_api_key_set': True,
       'language': 'en',
+      'llm_api_key_set': False,
       'theme': 'dark',
     }),
     'success': True,

tests/services/test_llm_service.py

@@ -0,0 +1,263 @@
+"""Tests for the LLMService."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from unittest import mock
+
+import pytest
+
+from py.services.llm_service import LLMService
+from py.services.errors import LLMNotConfiguredError, LLMRateLimitError, LLMResponseError
+
+
+class MockSettings:
+    """Minimal settings mock for LLMService tests."""
+
+    def __init__(self, **kwargs):
+        self._data = {
+            "llm_enabled": False,
+            "llm_provider": "openai",
+            "llm_api_key": "",
+            "llm_api_base": "",
+            "llm_model": "",
+        }
+        self._data.update(kwargs)
+
+    def get(self, key, default=None):
+        return self._data.get(key, default)
+
+
+class MockResponse:
+    """Mock aiohttp response."""
+
+    def __init__(self, status, json_data=None, text_data="", headers=None):
+        self.status = status
+        self._json_data = json_data
+        self._text_data = text_data
+        self.headers = headers or {}
+
+    async def json(self):
+        return self._json_data
+
+    async def text(self):
+        return self._text_data
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        pass
+
+
+class MockSession:
+    """Mock aiohttp ClientSession."""
+
+    def __init__(self, response):
+        self._response = response
+        self.closed = False
+
+    def post(self, url, json=None, headers=None):
+        self.last_url = url
+        self.last_json = json
+        self.last_headers = headers
+        return self._response
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        pass
+
+
+@pytest.fixture
+def llm_service():
+    """Create an LLMService with mock settings."""
+    LLMService.reset_instance()
+    settings = MockSettings(
+        llm_enabled=True,
+        llm_provider="openai",
+        llm_api_key="sk-test-key",
+        llm_api_base="",
+        llm_model="gpt-4o-mini",
+    )
+    return LLMService(settings)
+
+
+class TestLLMServiceConfiguration:
+    def test_is_configured_when_enabled_with_key_and_model(self, llm_service):
+        assert llm_service.is_configured() is True
+
+    def test_not_configured_when_disabled(self):
+        settings = MockSettings(
+            llm_enabled=False, llm_api_key="sk-test", llm_model="gpt-4o"
+        )
+        service = LLMService(settings)
+        # Lenient: model + API key is treated as configured even without
+        # the toggle, because the user clearly intends to use the feature.
+        assert service.is_configured() is True
+
+    def test_not_configured_without_model(self):
+        settings = MockSettings(llm_enabled=True, llm_api_key="sk-test", llm_model="")
+        service = LLMService(settings)
+        assert service.is_configured() is False
+
+    def test_not_configured_without_api_key_for_openai(self):
+        settings = MockSettings(llm_enabled=True, llm_api_key="", llm_model="gpt-4o")
+        service = LLMService(settings)
+        assert service.is_configured() is False
+
+    def test_ollama_configured_without_api_key(self):
+        settings = MockSettings(
+            llm_enabled=True, llm_provider="ollama", llm_api_key="", llm_model="llama3"
+        )
+        service = LLMService(settings)
+        assert service.is_configured() is True
+
+    def test_resolve_api_base_openai_default(self, llm_service):
+        assert llm_service._resolve_api_base("openai", "") == "https://api.openai.com/v1"
+
+    def test_resolve_api_base_ollama_default(self, llm_service):
+        assert llm_service._resolve_api_base("ollama", "") == "http://localhost:11434/v1"
+
+    def test_resolve_api_base_custom_override(self, llm_service):
+        assert llm_service._resolve_api_base("custom", "https://my.api.com/v1/") == "https://my.api.com/v1"
+
+    def test_ensure_configured_raises_when_disabled(self):
+        settings = MockSettings(llm_enabled=False)
+        service = LLMService(settings)
+        with pytest.raises(LLMNotConfiguredError):
+            service._ensure_configured()
+
+    def test_ensure_configured_raises_without_model(self):
+        settings = MockSettings(llm_enabled=True, llm_api_key="sk-test", llm_model="")
+        service = LLMService(settings)
+        with pytest.raises(LLMNotConfiguredError):
+            service._ensure_configured()
+
+    def test_not_configured_custom_without_api_base(self):
+        settings = MockSettings(
+            llm_enabled=True, llm_provider="custom",
+            llm_api_key="sk-test", llm_api_base="", llm_model="gpt-4o",
+        )
+        service = LLMService(settings)
+        assert service.is_configured() is False
+
+    def test_custom_configured_with_api_base(self):
+        settings = MockSettings(
+            llm_enabled=True, llm_provider="custom",
+            llm_api_key="sk-test",
+            llm_api_base="https://my.api.com/v1", llm_model="gpt-4o",
+        )
+        service = LLMService(settings)
+        assert service.is_configured() is True
+
+    def test_ensure_configured_raises_custom_without_api_base(self):
+        settings = MockSettings(
+            llm_enabled=True, llm_provider="custom",
+            llm_api_key="sk-test", llm_api_base="", llm_model="gpt-4o",
+        )
+        service = LLMService(settings)
+        with pytest.raises(LLMNotConfiguredError, match="API base URL"):
+            service._ensure_configured()
+
+
+class TestLLMServiceChatCompletion:
+    @pytest.mark.asyncio
+    async def test_chat_completion_success(self, llm_service):
+        mock_response = MockResponse(
+            200,
+            json_data={
+                "choices": [{"message": {"content": "Hello!"}}],
+                "usage": {"total_tokens": 10},
+                "model": "gpt-4o-mini",
+            },
+        )
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            result = await llm_service.chat_completion(
+                messages=[{"role": "user", "content": "Hi"}],
+            )
+
+        assert result["content"] == "Hello!"
+        assert result["usage"]["total_tokens"] == 10
+        assert result["model"] == "gpt-4o-mini"
+
+    @pytest.mark.asyncio
+    async def test_chat_completion_raises_on_not_configured(self):
+        settings = MockSettings(llm_enabled=False)
+        service = LLMService(settings)
+        with pytest.raises(LLMNotConfiguredError):
+            await service.chat_completion(messages=[])
+
+    @pytest.mark.asyncio
+    async def test_chat_completion_raises_on_http_error(self, llm_service):
+        mock_response = MockResponse(500, text_data="Internal Server Error")
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            with pytest.raises(LLMResponseError, match="HTTP 500"):
+                await llm_service.chat_completion(messages=[])
+
+    @pytest.mark.asyncio
+    async def test_chat_completion_raises_on_rate_limit(self, llm_service):
+        mock_response = MockResponse(429, text_data="Rate limited", headers={"Retry-After": "0"})
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            with pytest.raises(LLMRateLimitError):
+                await llm_service.chat_completion(
+                    messages=[], retry_on_rate_limit=False
+                )
+
+    @pytest.mark.asyncio
+    async def test_chat_completion_raises_on_bad_response_structure(self, llm_service):
+        mock_response = MockResponse(200, json_data={"unexpected": "data"})
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            with pytest.raises(LLMResponseError, match="Unexpected LLM response"):
+                await llm_service.chat_completion(messages=[])
+
+
+class TestLLMServiceChatCompletionJson:
+    @pytest.mark.asyncio
+    async def test_chat_completion_json_parses_json(self, llm_service):
+        mock_response = MockResponse(
+            200,
+            json_data={
+                "choices": [{"message": {"content": '{"key": "value"}'}}],
+                "usage": {},
+                "model": "gpt-4o-mini",
+            },
+        )
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            result = await llm_service.chat_completion_json(
+                system_prompt="You are helpful.",
+                user_prompt="Return JSON.",
+            )
+
+        assert result == {"key": "value"}
+
+    @pytest.mark.asyncio
+    async def test_chat_completion_json_raises_on_non_json(self, llm_service):
+        # Non-JSON content raises LLMResponseError (salvage also fails)
+        mock_response = MockResponse(
+            200,
+            json_data={
+                "choices": [{"message": {"content": "not json at all"}}],
+                "usage": {},
+            },
+        )
+        mock_session = MockSession(mock_response)
+
+        with mock.patch("aiohttp.ClientSession", return_value=mock_session):
+            with pytest.raises(LLMResponseError, match="could not be parsed as JSON"):
+                await llm_service.chat_completion_json(
+                    system_prompt="test",
+                    user_prompt="test",
+                )

tests/services/test_post_processor.py

@@ -0,0 +1,434 @@
+"""Tests for the PostProcessor (py/services/agent/post_processor.py).
+
+PostProcessor delegates all I/O to AgentCLI — these tests mock AgentCLI
+functions and verify the business logic (conditions, merges, dispatch).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from unittest import mock
+
+import pytest
+
+from py.services.agent.post_processor import PostProcessor
+
+
+@pytest.fixture
+def processor():
+    return PostProcessor()
+
+
+# ======================================================================
+# process() — routing
+# ======================================================================
+
+
+class TestProcessDispatch:
+    @pytest.mark.asyncio
+    async def test_unknown_skill_returns_error(self, processor):
+        result = await processor.process(
+            skill_name="nonexistent",
+            model_path="/p.safetensors",
+            llm_output={},
+            metadata={},
+        )
+        assert result["success"] is False
+        assert "nonexistent" in result["errors"][0]
+
+    @pytest.mark.asyncio
+    async def test_enrich_hf_metadata_routes_correctly(self, processor):
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview") as mock_dl,
+            mock.patch("py.metadata_ops.refresh_cache") as mock_ref,
+        ):
+            mock_apply.return_value = ["metadata_source"]
+            mock_dl.return_value = None
+
+            result = await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output={},
+                metadata={"from_civitai": True},
+            )
+
+        assert result["success"] is True
+
+
+# ======================================================================
+# enrich_hf_metadata — field-level logic
+# ======================================================================
+
+
+class TestEnrichHfMetadata:
+    """Business logic tests for the enrich_hf_metadata post-processor."""
+
+    MIN_LLM_OUTPUT = {
+        "base_model": "",
+        "trigger_words": [],
+        "short_description": "",
+        "tags": [],
+        "recommended_width": 0,
+        "recommended_height": 0,
+        "preview_url": "",
+        "confidence": "low",
+    }
+
+    # -- base_model ------------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_base_model_overwrites_empty(self, processor):
+        """Empty current base_model → new value is applied."""
+        llm = {**self.MIN_LLM_OUTPUT, "base_model": "Flux.1 D"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"base_model": ""},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert applied["base_model"] == "Flux.1 D"
+
+    @pytest.mark.asyncio
+    async def test_base_model_does_not_overwrite_existing_civitai(self, processor):
+        """Existing base_model from CivitAI → not overwritten."""
+        llm = {**self.MIN_LLM_OUTPUT, "base_model": "Flux.1 D"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"base_model": "SDXL 1.0", "from_civitai": True},
+            )
+        # apply IS called (metadata_source, llm_enriched_at) but base_model not in it
+        applied = mock_apply.call_args[0][1]
+        assert "base_model" not in applied
+
+    @pytest.mark.asyncio
+    async def test_base_model_overwrites_existing_hf_model(self, processor):
+        """Existing base_model from HF → overwritten (LLM is more reliable)."""
+        llm = {**self.MIN_LLM_OUTPUT, "base_model": "Flux.1 D"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"base_model": "SD 1.5", "from_civitai": False},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert applied["base_model"] == "Flux.1 D"
+
+    @pytest.mark.asyncio
+    async def test_base_model_skipped_when_llm_empty(self, processor):
+        """LLM returns empty base_model → nothing written."""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={"base_model": ""},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert "base_model" not in applied
+
+    # -- trigger_words ---------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_trigger_words_merged(self, processor):
+        """New trigger words written when current list is empty."""
+        llm = {**self.MIN_LLM_OUTPUT, "trigger_words": ["trigger1", "trigger2"]}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"trainedWords": []},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert applied["civitai"]["trainedWords"] == ["trigger1", "trigger2"]
+
+    # -- short_description → civitai.description -------------------------
+
+    @pytest.mark.asyncio
+    async def test_short_description_written_to_civitai(self, processor):
+        """short_description written to civitai.description for HF models."""
+        llm = {**self.MIN_LLM_OUTPUT, "short_description": "A short summary"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"from_civitai": False},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert applied["civitai"]["description"] == "A short summary"
+
+    @pytest.mark.asyncio
+    async def test_short_description_skipped_for_civitai_model(self, processor):
+        """short_description NOT written for CivitAI models (has own description)."""
+        llm = {**self.MIN_LLM_OUTPUT, "short_description": "A short summary"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"from_civitai": True},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert "civitai" not in applied or "description" not in applied.get("civitai", {})
+
+    # -- readme_content → modelDescription -------------------------------
+
+    @pytest.mark.asyncio
+    async def test_readme_content_converted_to_model_description(self, processor):
+        """Raw README converted to HTML and stored as modelDescription."""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={"from_civitai": False},
+                readme_content="# Hello\n\nThis is **bold**.",
+            )
+        applied = mock_apply.call_args[0][1]
+        assert "<h1>Hello</h1>" in applied.get("modelDescription", "")
+        assert "<strong>bold</strong>" in applied.get("modelDescription", "")
+
+    @pytest.mark.asyncio
+    async def test_readme_content_skipped_for_civitai_model(self, processor):
+        """README content NOT converted for CivitAI models."""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={"from_civitai": True},
+                readme_content="# Hello",
+            )
+        applied = mock_apply.call_args[0][1]
+        assert "modelDescription" not in applied
+
+    # -- gallery images → civitai.images ---------------------------------
+
+    @pytest.mark.asyncio
+    async def test_gallery_images_extracted_from_readme(self, processor):
+        """Widget entries in README → civitai.images."""
+        readme = """---
+widget:
+- text: "a cat"
+  output:
+    url: images/cat.png
+---
+Content
+"""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={
+                    "from_civitai": False,
+                    "hf_url": "https://huggingface.co/user/repo",
+                },
+                readme_content=readme,
+            )
+        applied = mock_apply.call_args[0][1]
+        images = applied.get("civitai", {}).get("images", [])
+        assert len(images) == 1
+        assert images[0]["url"] == (
+            "https://huggingface.co/user/repo/resolve/main/images/cat.png"
+        )
+        assert images[0]["meta"]["prompt"] == "a cat"
+
+    @pytest.mark.asyncio
+    async def test_gallery_images_skipped_for_civitai_model(self, processor):
+        """Gallery images NOT extracted for CivitAI models."""
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=None),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={
+                    "from_civitai": True,
+                    "hf_url": "https://huggingface.co/user/repo",
+                },
+                readme_content="---\nwidget:\n- text: a\n  output:\n    url: x.png\n---\n",
+            )
+        applied = mock_apply.call_args[0][1]
+        civitai = applied.get("civitai", {})
+        assert "images" not in civitai
+
+    # -- tags ------------------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_tags_merged_and_deduplicated(self, processor):
+        llm = {**self.MIN_LLM_OUTPUT, "tags": ["flux", "lora", "STYLE"]}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"tags": ["anime"], "from_civitai": False},
+            )
+        merged = mock_apply.call_args[0][1]["tags"]
+        assert "anime" in merged
+        assert "flux" in merged
+        assert "style" in merged  # lowercased
+        # "lora" and "STYLE" → "lora" and "style"
+        assert len(merged) == 4  # anime, flux, lora, style
+
+    # -- metadata_source & llm_enriched_at --------------------------------
+
+    @pytest.mark.asyncio
+    async def test_audit_fields_always_set(self, processor):
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={},
+            )
+        applied = mock_apply.call_args[0][1]
+        assert applied["metadata_source"] == "agent:enrich_hf_metadata"
+        assert "llm_enriched_at" in applied
+
+    # -- preview download ------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_preview_downloaded_when_url_provided(self, processor):
+        llm = {**self.MIN_LLM_OUTPUT, "preview_url": "https://ex.com/img.png"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
+            mock.patch("py.metadata_ops.download_preview") as mock_dl,
+            mock.patch("py.metadata_ops.refresh_cache"),
+        ):
+            mock_dl.return_value = "/p.webp"
+            result = await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={},
+            )
+        assert result["preview_downloaded"] is True
+        mock_dl.assert_awaited_once_with("/p.safetensors", "https://ex.com/img.png")
+        applied = mock_apply.call_args[0][1]
+        assert applied["preview_url"] == "/p.webp"
+
+    @pytest.mark.asyncio
+    async def test_preview_skipped_when_exists(self, processor):
+        """If current_preview file exists on disk, skip download."""
+        llm = {**self.MIN_LLM_OUTPUT, "preview_url": "https://ex.com/img.png"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates"),
+            mock.patch("py.metadata_ops.download_preview") as mock_dl,
+            mock.patch("py.metadata_ops.refresh_cache"),
+            mock.patch("os.path.exists", return_value=True),
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"preview_url": "/existing/preview.webp"},
+            )
+        mock_dl.assert_not_called()
+
+    # -- cache refresh ---------------------------------------------------
+
+    @pytest.mark.asyncio
+    async def test_cache_refreshed_when_updates_applied(self, processor):
+        llm = {**self.MIN_LLM_OUTPUT, "base_model": "Flux.1 D"}
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates", return_value=["base_model"]),
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache") as mock_ref,
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=llm,
+                metadata={"base_model": ""},
+            )
+        mock_ref.assert_awaited_once_with("/p.safetensors")
+
+    @pytest.mark.asyncio
+    async def test_cache_not_refreshed_when_nothing_changed(self, processor):
+        with (
+            mock.patch("py.metadata_ops.apply_metadata_updates", return_value=[]),
+            mock.patch("py.metadata_ops.download_preview", return_value=False),
+            mock.patch("py.metadata_ops.refresh_cache") as mock_ref,
+        ):
+            await processor.process(
+                skill_name="enrich_hf_metadata",
+                model_path="/p.safetensors",
+                llm_output=self.MIN_LLM_OUTPUT,
+                metadata={"base_model": ""},
+            )
+        mock_ref.assert_not_called()
+
+
+# ======================================================================
+# Unit: _merge_tags
+# ======================================================================
+
+
+class TestMergeTags:
+    def test_deduplicates_case_insensitive(self):
+        existing = ["anime", "Flux"]
+        new = ["flux", "LORA", "anime"]
+        result = PostProcessor._merge_tags(existing, new)
+        # All tags are lowercased (matching TagUpdateService behaviour)
+        assert result == ["anime", "flux", "lora"]

tests/services/test_skill_registry.py

@@ -0,0 +1,88 @@
+"""Tests for the SkillRegistry (``prompt.md`` discovery + prompt loading)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from py.services.agent.skill_registry import SkillRegistry
+from py.services.agent.skill_definition import SkillDefinition, SkillPermissions
+
+
+@pytest.fixture
+def registry():
+    """Create a SkillRegistry with the real skills directory."""
+    SkillRegistry.reset_instance()
+    reg = SkillRegistry()
+    reg._discover()
+    return reg
+
+
+class TestSkillRegistryDiscovery:
+    def test_discovers_enrich_hf_metadata_skill(self, registry):
+        skills = registry.list_skills()
+        assert len(skills) >= 1
+        skill = registry.get_skill("enrich_hf_metadata")
+        assert skill is not None
+        assert skill.name == "enrich_hf_metadata"
+        assert skill.llm_required is True
+
+    def test_skill_has_correct_model_type_filter(self, registry):
+        skill = registry.get_skill("enrich_hf_metadata")
+        # model_type_filter was removed from prompt.md — defaults to None (all types)
+        assert skill.model_type_filter is None
+
+    def test_skill_has_permissions(self, registry):
+        skill = registry.get_skill("enrich_hf_metadata")
+        assert skill.permissions.write_metadata is True
+        assert skill.permissions.write_previews is True
+        # network_domains defaults to () since permissions block was removed
+
+    def test_get_skill_returns_none_for_unknown(self, registry):
+        assert registry.get_skill("nonexistent_skill") is None
+
+
+class TestSkillRegistryLoading:
+    def test_load_prompt_returns_content(self, registry):
+        prompt = registry.load_prompt("enrich_hf_metadata")
+        assert isinstance(prompt, str)
+        assert len(prompt) > 100
+        assert "base_model" in prompt
+        assert "trigger_words" in prompt
+
+    def test_load_prompt_raises_for_unknown_skill(self, registry):
+        with pytest.raises((FileNotFoundError, ValueError)):
+            registry.load_prompt("nonexistent")
+
+
+class TestSkillDefinition:
+    def test_applies_to_model_type_with_filter(self):
+        sd = SkillDefinition(
+            name="test",
+            title="Test",
+            description="",
+            llm_required=False,
+            model_type_filter=["lora"],
+        )
+        assert sd.applies_to_model_type("lora") is True
+        assert sd.applies_to_model_type("checkpoint") is False
+
+    def test_applies_to_model_type_without_filter(self):
+        sd = SkillDefinition(
+            name="test",
+            title="Test",
+            description="",
+            llm_required=False,
+            model_type_filter=None,
+        )
+        assert sd.applies_to_model_type("lora") is True
+        assert sd.applies_to_model_type("checkpoint") is True
+
+
+class TestSkillPermissions:
+    def test_defaults(self):
+        sp = SkillPermissions()
+        assert sp.write_metadata is True
+        assert sp.write_previews is True
+        assert sp.network_domains == ()