refactor(agent): rename md_to_html to readme_processor, fix section extraction, widget parsing, and list_base_models

- Rename md_to_html.py → readme_processor.py (file no longer just HTML conversion)
- _extract_section: include YAML frontmatter, use heading-level-aware forward
  walk (sub-headings under # are included), increase walk limit past 30 lines
- _is_heading: exclude </hN> closing tags from boundary detection
- _heading_level: new helper for heading-level-aware section matching
- css: yield 0 for heading like closing tags, was unexpectedly caught by _is_heading
- extract_gallery_images: fix YAML block scalar (text: >-) prompt extraction;
  use endswith instead of == to detect the block marker
- _strip_widget_section: add to clean_readme_for_llm (widget text is handled
  by post-processor, not needed in LLM prompt)
- _strip_standalone_images: keep markdown image URLs intact for LLM preview
  extraction (was stripping to alt text only)
- list_base_models: switch from scanner-cache aggregation to
  CivitaiBaseModelService.get_base_models() - always returns full list
- Ollama: add num_ctx=32768 to payload options so thinking models have room
  to both reason and produce output
- Add tests/agent_cli/test_readme_processor.py: 59 tests covering extraction,
  cleaning, section matching, heading detection
- Update existing tests for behavioral changes
This commit is contained in:
Will Miao
2026-07-05 06:39:54 +08:00
parent 905c37290f
commit dd3aa97d0a
8 changed files with 733 additions and 159 deletions

View File

@@ -364,6 +364,9 @@ class LLMService:
"think": False,
"options": {
"temperature": temperature,
# Allow up to 32K context so the model has room to think
# AND produce output without hitting the 4K default limit.
"num_ctx": 32768,
},
}
if response_format is not None:
@@ -381,6 +384,16 @@ class LLMService:
if max_tokens is not None:
payload["max_tokens"] = max_tokens
if is_ollama:
logger.info(
"Ollama request: model=%s num_ctx=%s num_predict=%s format=%s think=%s",
payload.get("model"),
payload.get("options", {}).get("num_ctx"),
payload.get("options", {}).get("num_predict"),
payload.get("format", "none"),
payload.get("think"),
)
headers = self._build_headers(cfg["api_key"])
attempt = 0
@@ -507,8 +520,23 @@ class LLMService:
)
try:
return json.loads(result["content"])
parsed = json.loads(result["content"])
logger.info(
"LLM response base_model=%s tags=%s confidence=%s",
parsed.get("base_model", "?")[:50],
parsed.get("tags", []),
parsed.get("confidence", "?"),
)
logger.info(
"LLM raw content: %s",
(result.get("content") or "")[:1200],
)
return parsed
except (json.JSONDecodeError, TypeError) as exc:
logger.info(
"LLM raw response (first 800 chars): %s",
(result.get("content") or "")[:800],
)
logger.warning(
"LLM JSON parse failed on first attempt: %s. Retrying.", exc
)