fix(agent): persist the LLM confidence through metadata writes

The post-processor stored the LLM's confidence as `_llm_confidence`, but
that value could never be read back: `BaseModelMetadata.from_dict()`
deliberately excludes underscore-prefixed keys from `_unknown_fields` and
`to_dict()` strips private fields, so it was erased by the next metadata
write and was invisible to `read_metadata()`.  The enrichment evaluation
harness reads this field to score runs, so confidence was always scored
as blank.

Store it as `llm_confidence`, which round-trips as an ordinary unknown
field — the same mechanism `llm_enriched_at` already relies on.  Nothing
else consumed the old name, and the harness still accepts it so sidecars
written by earlier versions keep evaluating.

Covered by a metadata load/save round-trip regression test plus
assertions that the post-processor writes the persisted key and no longer
writes the private one.
This commit is contained in:
Will Miao
2026-09-14 20:42:14 +08:00
parent 4064ea7d3a
commit 51de85a6ca
4 changed files with 92 additions and 3 deletions
+39
View File
@@ -439,6 +439,45 @@ Content
assert applied["metadata_source"] == "agent:enrich_hf_metadata"
assert "llm_enriched_at" in applied
@pytest.mark.asyncio
async def test_confidence_is_stored_under_a_persisted_key(self, processor):
"""`llm_confidence` must not be underscore-prefixed.
Underscore-prefixed keys are dropped by `BaseModelMetadata`, which made
`_llm_confidence` vanish on the next metadata write.
"""
llm = {**self.MIN_LLM_OUTPUT, "confidence": "medium"}
with (
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
mock.patch("py.metadata_ops.download_preview", return_value=False),
mock.patch("py.metadata_ops.refresh_cache"),
):
await processor.process(
skill_name="enrich_hf_metadata",
model_path="/p.safetensors",
llm_output=llm,
metadata={},
)
applied = mock_apply.call_args[0][1]
assert applied["llm_confidence"] == "medium"
assert "_llm_confidence" not in applied
@pytest.mark.asyncio
async def test_confidence_absent_when_the_llm_reported_none(self, processor):
llm = {**self.MIN_LLM_OUTPUT, "confidence": ""}
with (
mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply,
mock.patch("py.metadata_ops.download_preview", return_value=False),
mock.patch("py.metadata_ops.refresh_cache"),
):
await processor.process(
skill_name="enrich_hf_metadata",
model_path="/p.safetensors",
llm_output=llm,
metadata={},
)
assert "llm_confidence" not in mock_apply.call_args[0][1]
# -- preview download ------------------------------------------------
@pytest.mark.asyncio