diff --git a/py/services/agent/post_processor.py b/py/services/agent/post_processor.py index d8f12d6e..2d663455 100644 --- a/py/services/agent/post_processor.py +++ b/py/services/agent/post_processor.py @@ -273,10 +273,13 @@ class PostProcessor: updates["metadata_source"] = "agent:enrich_hf_metadata" updates["llm_enriched_at"] = datetime.now(timezone.utc).isoformat() - # Store LLM confidence in metadata so it's accessible for evaluation + # LLM confidence, stored for the enrichment evaluation harness. The key + # must NOT start with an underscore: `BaseModelMetadata.from_dict()` + # deliberately drops underscore-prefixed keys so they never round-trip, + # which silently erased this field on the next metadata write. raw_confidence = (llm_output.get("confidence") or "").strip() if raw_confidence: - updates["_llm_confidence"] = raw_confidence + updates["llm_confidence"] = raw_confidence # Fallback: use the trigger words the site records for this exact file, # then the README's YAML `instance_prompt`, when the LLM returned none. diff --git a/tests/enrich_hf_validation/evaluation_engine.py b/tests/enrich_hf_validation/evaluation_engine.py index 74181e85..17c3db5c 100644 --- a/tests/enrich_hf_validation/evaluation_engine.py +++ b/tests/enrich_hf_validation/evaluation_engine.py @@ -108,7 +108,12 @@ def evaluate_model( model_description: str = metadata.get("modelDescription") or "" base_model: str = metadata.get("base_model") or "" preview_url: str = metadata.get("preview_url") or "" - confidence: str = metadata.get("_llm_confidence") or "" + # `_llm_confidence` is the legacy key: underscore-prefixed metadata keys are + # deliberately not persisted through `BaseModelMetadata`, so older sidecars + # may still carry it while current ones use `llm_confidence`. + confidence: str = ( + metadata.get("llm_confidence") or metadata.get("_llm_confidence") or "" + ) # --- base_model --- base_model_valid = base_model in SUPPORTED_BASE_MODELS diff --git a/tests/services/test_post_processor.py b/tests/services/test_post_processor.py index 567aeb06..2780ab36 100644 --- a/tests/services/test_post_processor.py +++ b/tests/services/test_post_processor.py @@ -439,6 +439,45 @@ Content assert applied["metadata_source"] == "agent:enrich_hf_metadata" assert "llm_enriched_at" in applied + @pytest.mark.asyncio + async def test_confidence_is_stored_under_a_persisted_key(self, processor): + """`llm_confidence` must not be underscore-prefixed. + + Underscore-prefixed keys are dropped by `BaseModelMetadata`, which made + `_llm_confidence` vanish on the next metadata write. + """ + llm = {**self.MIN_LLM_OUTPUT, "confidence": "medium"} + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=False), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=llm, + metadata={}, + ) + applied = mock_apply.call_args[0][1] + assert applied["llm_confidence"] == "medium" + assert "_llm_confidence" not in applied + + @pytest.mark.asyncio + async def test_confidence_absent_when_the_llm_reported_none(self, processor): + llm = {**self.MIN_LLM_OUTPUT, "confidence": ""} + with ( + mock.patch("py.metadata_ops.apply_metadata_updates") as mock_apply, + mock.patch("py.metadata_ops.download_preview", return_value=False), + mock.patch("py.metadata_ops.refresh_cache"), + ): + await processor.process( + skill_name="enrich_hf_metadata", + model_path="/p.safetensors", + llm_output=llm, + metadata={}, + ) + assert "llm_confidence" not in mock_apply.call_args[0][1] + # -- preview download ------------------------------------------------ @pytest.mark.asyncio diff --git a/tests/utils/test_metadata_manager.py b/tests/utils/test_metadata_manager.py index 0be417fe..2097e8e7 100644 --- a/tests/utils/test_metadata_manager.py +++ b/tests/utils/test_metadata_manager.py @@ -212,3 +212,45 @@ async def test_self_healed_sidecar_is_parseable(tmp_path) -> None: assert metadata.file_name == "MyModel" assert metadata.model_name == "My Model" assert metadata.sha256 == "abc123" + + +@pytest.mark.asyncio +async def test_provenance_fields_survive_a_load_save_round_trip(tmp_path) -> None: + """Non-underscore extras must persist; underscore keys are ephemeral. + + Regression test for `llm_confidence`: it was written as `_llm_confidence`, + which `BaseModelMetadata.from_dict()` drops (and `to_dict()` strips), so the + value was erased by the next metadata write and was invisible to + `read_metadata()`. The enrichment evaluation harness depends on it. + """ + model_path = tmp_path / "Model.safetensors" + model_path.write_bytes(b"fake model data") + metadata_path = tmp_path / "Model.metadata.json" + + payload = { + "file_path": str(model_path), + "file_name": "Model", + "model_name": "Model", + "sha256": "deadbeef", + "base_model": "Krea 2", + "preview_url": "", + "metadata_source": "agent:enrich_hf_metadata", + "llm_enriched_at": "2026-01-01T00:00:00+00:00", + "llm_confidence": "medium", + "_llm_confidence": "medium", + } + assert await MetadataManager.save_metadata(str(model_path), payload) is True + + # A read must surface the supported key... + loaded = await MetadataManager.load_metadata_payload(str(model_path)) + assert loaded["llm_confidence"] == "medium" + assert loaded["metadata_source"] == "agent:enrich_hf_metadata" + + # ...but the underscore alias is intentionally not persisted. + assert "_llm_confidence" not in loaded + + # Re-saving what we read must not lose the confidence. + assert await MetadataManager.save_metadata(str(model_path), loaded) is True + saved = json.loads(metadata_path.read_text(encoding="utf-8")) + assert saved["llm_confidence"] == "medium" + assert saved["llm_enriched_at"] == "2026-01-01T00:00:00+00:00"