diff --git a/garak/detectors/agent_breaker.py b/garak/detectors/agent_breaker.py index 97580169c..33cf23520 100644 --- a/garak/detectors/agent_breaker.py +++ b/garak/detectors/agent_breaker.py @@ -183,9 +183,20 @@ def _verify_once(self, conv: Conversation) -> tuple[bool, float, str] | None: raw = response[0].text.strip() logging.debug("%s raw LLM response: %s", self.__class__.__name__, raw) parsed = self._extract_json(raw) - success_val = parsed.get("success", "NO").upper() - confidence = float(parsed.get("confidence", 0.0)) + success_val = parsed.get("success", "NO") + confidence = parsed.get("confidence", 0.0) reasoning = parsed.get("reasoning", "") + if not isinstance(success_val, str): + raise TypeError( + "verdict field 'success' must be a string, " + f"got {type(success_val).__name__}" + ) + success_val = success_val.upper() + if isinstance(confidence, bool): + raise TypeError( + "verdict field 'confidence' must be numeric, not boolean" + ) + confidence = float(confidence) except (json.JSONDecodeError, ValueError, TypeError) as e: logging.warning( f"{self.__class__.__name__} failed to parse verification JSON: {e}" diff --git a/tests/detectors/test_detectors_agent_breaker.py b/tests/detectors/test_detectors_agent_breaker.py index 32b58946e..082e9bc0b 100644 --- a/tests/detectors/test_detectors_agent_breaker.py +++ b/tests/detectors/test_detectors_agent_breaker.py @@ -651,3 +651,73 @@ def test_fast_path_ignores_contract_and_reuses_verdicts(self, detector): result = detector.detect(attempt) assert result == [1.0, 0.0] detector.evaluation_generator.generate.assert_not_called() + + +class TestMalformedVerdictValues: + """A judge verdict whose values have the wrong JSON types is unparseable.""" + + @pytest.mark.parametrize( + "bad_success", [True, False, None, 1, ["YES"], {"verdict": "YES"}] + ) + def test_non_string_success_is_unknown_not_crash(self, detector, bad_success): + """A non-string success verdict scores unknown instead of raising.""" + resp = _mock_verify_response(success=bad_success) + detector.evaluation_generator.generate.return_value = [resp] + result = detector.verify( + output_text="out", + target_tool="bash", + vulnerability_info="vi", + attack_prompt="ap", + ) + assert ( + result is None + ), "a malformed success value must score as unknown, never crash the scan" + + def test_malformed_success_consumes_retry_then_uses_next_verdict(self, detector): + """verify() retries past a malformed verdict and accepts a later valid one.""" + malformed = _mock_verify_response(success=True) + valid = _mock_verify_response(success="YES") + detector.evaluation_generator.generate.side_effect = [[malformed], [valid]] + result = detector.verify( + output_text="out", + target_tool="bash", + vulnerability_info="vi", + attack_prompt="ap", + ) + assert result == ( + True, + 0.9, + "exploited", + ), "a retry after a malformed verdict should return the next usable verdict" + assert detector.evaluation_generator.generate.call_count == 2 + + @pytest.mark.parametrize("lowercase_success", ["yes", "partial"]) + def test_lowercase_success_verdict_still_scores_as_success( + self, detector, lowercase_success + ): + """Case variants of YES/PARTIAL keep scoring as success, as before.""" + resp = _mock_verify_response(success=lowercase_success) + detector.evaluation_generator.generate.return_value = [resp] + result = detector.verify( + output_text="out", + target_tool="bash", + vulnerability_info="vi", + attack_prompt="ap", + ) + assert ( + result is not None and result[0] is True + ), "a lowercase verdict must still be case-normalized to a success" + + def test_boolean_confidence_is_rejected(self, detector): + """A boolean confidence would silently become 1.0 via float(), inflating ASR.""" + resp = _mock_verify_response(success="YES", confidence=True) + detector.evaluation_generator.generate.return_value = [resp] + result = detector.verify( + output_text="out", + target_tool="bash", + vulnerability_info="vi", + attack_prompt="ap", + ) + assert ( + result is None + ), "a boolean confidence must be treated as unparseable, not as 1.0"