From 785c21753656b2c949f6be46a72027881fbcc773 Mon Sep 17 00:00:00 2001
From: Jeeho Lee <jeeholife2@gmail.com>
Date: Tue, 13 Oct 2026 09:19:49 +0000
Subject: [PATCH] fix(eval): avoid passing unknown alternate judge verdicts

Assisted-by: OpenAI Codex
---
 .../adk/evaluation/final_response_match_v2.py | 11 ++++---
 .../test_final_response_match_v2.py           | 31 +++++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/google/adk/evaluation/final_response_match_v2.py b/src/google/adk/evaluation/final_response_match_v2.py
index 5579f52fd15..dc07f25a57d 100644
--- a/src/google/adk/evaluation/final_response_match_v2.py
+++ b/src/google/adk/evaluation/final_response_match_v2.py
@@ -116,11 +116,12 @@ def _parse_critique(response: str) -> Label:
       label = Label.NOT_FOUND
   elif label_match_is_response_invalid:
     label = label_match_is_response_invalid.group(1).strip().rstrip(",}")
-    label = (
-        Label.INVALID
-        if label in [Label.TRUE.value, Label.INVALID.value]
-        else Label.VALID
-    )
+    if label in [Label.TRUE.value, Label.INVALID.value]:
+      label = Label.INVALID
+    elif label in [Label.FALSE.value, Label.VALID.value]:
+      label = Label.VALID
+    else:
+      label = Label.NOT_FOUND
   else:
     label = Label.NOT_FOUND
   return label
diff --git a/tests/unittests/evaluation/test_final_response_match_v2.py b/tests/unittests/evaluation/test_final_response_match_v2.py
index 2f6bc3b3385..bfcbc0b3c71 100644
--- a/tests/unittests/evaluation/test_final_response_match_v2.py
+++ b/tests/unittests/evaluation/test_final_response_match_v2.py
@@ -336,6 +336,37 @@ def test_convert_auto_rater_response_to_score_missing_key():
   assert auto_rater_score == AutoRaterScore()
 
 
+@pytest.mark.parametrize(
+    "label, expected_score",
+    [
+        ("unknown", None),
+        ("true", 0.0),
+        ("invalid", 0.0),
+        ("false", 1.0),
+        ("valid", 1.0),
+    ],
+)
+def test_alternate_invalidity_label_requires_a_known_verdict(
+    label, expected_score
+):
+  """Unrecognized alternate judge labels do not become passing scores."""
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  response = LlmResponse(
+      content=genai_types.Content(
+          role="model",
+          parts=[
+              genai_types.Part(
+                  text=f'{{"is_the_agent_response_invalid": "{label}"}}'
+              )
+          ],
+      )
+  )
+
+  result = evaluator.convert_auto_rater_response_to_score(response)
+
+  assert result.score == expected_score
+
+
 def test_aggregate_per_invocation_samples_none_evaluated():
   evaluator = _create_test_evaluator_gemini(threshold=0.5)