diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index d5dbbaa0..b8010dc3 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -12,6 +12,7 @@ from datetime import UTC, datetime from typing import Any, Literal, cast +from ..content import tool_result_string from ..judge_scoring import ( FORMATTING_INSTRUCTIONS, build_message_history, @@ -822,7 +823,7 @@ def _judge_variables( variables.update( { "input": row_result.get("input") or "", - "response_to_evaluate": output if output is not None else "", + "response_to_evaluate": tool_result_string(output), "message_history": build_message_history( user_input=row_result.get("input"), trajectory=trajectory, @@ -949,7 +950,7 @@ async def _run_ld_judge_for_result( try: result = await execution.handler( dict(resolved.config), - row.get("output"), + variables["response_to_evaluate"], {}, { **variables, diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index b782e497..8a784043 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -15,6 +15,7 @@ from math import isfinite from typing import Any +from .content import tool_result_string from .utils import parse_json_with_possible_fences FORMATTING_INSTRUCTIONS = "\n".join( @@ -45,10 +46,14 @@ def build_message_history( the AI Library's default templates read the JSON shape from ``{{message_history}}``, and one that stopped being told it would return prose and fail every result as invalid output. + + A non-string output is shown as JSON, by the rule tool results follow, so a + rubric reads the same text in every SDK. """ + output_text = tool_result_string(output) if output else None return "\n\n".join( str(part) - for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS) + for part in (user_input, trajectory, output_text, FORMATTING_INSTRUCTIONS) if part ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index b01eec75..f29d21fa 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2266,6 +2266,61 @@ async def handler( assert result.passed is True +@pytest.mark.asyncio +async def test_structured_output_reaches_the_judge_as_json( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page([{"rowIndex": 0, "input": "Question"}], total=1), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}}, + ), + ] + ) + accuracy_judge_variation(monkeypatch) + evals = init_evaluations( + project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport + ) + judged: dict[str, str] = {} + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + if "Judge" in prompt_text(config): + assert user_input == variables["response_to_evaluate"] + judged["response"] = variables["response_to_evaluate"] + judged["history"] = variables["message_history"] + return {"output": '{"score": 1, "reasoning": "ok"}'} + return {"output": {"answer": "original", "tags": ["a"], "final": True}} + + await evals.run( + key="support-qa", + dataset="golden", + handler=tagged(handler), + generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + rendered = '{"answer": "original", "tags": ["a"], "final": true}' + assert judged["response"] == rendered + assert rendered in judged["history"] + + @pytest.mark.asyncio async def test_duplicate_criteria_rejected_before_any_request() -> None: transport = SequencedTransport([]) diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py index a0472a02..3091ecce 100644 --- a/packages/client/tests/test_judge_scoring.py +++ b/packages/client/tests/test_judge_scoring.py @@ -5,6 +5,8 @@ import pytest from launchdarkly_ai_server.judge_scoring import ( + FORMATTING_INSTRUCTIONS, + build_message_history, parse_judge_response, typesafe_judge_entries, ) @@ -81,3 +83,19 @@ def test_typesafe_entries_keep_label_order() -> None: "reason": "", }, ] + + +def test_message_history_renders_a_structured_output_as_json() -> None: + history = build_message_history( + user_input="q", output={"answer": "oui", "ok": True, "note": None} + ) + assert history == "\n\n".join( + ["q", '{"answer": "oui", "ok": true, "note": null}', FORMATTING_INSTRUCTIONS] + ) + + +def test_message_history_keeps_non_ascii_and_skips_an_empty_output() -> None: + assert '{"city": "Zürich"}' in build_message_history(output={"city": "Zürich"}) + assert build_message_history(user_input="q", output={}) == "\n\n".join( + ["q", FORMATTING_INSTRUCTIONS] + )