From 62978193b05bd795a873e143262014dfb4f7b7a9 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 15 Oct 2026 18:52:30 -0400 Subject: [PATCH 1/3] feat(integrations): support multimodal decision spans Preserve box answers and image associations across TypeSafe, OpenAI Decisions, and LiteLLM Decisions normalization. Capture nested image assets without changing provider requests, and reuse a shared media walker with per-call image caching. Test Jev 1.14 boxes and images with TypeSafe SDK 0.7.5 while retaining 0.7.4 coverage. Add focused regressions for named answers, nullable confidence, and attachment conversion fallback. --- py/pyproject.toml | 5 +- py/src/braintrust/fixtures/decision-box.png | Bin 0 -> 1472 bytes .../integrations/litellm/tracing.py | 14 ++- .../braintrust/integrations/openai/tracing.py | 38 +++--- py/src/braintrust/integrations/question.py | 111 ++++++++++++++++-- .../braintrust/integrations/test_decisions.py | 78 +++++++++++- .../cassettes/0.7.4/test_auto_typesafe.yaml | 45 +++++++ .../test_setup_typesafe_system_one_async.yaml | 48 ++++++++ .../test_wrap_typesafe_system_one_sync.yaml | 45 +++++++ ...test_typesafe_boxes_and_images[async].yaml | 98 ++++++++++++++++ .../test_typesafe_boxes_and_images[sync].yaml | 98 ++++++++++++++++ .../test_typesafe_nested_image_assets.yaml | 94 +++++++++++++++ .../integrations/typesafe/test_typesafe.py | 111 +++++++++++++++++- .../integrations/typesafe/tracing.py | 97 +++++++++++++-- 14 files changed, 838 insertions(+), 44 deletions(-) create mode 100644 py/src/braintrust/fixtures/decision-box.png create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml create mode 100644 py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml diff --git a/py/pyproject.toml b/py/pyproject.toml index f2b01895e..d2393f71e 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -390,7 +390,7 @@ explicit = true # --------------------------------------------------------------------------- [tool.uv] -exclude-newer = "2026-10-13T16:16:21Z" +exclude-newer = "2026-10-15T00:00:00Z" conflicts = [ # Groups that pin or transitively require conflicting openai versions. [ @@ -577,7 +577,8 @@ latest = "mistralai==3.2.0" "1.12.4" = "mistralai==1.12.4" [tool.braintrust.matrix.typesafe-sdk] -latest = "typesafe-sdk==0.7.4" +latest = "typesafe-sdk==0.7.5" +"0.7.4" = "typesafe-sdk==0.7.4" [tool.braintrust.matrix.huggingface-hub] # Floor pinned to 0.32.0: the earliest release that exposes the diff --git a/py/src/braintrust/fixtures/decision-box.png b/py/src/braintrust/fixtures/decision-box.png new file mode 100644 index 0000000000000000000000000000000000000000..cb7d3837689c1288bf4445926c6dbce556b6bf08 GIT binary patch literal 1472 zcmeAS@N?(olHy`uVBq!ia0y~yV3Yu14kn-o$D6v_3=FL8o-U3d6?5KRZscTk;9=f) zzy4)e$1E#}%q9uF_soCZY~^oYmE&Zx5n`!O;rQSo@L;0C0cFPq_a=sZ7RDhgVerGF z|MRWhC%S%*_LM1Cc2Blgf7j#QBb9U6=99KtbdE}mhQ?^x7*yH8+=BDp`QcFXU@1at a7^82rE4x2?%?B)*89ZJ6T-G@yGywn#NuK=x literal 0 HcmV?d00001 diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index abde530fd..864b0c3bb 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -357,16 +357,19 @@ def _decisions_span_payload(args, kwargs): def _decisions_wrapper(wrapped, instance, args, kwargs): + payload = _decisions_span_payload(args, kwargs) with start_span( **merge_dicts( dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_payload(args, kwargs), + payload, ) ) as span: response = wrapped(*args, **kwargs) span.log( output=_decisions_output( - _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + _get_field(response, "answers"), + kwargs.get("questions", args[2] if len(args) > 2 else []), + payload.get("input"), ), metrics=_parse_metrics_from_usage(_get_field(response, "usage")), ) @@ -374,16 +377,19 @@ def _decisions_wrapper(wrapped, instance, args, kwargs): async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): + payload = _decisions_span_payload(args, kwargs) with start_span( **merge_dicts( dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_payload(args, kwargs), + payload, ) ) as span: response = await wrapped(*args, **kwargs) span.log( output=_decisions_output( - _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + _get_field(response, "answers"), + kwargs.get("questions", args[2] if len(args) > 2 else []), + payload.get("input"), ), metrics=_parse_metrics_from_usage(_get_field(response, "usage")), ) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 37ee6f5a5..a07887a9a 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2114,23 +2114,31 @@ def _decisions_input(state: Any, questions: Any) -> dict[str, Any]: return {"state": state, "questions": []} -def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: +def _decisions_output(answers: Any, questions: Any, span_input: Any = None) -> dict[str, Any]: try: if isinstance(answers, Mapping): - return _question_span_output(answers) + return _question_span_output(answers, span_input) questions = [_try_to_dict(question) for question in questions] + identifiers = _decision_question_ids(questions) + named_questions = { + question["name"]: (identifier, question) + for identifier, question in zip(identifiers, questions) + if question.get("name") == identifier + } normalized = [] - # OpenAI guarantees answer order, including refusals and unnamed questions. - # Use the request's IDs so all answers remain linked to their questions. - for identifier, question, answer in zip(_decision_question_ids(questions), questions, answers or []): + # Named answers identify their question even if reordered. Unnamed + # answers use the ordering guaranteed by OpenAI's Decisions API. + for identifier, question, answer in zip(identifiers, questions, answers or []): answer = _try_to_dict(answer) if not isinstance(answer, dict): continue + if answer.get("name") in named_questions: + identifier, question = named_questions[answer["name"]] kind = answer.get("type") payload = {"id": identifier, "type": "boolean" if kind == "predicate" else kind} - for key in ("probability", "score", "confidence"): + for key in ("noul", "probability", "score", "confidence", "box", "image_path"): if key in answer: - payload[key] = answer[key] + payload[key] = _try_to_dict(answer[key]) if key == "box" else answer[key] if kind == "choice": keys = _decision_choice_keys(question) if "choice" in answer: @@ -2147,7 +2155,7 @@ def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: } payload["legend"] = {str(item["value"]): item["label"] for item in answer["probabilities"]} normalized.append(payload) - return _question_span_output(normalized) + return _question_span_output(normalized, span_input) except Exception: _log.debug("Failed to normalize Decisions answers", exc_info=True) return {"answers": []} @@ -2184,11 +2192,11 @@ def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: ) -def _log_decisions_response(span: Span, response: Any, questions: Any): +def _log_decisions_response(span: Span, response: Any, questions: Any, span_input: Any): response = _try_to_dict(response) if isinstance(response, dict): span.log( - output=_decisions_output(response.get("answers"), questions or []), + output=_decisions_output(response.get("answers"), questions or [], span_input), metrics=_parse_metrics_from_usage(response.get("usage")), ) @@ -2197,25 +2205,27 @@ def _decisions_create_wrapper(wrapped, instance, args, kwargs): if _is_async_callable(wrapped): async def call(): + params = _decisions_span_params(kwargs) with start_span( **merge_dicts( dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_params(kwargs), + params, ) ) as span: response = await wrapped(*args, **kwargs) - _log_decisions_response(span, response, kwargs.get("questions")) + _log_decisions_response(span, response, kwargs.get("questions"), params.get("input")) return response return call() + params = _decisions_span_params(kwargs) with start_span( **merge_dicts( dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_params(kwargs), + params, ) ) as span: response = wrapped(*args, **kwargs) - _log_decisions_response(span, response, kwargs.get("questions")) + _log_decisions_response(span, response, kwargs.get("questions"), params.get("input")) return response diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py index 4376f41d7..dac8aa432 100644 --- a/py/src/braintrust/integrations/question.py +++ b/py/src/braintrust/integrations/question.py @@ -4,14 +4,74 @@ helpers. Named maps become arrays linked by ID; arrays already carry their IDs. """ -from collections.abc import Mapping +import logging +from collections.abc import Callable, Mapping from typing import Any -from .utils import _try_to_dict +from braintrust.logger import Attachment + +from .utils import _materialize_attachment, _try_to_dict _QUESTION_FIELDS = ("type", "instructions", "criteria") -_ANSWER_FIELDS = ("type", "noul", "probability", "choice", "score", "confidence", "legend", "probabilities") +_ANSWER_FIELDS = ( + "type", + "noul", + "probability", + "choice", + "score", + "confidence", + "legend", + "probabilities", + "box", + "image_path", +) +_ATTACHMENT_TYPES = ("braintrust_attachment", "external_attachment", "inline_attachment") +_log = logging.getLogger(__name__) + + +def _decision_media(value: Any, media_transform: Callable[[Any], Any] | None = None) -> Any: + """Replace inline media in the logged payload without changing request values.""" + if isinstance(value, Mapping): + if value.get("type") in _ATTACHMENT_TYPES: + return value + return {key: _decision_media(item, media_transform) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_decision_media(item, media_transform) for item in value] + if media_transform is not None: + value = media_transform(value) + if isinstance(value, str) and value.startswith("data:"): + resolved = _materialize_attachment(value) + if resolved is not None: + return resolved.attachment + return value + + +def _decision_image_paths(value: Any, path: str = "") -> list[str]: + """Find image occurrences, retaining repeated locations and JSON Pointer escaping.""" + if isinstance(value, Attachment): + return [path] if value.reference.get("content_type", "").startswith("image/") else [] + if isinstance(value, Mapping): + if value.get("type") in _ATTACHMENT_TYPES: + return [path] if str(value.get("content_type", "")).startswith("image/") else [] + if value.get("type") == "input_image" and isinstance(value.get("image_url"), str): + return [path + "/image_url"] + if value.get("type") == "image_url" and isinstance(value.get("image_url"), Mapping): + return [path + "/image_url/url"] + return [ + image_path + for key, item in value.items() + for image_path in _decision_image_paths(item, path + "/" + str(key).replace("~", "~0").replace("/", "~1")) + ] + if isinstance(value, (list, tuple)): + return [ + image_path + for index, item in enumerate(value) + for image_path in _decision_image_paths(item, f"{path}/{index}") + ] + if isinstance(value, str) and value.startswith("data:image/"): + return [path] + return [] def _question_items(values: Any, fields: tuple[str, ...]) -> Any: @@ -21,13 +81,48 @@ def _question_items(values: Any, fields: tuple[str, ...]) -> Any: for identifier, value in values.items(): value = _try_to_dict(value) if isinstance(value, dict): - items.append({**{key: value[key] for key in fields if key in value}, "id": identifier}) + payload = {key: value[key] for key in fields if key in value} + if "box" in payload: + payload["box"] = _try_to_dict(payload["box"]) + items.append({**payload, "id": identifier}) return items -def _question_span_input(state: Any, questions: Any) -> dict[str, Any]: - return {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} +def _question_span_input( + state: Any, questions: Any, *, media_transform: Callable[[Any], Any] | None = None +) -> dict[str, Any]: + payload = {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} + try: + return { + "state": _decision_media(state, media_transform), + "questions": [ + { + key: _decision_media(value, media_transform) if key in ("instructions", "criteria") else value + for key, value in question.items() + } + for question in payload["questions"] + ] + if isinstance(payload["questions"], list) + else payload["questions"], + } + except Exception: + # Abort the whole rewrite if a conversion fails; retain all original media. + _log.debug("Failed to capture decision media", exc_info=True) + return payload -def _question_span_output(answers: Any) -> dict[str, Any]: - return {"answers": _question_items(answers, _ANSWER_FIELDS)} +def _question_span_output(answers: Any, span_input: Any = None) -> dict[str, Any]: + items = _question_items(answers, _ANSWER_FIELDS) + if not isinstance(items, list) or not any( + isinstance(answer, dict) and answer.get("type") == "box" and "image_path" not in answer for answer in items + ): + return {"answers": items} + image_paths = _decision_image_paths(span_input) + if len(image_paths) == 1: + items = [ + {**answer, "image_path": image_paths[0]} + if isinstance(answer, dict) and answer.get("type") == "box" and "image_path" not in answer + else answer + for answer in items + ] + return {"answers": items} diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py index 4742cf95a..f2c5d7233 100644 --- a/py/src/braintrust/integrations/test_decisions.py +++ b/py/src/braintrust/integrations/test_decisions.py @@ -6,6 +6,7 @@ from braintrust.integrations.litellm.tracing import _parse_metrics_from_usage as parse_litellm_usage from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage +from braintrust.integrations.question import _question_span_input from braintrust.test_helpers import assert_dict_matches @@ -195,13 +196,82 @@ def test_decisions_refusals_and_allowlists(): def test_decisions_system_one_maps_match_typesafe_structure(): - questions = {"positive": {"type": "noul", "instructions": "Is this positive?", "secret": "omit"}} - answers = {"positive": {"type": "noul", "noul": 0.9, "secret": "omit"}} + questions = {"present": {"type": "noul", "instructions": "Is the target present?", "secret": "omit"}} + answers = {"present": {"type": "noul", "noul": 0.01, "secret": "omit"}} + box = {"top": 0, "bottom": 1, "left": 0.123456789, "right": 0.9} + for identifier, optional_fields in ( + ("zero", {"confidence": 0}), + ("null", {"confidence": None}), + ("absent", {}), + ): + questions[identifier] = {"type": "box", "instructions": {"target": "label"}} + answers[identifier] = {"type": "box", "box": box, "image_path": "/state/photos/1", **optional_fields} + # Map keys link answers to questions, independently of return order. + answers = dict(reversed(answers.items())) assert _decisions_input("state", questions) == { "state": "state", - "questions": [{"id": "positive", "type": "noul", "instructions": "Is this positive?"}], + "questions": [ + {"id": "present", "type": "noul", "instructions": "Is the target present?"}, + *({"id": identifier, **questions[identifier]} for identifier in ("zero", "null", "absent")), + ], + } + assert _decisions_output(answers, questions) == { + "answers": [ + *({"id": identifier, **answers[identifier]} for identifier in ("absent", "null", "zero")), + {"id": "present", "type": "noul", "noul": 0.01}, + ] + } + + +def test_decisions_named_answers_match_questions_when_reordered(): + questions = [ + {"type": "box", "name": "bounds", "instructions": "The target"}, + {"type": "predicate", "name": "present", "instructions": "Is it present?"}, + ] + box = {"top": -0.1, "bottom": 0, "left": 0.8, "right": 0.2} + assert _decisions_output( + [{"type": "predicate", "name": "present", "probability": 0.01}, {"type": "box", "name": "bounds", "box": box}], + questions, + ) == { + "answers": [ + {"id": "present", "type": "boolean", "probability": 0.01}, + {"id": "bounds", "type": "box", "box": box}, + ] + } + + +def test_decisions_nested_media_and_existing_references(): + image = decisions_image_input()[0]["content"][1]["image_url"] + reference = {"type": "external_attachment", "url": "https://example.com/image.png", "content_type": "image/png"} + questions = {"bounds": {"type": "box", "instructions": {"image": image, "text": "target"}}} + span_input = _decisions_input({"photos": [reference, image], "remote": "https://example.com/a.png"}, questions) + assert span_input["state"]["photos"][0] == reference + assert isinstance(span_input["state"]["photos"][1], Attachment) + assert isinstance(span_input["questions"][0]["instructions"]["image"], Attachment) + assert span_input["state"]["remote"] == "https://example.com/a.png" + assert questions["bounds"]["instructions"]["image"] == image + + +def test_decisions_media_failure_keeps_entire_payload(monkeypatch): + image = decisions_image_input()[0]["content"][1]["image_url"] + from braintrust.integrations import question + + materialize = question._materialize_attachment + calls = 0 + + def fail_second(value): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("conversion failed") + return materialize(value) + + monkeypatch.setattr(question, "_materialize_attachment", fail_second) + questions = {"bounds": {"type": "box", "instructions": {"image": image}}} + assert _question_span_input({"image": image}, questions) == { + "state": {"image": image}, + "questions": [{"id": "bounds", **questions["bounds"]}], } - assert _decisions_output(answers, questions) == {"answers": [{"id": "positive", "type": "noul", "noul": 0.9}]} @pytest.mark.parametrize("parse_usage", (parse_openai_usage, parse_litellm_usage), ids=("openai", "litellm")) diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml new file mode 100644 index 000000000..0c0bd4d51 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml @@ -0,0 +1,45 @@ +interactions: +- request: + body: '{"state":"The package arrived intact and on time.","model":"jev-latest","questions":{"positive":{"type":"noul","instructions":"Is + this feedback positive?"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '157' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"positive":{"type":"noul","noul":0.97}},"usage":{"input_tokens":279,"output_tokens":20}}' + headers: + content-length: + - '121' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:11 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '77' + x-typesafe-request-id: + - req_01a0c993f3ff71eeb69776ee462670ac + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml new file mode 100644 index 000000000..65cedc0d6 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"state":{"message":"I was charged twice. Please refund the duplicate charge + today."},"model":"jev-latest","questions":{"category":{"type":"choice","instructions":"Which + team should handle this?","criteria":{"billing":null,"technical":null,"other":null}},"urgency":{"type":"score","instructions":"How + urgent is this request?","criteria":["routine","soon","urgent"]},"duplicate_charge":{"type":"noul","instructions":{"question":"Does + the customer report a duplicate charge?"}}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '477' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"category":{"type":"choice","choice":"billing","confidence":1.0,"probabilities":{"technical":0.0,"billing":1.0,"other":0.0}},"urgency":{"type":"score","score":1.78,"confidence":0.67,"legend":{"0":"routine","1":"soon","2":"urgent"},"probabilities":{"0":0.0,"1":0.22,"2":0.78}},"duplicate_charge":{"type":"noul","noul":0.99}},"usage":{"input_tokens":370,"output_tokens":70}}' + headers: + content-length: + - '405' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:10 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '132' + x-typesafe-request-id: + - req_01a0c993f0b57238bb5b0d6f611be2bf + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml new file mode 100644 index 000000000..d75e6c0aa --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml @@ -0,0 +1,45 @@ +interactions: +- request: + body: '{"state":"The package arrived intact and on time.","model":"jev-latest","questions":{"positive":{"type":"noul","instructions":"Is + this feedback positive?","not_allowlisted":"secret"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '184' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"positive":{"type":"noul","noul":0.97}},"usage":{"input_tokens":279,"output_tokens":20}}' + headers: + content-length: + - '121' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:10 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '101' + x-typesafe-request-id: + - req_01a0c993ef287b88a5f5929d703f4eb0 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml new file mode 100644 index 000000000..24b4cf860 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: !!binary | + LS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0NjhjOGY3MGNlMQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zL34iOlsiPGltYWdlOjIzZDNlMTdkOGUzMTRiYzQ4MmZiZjAw + YzcxNTYzZTYwPiJdLCJkZXNjcmlwdGlvbiI6IkEgdGVzdCBpbWFnZSJ9LCJtb2RlbCI6Impldi0x + LjE0LjAiLCJxdWVzdGlvbnMiOnsiYm91bmRzIjp7InR5cGUiOiJib3giLCJpbnN0cnVjdGlvbnMi + OiJUaGUgbWFpbiBvYmplY3QgaW4gYHBob3Rvcy9+WzBdYCJ9LCJwcmVzZW50Ijp7InR5cGUiOiJu + b3VsIiwiaW5zdHJ1Y3Rpb25zIjoiSXMgdGhlcmUgYSBtYWluIG9iamVjdCBpbiB0aGUgaW1hZ2U/ + In0sImNhdGVnb3J5Ijp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJXaGF0IGlzIGlu + IHRoZSBpbWFnZT8iLCJjcml0ZXJpYSI6eyJvYmplY3QiOm51bGwsInRleHQiOm51bGx9fSwic2l6 + ZSI6eyJ0eXBlIjoic2NvcmUiLCJpbnN0cnVjdGlvbnMiOiJIb3cgbXVjaCBvZiB0aGUgaW1hZ2Ug + aXMgb2NjdXBpZWQ/IiwiY3JpdGVyaWEiOlsibGl0dGxlIiwibW9zdCJdfX0sImFzc2V0cyI6eyI8 + aW1hZ2U6MjNkM2UxN2Q4ZTMxNGJjNDgyZmJmMDBjNzE1NjNlNjA+Ijp7InR5cGUiOiJpbWFnZSIs + InBhcnQiOiJhc3NldC0wIn19fQ0KLS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0NjhjOGY3MGNlMQ0K + Q29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9 + ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAAB + GAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0Njhj + OGY3MGNlMS0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '2348' + content-type: + - multipart/form-data; boundary=6b59ed6043f44d12ba8a6468c8f70ce1 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"bounds":{"type":"box","box":{"top":0.04,"bottom":0.97,"left":0.13,"right":0.75},"confidence":0.24},"present":{"type":"noul","noul":0.47},"category":{"type":"choice","choice":"object","confidence":0.9,"probabilities":{"object":0.95,"text":0.05}},"size":{"type":"score","score":0.54,"confidence":0.08,"legend":{"0":"little","1":"most"},"probabilities":{"0":0.46,"1":0.54}}},"usage":{"input_tokens":610,"output_tokens":77}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b0b61409290eff-YYZ + connection: + - keep-alive + content-length: + - '454' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 17:58:19 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '46' + x-typesafe-request-id: + - req_01a140b708a5708fae9b3420db044484 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml new file mode 100644 index 000000000..5b6f600eb --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: !!binary | + LS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1YjQxNjRjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zL34iOlsiPGltYWdlOjBkNTU5OWNkMGFkMjQ1ZWI5MmQ0NDU5 + N2ZjMTkxNmM1PiJdLCJkZXNjcmlwdGlvbiI6IkEgdGVzdCBpbWFnZSJ9LCJtb2RlbCI6Impldi0x + LjE0LjAiLCJxdWVzdGlvbnMiOnsiYm91bmRzIjp7InR5cGUiOiJib3giLCJpbnN0cnVjdGlvbnMi + OiJUaGUgbWFpbiBvYmplY3QgaW4gYHBob3Rvcy9+WzBdYCJ9LCJwcmVzZW50Ijp7InR5cGUiOiJu + b3VsIiwiaW5zdHJ1Y3Rpb25zIjoiSXMgdGhlcmUgYSBtYWluIG9iamVjdCBpbiB0aGUgaW1hZ2U/ + In0sImNhdGVnb3J5Ijp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJXaGF0IGlzIGlu + IHRoZSBpbWFnZT8iLCJjcml0ZXJpYSI6eyJvYmplY3QiOm51bGwsInRleHQiOm51bGx9fSwic2l6 + ZSI6eyJ0eXBlIjoic2NvcmUiLCJpbnN0cnVjdGlvbnMiOiJIb3cgbXVjaCBvZiB0aGUgaW1hZ2Ug + aXMgb2NjdXBpZWQ/IiwiY3JpdGVyaWEiOlsibGl0dGxlIiwibW9zdCJdfX0sImFzc2V0cyI6eyI8 + aW1hZ2U6MGQ1NTk5Y2QwYWQyNDVlYjkyZDQ0NTk3ZmMxOTE2YzU+Ijp7InR5cGUiOiJpbWFnZSIs + InBhcnQiOiJhc3NldC0wIn19fQ0KLS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1YjQxNjRjNQ0K + Q29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9 + ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAAB + GAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1 + YjQxNjRjNS0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '2348' + content-type: + - multipart/form-data; boundary=b4f783e2c95a44c98937a6895b4164c5 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"bounds":{"type":"box","box":{"top":0.05,"bottom":0.97,"left":0.14,"right":0.69},"confidence":0.25},"present":{"type":"noul","noul":0.51},"category":{"type":"choice","choice":"object","confidence":0.88,"probabilities":{"object":0.94,"text":0.06}},"size":{"type":"score","score":0.59,"confidence":0.18,"legend":{"0":"little","1":"most"},"probabilities":{"0":0.41,"1":0.59}}},"usage":{"input_tokens":608,"output_tokens":77}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b0b612be89aac7-YYZ + connection: + - keep-alive + content-length: + - '455' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 17:58:19 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '56' + x-typesafe-request-id: + - req_01a140b707ce747a90cf04ef22708fcb + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml new file mode 100644 index 000000000..8f938f4a9 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml @@ -0,0 +1,94 @@ +interactions: +- request: + body: !!binary | + LS02MjEwOGExNTFmZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zIjpbInBob3RvLXJlZmVyZW5jZSIsInBob3RvLXJlZmVyZW5j + ZSJdfSwibW9kZWwiOiJqZXYtMS4xNC4wIiwicXVlc3Rpb25zIjp7InBob3RvLXJlZmVyZW5jZSI6 + eyJ0eXBlIjoiYm94IiwiaW5zdHJ1Y3Rpb25zIjp7ImltYWdlIjoicGhvdG8tcmVmZXJlbmNlIiwi + dGFyZ2V0IjoiVGhlIGJsYWNrIHNxdWFyZSJ9fSwibWF0Y2giOnsidHlwZSI6ImNob2ljZSIsImlu + c3RydWN0aW9ucyI6IkNob29zZSB0aGUgbWF0Y2hpbmcgaW1hZ2UiLCJjcml0ZXJpYSI6eyJzcXVh + cmUiOiJwaG90by1yZWZlcmVuY2UiLCJub25lIjpudWxsfX19LCJhc3NldHMiOnsicGhvdG8tcmVm + ZXJlbmNlIjp7InR5cGUiOiJpbWFnZSIsInBhcnQiOiJhc3NldC0wIn19fQ0KLS02MjEwOGExNTFm + ZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBu + YW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3Bu + Zw0KDQqJUE5HDQoaCgAAAA1JSERSAAABGAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx + 33/pdoiaOhhpghgu7wP87LUPgAUeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ + 4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GA + R4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEe + CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgk + CPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwS + BHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ + 4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GA + R4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEe + CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS02 + MjEwOGExNTFmZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNS0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '2144' + content-type: + - multipart/form-data; boundary=62108a151fee4feab3ac1bac9dec57c5 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"photo-reference":{"type":"box","box":{"top":0.05,"bottom":0.98,"left":0.13,"right":0.75},"confidence":0.19},"match":{"type":"choice","choice":"square","confidence":0.86,"probabilities":{"square":0.93,"none":0.07}}},"usage":{"input_tokens":1147,"output_tokens":48}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b0baf1182baeb6-YYZ + connection: + - keep-alive + content-length: + - '298' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 18:01:38 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '66' + x-typesafe-request-id: + - req_01a140ba12c97099a03a30bf5a415906 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index b12816527..ef3df7aa9 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -1,7 +1,9 @@ import os +from pathlib import Path import pytest -from braintrust import logger +import typesafe_sdk +from braintrust import Attachment, logger from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.integrations.typesafe import setup_typesafe, wrap_typesafe from braintrust.span_types import SpanTypeAttribute @@ -126,3 +128,110 @@ async def test_setup_typesafe_system_one_async(memory_logger): def test_auto_instrument_typesafe_subprocess(): verify_autoinstrument_script("test_auto_typesafe.py") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.vcr +async def test_typesafe_boxes_and_images(memory_logger, is_async): + if not hasattr(typesafe_sdk, "Box"): + pytest.skip("Box and Image require TypeSafe SDK >=0.7.5") + + class CustomImage(typesafe_sdk.Image): + pass + + image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" + image = CustomImage(image_file) + state = {"photos/~": [image], "description": "A test image"} + questions = { + "bounds": typesafe_sdk.Box(instructions="The main object in `photos/~[0]`"), + "present": Noul(instructions="Is there a main object in the image?"), + "category": Choice(instructions="What is in the image?", criteria={"object": None, "text": None}), + "size": Score(instructions="How much of the image is occupied?", criteria=["little", "most"]), + } + assert setup_typesafe() + client_type = AsyncTypeSafeClient if is_async else TypeSafeClient + client = client_type(api_key=os.environ["TYPESAFE_API_KEY"], compression=False) + kwargs = dict( + model="jev-1.14.0", + state=state, + questions=questions, + extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, + ) + if is_async: + async with client: + response = await client.system_one(**kwargs) + else: + with client: + response = client.system_one(**kwargs) + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + _assert_span(span, question_ids=list(questions)) + assert span["metadata"]["model"] == "jev-1.14.0" + attachment = span["input"]["state"]["photos/~"][0] + assert isinstance(attachment, Attachment) + assert attachment.data == image_file.read_bytes() + assert attachment.reference["content_type"] == "image/png" + assert state["photos/~"][0] is image + assert span["input"]["questions"][0] == { + "id": "bounds", + "type": "box", + "instructions": questions["bounds"].instructions, + } + answers = {answer["id"]: answer for answer in span["output"]["answers"]} + assert answers["bounds"] == { + "id": "bounds", + "type": "box", + "box": response.answers["bounds"].box.model_dump(), + "confidence": response.answers["bounds"].confidence, + "image_path": "/state/photos~1~0/0", + } + assert answers["present"]["noul"] == response.answers["present"].noul + assert answers["size"]["score"] == response.answers["size"].score + assert "scores" not in span + assert "classifications" not in span + + +@pytest.mark.vcr +def test_typesafe_nested_image_assets(memory_logger): + if not hasattr(typesafe_sdk, "Box"): + pytest.skip("Box and Image require TypeSafe SDK >=0.7.5") + + image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" + image = typesafe_sdk.Image(image_file) + state = {"photos": ["photo-reference", "photo-reference"]} + questions = { + # IDs are text, even when they happen to match an asset reference. + "photo-reference": typesafe_sdk.Box(instructions={"image": image, "target": "The black square"}), + "match": Choice(instructions="Choose the matching image", criteria={"square": image, "none": None}), + } + assert setup_typesafe() + with TypeSafeClient(api_key=os.environ["TYPESAFE_API_KEY"], compression=False) as client: + response = client.system_one( + model="jev-1.14.0", + state=state, + questions=questions, + assets={"photo-reference": image}, + extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, + ) + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + _assert_span(span, question_ids=list(questions)) + captured = span["input"] + images = [ + *captured["state"]["photos"], + captured["questions"][0]["instructions"]["image"], + captured["questions"][1]["criteria"]["square"], + ] + assert all(isinstance(item, Attachment) and item.data == image_file.read_bytes() for item in images) + answers = {answer["id"]: answer for answer in span["output"]["answers"]} + assert answers["photo-reference"]["box"] == response.answers["photo-reference"].box.model_dump() + # Multiple image occurrences have no structured selector in TypeSafe; + # instruction text must not be used to guess their association. + assert "image_path" not in answers["photo-reference"] + assert state == {"photos": ["photo-reference", "photo-reference"]} + assert questions["photo-reference"].instructions == {"image": image, "target": "The black square"} diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index f628bda25..59559e953 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -1,5 +1,8 @@ """TypeSafe-specific tracing helpers.""" +import os +import re +import sys from collections.abc import Mapping from typing import Any @@ -9,7 +12,7 @@ _question_span_input, _question_span_output, ) -from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span +from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span, _materialize_attachment from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute @@ -71,10 +74,72 @@ def _request_parts(instance: Any, args: Any, kwargs: dict[str, Any]) -> tuple[An return state, questions, model -def _span_input(state: Any, questions: Any) -> dict[str, Any]: +def _image_payload(value: Any) -> Any: + # Only interpret TypeSafe's Image objects as files/buffers. Arbitrary JSON + # strings in the request must never be treated as local paths. + image_type = getattr(sys.modules.get("typesafe_sdk"), "Image", ()) + if isinstance(value, image_type): + data = value.data + if isinstance(data, os.PathLike): + with open(data, "rb") as image_file: + data = image_file.read() + else: + data = bytes(data) + mime_type = ( + "image/png" + if data.startswith(b"\x89PNG\r\n\x1a\n") + else "image/jpeg" + if data.startswith(b"\xff\xd8\xff") + else None + ) + if mime_type is not None: + resolved = _materialize_attachment(data, mime_type=mime_type) + if resolved is not None: + return resolved.attachment + if isinstance(value, Mapping) and "data_base64" in value: + image_format = value.get("format") + if image_format in ("png", "jpg", "jpeg") and isinstance(value["data_base64"], str): + resolved = _materialize_attachment( + value["data_base64"], mime_type="image/png" if image_format == "png" else "image/jpeg" + ) + if resolved is not None: + return resolved.attachment + return value + + +def _span_input(state: Any, questions: Any, assets: Mapping[str, Any]) -> dict[str, Any]: if isinstance(questions, Mapping): questions = {identifier: _question_payload(value) for identifier, value in questions.items()} - return _question_span_input(state, questions) + references = [re.escape(key) for key in assets if isinstance(key, str) and key] + pattern = re.compile("(" + "|".join(references) + ")") if references else None + captured_images: dict[int, Any] = {} + + def capture(value): + key = id(value) + if key in captured_images: + return captured_images[key] + captured = _image_payload(value) + if captured is not value: + captured_images[key] = captured + return captured + + def resolve(reference): + asset = assets[reference] + captured = capture(asset) + return reference if captured is asset else captured + + def media_transform(value): + if isinstance(value, str) and assets: + if value in assets: + return resolve(value) + # TypeSafe expands references inside text as well as whole values. + if pattern is not None: + parts = pattern.split(value) + if len(parts) > 1: + return [resolve(part) if part in assets else part for part in parts if part] + return capture(value) + + return _question_span_input(state, questions, media_transform=media_transform) def _token_metrics(response: Any) -> dict[str, int]: @@ -91,14 +156,14 @@ def _token_metrics(response: Any) -> dict[str, int]: return metrics -def _log_response(span: Any, response: Any) -> None: +def _log_response(span: Any, response: Any, span_input: Any) -> None: answers = getattr(response, "answers", None) if isinstance(answers, Mapping): answers = {identifier: _answer_payload(value) for identifier, value in answers.items()} response_model = getattr(response, "model", None) _log_and_end_span( span, - output=_question_span_output(answers), + output=_question_span_output(answers, span_input), metrics=_token_metrics(response), metadata={"model": response_model} if isinstance(response_model, str) else None, ) @@ -109,16 +174,26 @@ def _start_system_one_span(instance: Any, args: Any, kwargs: dict[str, Any]): metadata = {"provider": "typesafe"} if isinstance(model, str): metadata["model"] = model + assets = dict(kwargs.get("assets") or {}) + attached = getattr(instance, "_attached", None) + if attached is not None: + assets.update(attached.snapshot()) + extra_body = kwargs.get("extra_body") + if isinstance(extra_body, Mapping) and isinstance(extra_body.get("assets"), Mapping): + assets = dict(extra_body["assets"]) + # Images may also be referenced in text by their SDK-generated reference. + assets.update({str(image): image for image in list(assets.values()) if not isinstance(image, Mapping)}) + span_input = _span_input(state, questions, assets) return start_span( name="typesafe.systemOne", type=SpanTypeAttribute.DECISION, - input=_span_input(state, questions), + input=span_input, metadata=metadata, - ) + ), span_input def _system_one_wrapper(wrapped, instance, args, kwargs): - span = _start_system_one_span(instance, args, kwargs) + span, span_input = _start_system_one_span(instance, args, kwargs) try: response = wrapped(*args, **kwargs) @@ -126,12 +201,12 @@ def _system_one_wrapper(wrapped, instance, args, kwargs): _log_error_and_end_span(span, error) raise - _log_response(span, response) + _log_response(span, response, span_input) return response async def _async_system_one_wrapper(wrapped, instance, args, kwargs): - span = _start_system_one_span(instance, args, kwargs) + span, span_input = _start_system_one_span(instance, args, kwargs) try: response = await wrapped(*args, **kwargs) @@ -139,5 +214,5 @@ async def _async_system_one_wrapper(wrapped, instance, args, kwargs): _log_error_and_end_span(span, error) raise - _log_response(span, response) + _log_response(span, response, span_input) return response From cf3a10292b426267eea4d7fe0d4dd1567e166346 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 15 Oct 2026 19:09:57 -0400 Subject: [PATCH 2/3] fix(typesafe): normalize custom response answer containers --- .../integrations/typesafe/test_typesafe.py | 25 ++++++++++++++++--- .../integrations/typesafe/tracing.py | 9 +++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index ef3df7aa9..0c3393b93 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -8,6 +8,7 @@ from braintrust.integrations.typesafe import setup_typesafe, wrap_typesafe from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger +from pydantic import BaseModel from typesafe_sdk import AsyncTypeSafeClient, Choice, Noul, Score, TypeSafeClient, TypeSafeError @@ -140,6 +141,17 @@ async def test_typesafe_boxes_and_images(memory_logger, is_async): class CustomImage(typesafe_sdk.Image): pass + class CustomAnswers(BaseModel): + bounds: typesafe_sdk.BoxAnswer + present: typesafe_sdk.NoulAnswer + category: typesafe_sdk.ChoiceAnswer + size: typesafe_sdk.ScoreAnswer + + class CustomResponse(BaseModel): + model: str + usage: typesafe_sdk.Usage + answers: CustomAnswers + image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" image = CustomImage(image_file) state = {"photos/~": [image], "description": "A test image"} @@ -159,11 +171,16 @@ class CustomImage(typesafe_sdk.Image): extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, ) if is_async: + kwargs["response_model"] = CustomResponse async with client: response = await client.system_one(**kwargs) + assert isinstance(response, CustomResponse) + assert isinstance(response.answers, CustomAnswers) + response_answers = {identifier: getattr(response.answers, identifier) for identifier in questions} else: with client: response = client.system_one(**kwargs) + response_answers = response.answers spans = memory_logger.pop() assert len(spans) == 1 @@ -184,12 +201,12 @@ class CustomImage(typesafe_sdk.Image): assert answers["bounds"] == { "id": "bounds", "type": "box", - "box": response.answers["bounds"].box.model_dump(), - "confidence": response.answers["bounds"].confidence, + "box": response_answers["bounds"].box.model_dump(), + "confidence": response_answers["bounds"].confidence, "image_path": "/state/photos~1~0/0", } - assert answers["present"]["noul"] == response.answers["present"].noul - assert answers["size"]["score"] == response.answers["size"].score + assert answers["present"]["noul"] == response_answers["present"].noul + assert answers["size"]["score"] == response_answers["size"].score assert "scores" not in span assert "classifications" not in span diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index 59559e953..eb3176854 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -12,7 +12,12 @@ _question_span_input, _question_span_output, ) -from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span, _materialize_attachment +from braintrust.integrations.utils import ( + _log_and_end_span, + _log_error_and_end_span, + _materialize_attachment, + _try_to_dict, +) from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute @@ -157,7 +162,7 @@ def _token_metrics(response: Any) -> dict[str, int]: def _log_response(span: Any, response: Any, span_input: Any) -> None: - answers = getattr(response, "answers", None) + answers = _try_to_dict(getattr(response, "answers", None)) if isinstance(answers, Mapping): answers = {identifier: _answer_payload(value) for identifier, value in answers.items()} response_model = getattr(response, "model", None) From 7e08e8041ad74e5aae3b9db23605a9fff6fd1068 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 15 Oct 2026 19:48:06 -0400 Subject: [PATCH 3/3] fix(integrations): preserve decision answer and image associations Reserve named answer matches before pairing the remaining questions and answers, so reordering across unnamed questions cannot duplicate an answer ID. Prefer longer TypeSafe asset references in embedded text, and let media transformers capture serialized image mappings before recursive traversal. Cover the regressions with focused normalization tests and the existing TypeSafe asset-reference recording. --- .../braintrust/integrations/openai/tracing.py | 24 +++++-- py/src/braintrust/integrations/question.py | 10 +-- .../braintrust/integrations/test_decisions.py | 25 +++++-- .../test_typesafe_nested_image_assets.yaml | 66 +++++++++++++------ .../integrations/typesafe/test_typesafe.py | 19 +++--- .../integrations/typesafe/tracing.py | 1 + 6 files changed, 104 insertions(+), 41 deletions(-) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index a07887a9a..ee322ac9d 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2125,15 +2125,27 @@ def _decisions_output(answers: Any, questions: Any, span_input: Any = None) -> d for identifier, question in zip(identifiers, questions) if question.get("name") == identifier } + answers = [_try_to_dict(answer) for answer in answers or []] + named_answer_ids = { + answer["name"] for answer in answers if isinstance(answer, dict) and answer.get("name") in named_questions + } + remaining_questions = iter( + (identifier, question) + for identifier, question in zip(identifiers, questions) + if identifier not in named_answer_ids + ) normalized = [] - # Named answers identify their question even if reordered. Unnamed - # answers use the ordering guaranteed by OpenAI's Decisions API. - for identifier, question, answer in zip(identifiers, questions, answers or []): - answer = _try_to_dict(answer) + # Reserve named matches before pairing the remaining answers and + # questions in order, so reordering cannot duplicate a question ID. + for answer in answers: if not isinstance(answer, dict): continue - if answer.get("name") in named_questions: - identifier, question = named_questions[answer["name"]] + match = named_questions.pop(answer.get("name"), None) + if match is None: + match = next(remaining_questions, None) + if match is None: + continue + identifier, question = match kind = answer.get("type") payload = {"id": identifier, "type": "boolean" if kind == "predicate" else kind} for key in ("noul", "probability", "score", "confidence", "box", "image_path"): diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py index dac8aa432..19c50b1d3 100644 --- a/py/src/braintrust/integrations/question.py +++ b/py/src/braintrust/integrations/question.py @@ -32,14 +32,16 @@ def _decision_media(value: Any, media_transform: Callable[[Any], Any] | None = None) -> Any: """Replace inline media in the logged payload without changing request values.""" + if isinstance(value, Mapping) and value.get("type") in _ATTACHMENT_TYPES: + return value + if media_transform is not None: + transformed = media_transform(value) + if transformed is not value: + return transformed if isinstance(value, Mapping): - if value.get("type") in _ATTACHMENT_TYPES: - return value return {key: _decision_media(item, media_transform) for key, item in value.items()} if isinstance(value, (list, tuple)): return [_decision_media(item, media_transform) for item in value] - if media_transform is not None: - value = media_transform(value) if isinstance(value, str) and value.startswith("data:"): resolved = _materialize_attachment(value) if resolved is not None: diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py index f2c5d7233..4481adfb5 100644 --- a/py/src/braintrust/integrations/test_decisions.py +++ b/py/src/braintrust/integrations/test_decisions.py @@ -7,6 +7,7 @@ from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage from braintrust.integrations.question import _question_span_input +from braintrust.integrations.typesafe.tracing import _image_payload from braintrust.test_helpers import assert_dict_matches @@ -223,18 +224,19 @@ def test_decisions_system_one_maps_match_typesafe_structure(): } -def test_decisions_named_answers_match_questions_when_reordered(): +@pytest.mark.parametrize("name", ["present", None], ids=["named", "unnamed"]) +def test_decisions_named_answers_match_questions_when_reordered(name): questions = [ {"type": "box", "name": "bounds", "instructions": "The target"}, - {"type": "predicate", "name": "present", "instructions": "Is it present?"}, + {"type": "predicate", "instructions": "Is it present?", **({"name": name} if name else {})}, ] box = {"top": -0.1, "bottom": 0, "left": 0.8, "right": 0.2} assert _decisions_output( - [{"type": "predicate", "name": "present", "probability": 0.01}, {"type": "box", "name": "bounds", "box": box}], + [{"type": "predicate", "name": name, "probability": 0.01}, {"type": "box", "name": "bounds", "box": box}], questions, ) == { "answers": [ - {"id": "present", "type": "boolean", "probability": 0.01}, + {"id": name or "question_1", "type": "boolean", "probability": 0.01}, {"id": "bounds", "type": "box", "box": box}, ] } @@ -252,6 +254,21 @@ def test_decisions_nested_media_and_existing_references(): assert questions["bounds"]["instructions"]["image"] == image +def test_decisions_media_transform_handles_serialized_images(): + data_url = decisions_image_input()[0]["content"][1]["image_url"] + image = {"type": "image", "format": "png", "data_base64": data_url.split(",", 1)[1]} + questions = {"bounds": {"type": "box", "instructions": "The target"}} + span_input = _question_span_input({"photo": image}, questions, media_transform=_image_payload) + attachment = span_input["state"]["photo"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image["data_base64"]) + box = {"top": 0, "bottom": 1, "left": 0, "right": 1} + assert _decisions_output({"bounds": {"type": "box", "box": box}}, questions, span_input) == { + "answers": [{"id": "bounds", "type": "box", "box": box, "image_path": "/state/photo"}] + } + assert image["data_base64"] == data_url.split(",", 1)[1] + + def test_decisions_media_failure_keeps_entire_payload(monkeypatch): image = decisions_image_input()[0]["content"][1]["image_url"] from braintrust.integrations import question diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml index 8f938f4a9..6d4052e16 100644 --- a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml @@ -1,16 +1,16 @@ interactions: - request: body: !!binary | - LS02MjEwOGExNTFmZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + LS1mMzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjog Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u - DQoNCnsic3RhdGUiOnsicGhvdG9zIjpbInBob3RvLXJlZmVyZW5jZSIsInBob3RvLXJlZmVyZW5j - ZSJdfSwibW9kZWwiOiJqZXYtMS4xNC4wIiwicXVlc3Rpb25zIjp7InBob3RvLXJlZmVyZW5jZSI6 - eyJ0eXBlIjoiYm94IiwiaW5zdHJ1Y3Rpb25zIjp7ImltYWdlIjoicGhvdG8tcmVmZXJlbmNlIiwi - dGFyZ2V0IjoiVGhlIGJsYWNrIHNxdWFyZSJ9fSwibWF0Y2giOnsidHlwZSI6ImNob2ljZSIsImlu - c3RydWN0aW9ucyI6IkNob29zZSB0aGUgbWF0Y2hpbmcgaW1hZ2UiLCJjcml0ZXJpYSI6eyJzcXVh - cmUiOiJwaG90by1yZWZlcmVuY2UiLCJub25lIjpudWxsfX19LCJhc3NldHMiOnsicGhvdG8tcmVm - ZXJlbmNlIjp7InR5cGUiOiJpbWFnZSIsInBhcnQiOiJhc3NldC0wIn19fQ0KLS02MjEwOGExNTFm - ZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBu + DQoNCnsic3RhdGUiOnsicGhvdG9zIjpbImltYWdlIiwiaW5zcGVjdCBpbWFnZV8yIl19LCJtb2Rl + bCI6Impldi0xLjE0LjAiLCJxdWVzdGlvbnMiOnsiaW1hZ2VfMiI6eyJ0eXBlIjoiYm94IiwiaW5z + dHJ1Y3Rpb25zIjp7ImltYWdlIjoiaW1hZ2VfMiIsInRhcmdldCI6IlRoZSBibGFjayBzcXVhcmUi + fX0sIm1hdGNoIjp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJDaG9vc2UgdGhlIG1h + dGNoaW5nIGltYWdlIiwiY3JpdGVyaWEiOnsic3F1YXJlIjoiaW1hZ2VfMiIsIm5vbmUiOm51bGx9 + fX0sImFzc2V0cyI6eyJpbWFnZSI6eyJ0eXBlIjoiaW1hZ2UiLCJwYXJ0IjoiYXNzZXQtMCJ9LCJp + bWFnZV8yIjp7InR5cGUiOiJpbWFnZSIsInBhcnQiOiJhc3NldC0xIn19fQ0KLS1mMzU1ZDcyNTdi + MzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBu YW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3Bu Zw0KDQqJUE5HDQoaCgAAAA1JSERSAAABGAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx 33/pdoiaOhhpghgu7wP87LUPgAUeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ @@ -37,8 +37,36 @@ interactions: BHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ 4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GA R4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEe - CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS02 - MjEwOGExNTFmZWU0ZmVhYjNhYzFiYWM5ZGVjNTdjNS0tDQo= + CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS1m + MzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9y + bS1kYXRhOyBuYW1lPSJhc3NldC0xIjsgZmlsZW5hbWU9ImFzc2V0LTEiDQpDb250ZW50LVR5cGU6 + IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAABGAAAARgIAgAAAAjsftsAAAWHSURBVHic + 7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkC + PBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjw + SBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAj + QYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8E + AR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeC + AI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkC + PBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjw + SBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAj + QYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5E + rkJggg0KLS1mMzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMC0tDQo= headers: accept: - application/json @@ -47,9 +75,9 @@ interactions: connection: - keep-alive content-length: - - '2144' + - '3749' content-type: - - multipart/form-data; boundary=62108a151fee4feab3ac1bac9dec57c5 + - multipart/form-data; boundary=f355d7257b334ac0b4778f8cfa5fa2e0 host: - api.typesafe.ai typesafe-danger-zone-images: @@ -64,30 +92,30 @@ interactions: uri: https://api.typesafe.ai/v1/systemone response: body: - string: '{"model":"jev-1.14.0","answers":{"photo-reference":{"type":"box","box":{"top":0.05,"bottom":0.98,"left":0.13,"right":0.75},"confidence":0.19},"match":{"type":"choice","choice":"square","confidence":0.86,"probabilities":{"square":0.93,"none":0.07}}},"usage":{"input_tokens":1147,"output_tokens":48}}' + string: '{"model":"jev-1.14.0","answers":{"image_2":{"type":"box","box":{"top":0.2,"bottom":0.97,"left":0.13,"right":0.75},"confidence":0.16},"match":{"type":"choice","choice":"square","confidence":0.96,"probabilities":{"square":0.98,"none":0.02}}},"usage":{"input_tokens":1352,"output_tokens":49}}' headers: alt-svc: - h3=":443"; ma=86400 cf-cache-status: - DYNAMIC cf-ray: - - a4b0baf1182baeb6-YYZ + - a4b2b31e8ec9ac72-YYZ connection: - keep-alive content-length: - - '298' + - '289' content-type: - application/json date: - - Thu, 15 Oct 2026 18:01:38 GMT + - Thu, 15 Oct 2026 23:45:50 GMT server: - cloudflare transfer-encoding: - chunked x-envoy-upstream-service-time: - - '66' + - '151' x-typesafe-request-id: - - req_01a140ba12c97099a03a30bf5a415906 + - req_01a141f52f337112bdff11dcbe39f5b2 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index 0c3393b93..be76da339 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -218,10 +218,10 @@ def test_typesafe_nested_image_assets(memory_logger): image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" image = typesafe_sdk.Image(image_file) - state = {"photos": ["photo-reference", "photo-reference"]} + state = {"photos": ["image", "inspect image_2"]} questions = { # IDs are text, even when they happen to match an asset reference. - "photo-reference": typesafe_sdk.Box(instructions={"image": image, "target": "The black square"}), + "image_2": typesafe_sdk.Box(instructions={"image": image, "target": "The black square"}), "match": Choice(instructions="Choose the matching image", criteria={"square": image, "none": None}), } assert setup_typesafe() @@ -230,7 +230,7 @@ def test_typesafe_nested_image_assets(memory_logger): model="jev-1.14.0", state=state, questions=questions, - assets={"photo-reference": image}, + assets={"image": image, "image_2": image}, extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, ) @@ -239,16 +239,19 @@ def test_typesafe_nested_image_assets(memory_logger): span = spans[0] _assert_span(span, question_ids=list(questions)) captured = span["input"] + photos = captured["state"]["photos"] + assert photos[1] == ["inspect ", photos[0]] images = [ - *captured["state"]["photos"], + photos[0], + photos[1][1], captured["questions"][0]["instructions"]["image"], captured["questions"][1]["criteria"]["square"], ] assert all(isinstance(item, Attachment) and item.data == image_file.read_bytes() for item in images) answers = {answer["id"]: answer for answer in span["output"]["answers"]} - assert answers["photo-reference"]["box"] == response.answers["photo-reference"].box.model_dump() + assert answers["image_2"]["box"] == response.answers["image_2"].box.model_dump() # Multiple image occurrences have no structured selector in TypeSafe; # instruction text must not be used to guess their association. - assert "image_path" not in answers["photo-reference"] - assert state == {"photos": ["photo-reference", "photo-reference"]} - assert questions["photo-reference"].instructions == {"image": image, "target": "The black square"} + assert "image_path" not in answers["image_2"] + assert state == {"photos": ["image", "inspect image_2"]} + assert questions["image_2"].instructions == {"image": image, "target": "The black square"} diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index eb3176854..4d478a31d 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -116,6 +116,7 @@ def _span_input(state: Any, questions: Any, assets: Mapping[str, Any]) -> dict[s if isinstance(questions, Mapping): questions = {identifier: _question_payload(value) for identifier, value in questions.items()} references = [re.escape(key) for key in assets if isinstance(key, str) and key] + references.sort(key=len, reverse=True) pattern = re.compile("(" + "|".join(references) + ")") if references else None captured_images: dict[int, Any] = {}