diff --git a/py/pyproject.toml b/py/pyproject.toml index f2b01895e..d2393f71e 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -390,7 +390,7 @@ explicit = true # --------------------------------------------------------------------------- [tool.uv] -exclude-newer = "2026-10-13T16:16:21Z" +exclude-newer = "2026-10-15T00:00:00Z" conflicts = [ # Groups that pin or transitively require conflicting openai versions. [ @@ -577,7 +577,8 @@ latest = "mistralai==3.2.0" "1.12.4" = "mistralai==1.12.4" [tool.braintrust.matrix.typesafe-sdk] -latest = "typesafe-sdk==0.7.4" +latest = "typesafe-sdk==0.7.5" +"0.7.4" = "typesafe-sdk==0.7.4" [tool.braintrust.matrix.huggingface-hub] # Floor pinned to 0.32.0: the earliest release that exposes the diff --git a/py/src/braintrust/fixtures/decision-box.png b/py/src/braintrust/fixtures/decision-box.png new file mode 100644 index 000000000..cb7d38376 Binary files /dev/null and b/py/src/braintrust/fixtures/decision-box.png differ diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index be96fa495..6e14a8d59 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -367,16 +367,19 @@ def _decisions_span_payload(args, kwargs): def _decisions_wrapper(wrapped, instance, args, kwargs): + payload = _decisions_span_payload(args, kwargs) with start_span( **merge_dicts( dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_payload(args, kwargs), + payload, ) ) as span: response = wrapped(*args, **kwargs) span.log( output=_decisions_output( - _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + _get_field(response, "answers"), + kwargs.get("questions", args[2] if len(args) > 2 else []), + payload.get("input"), ), metrics=_parse_metrics_from_usage(_get_field(response, "usage")), ) @@ -384,16 +387,19 @@ def _decisions_wrapper(wrapped, instance, args, kwargs): async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): + payload = _decisions_span_payload(args, kwargs) with start_span( **merge_dicts( dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_payload(args, kwargs), + payload, ) ) as span: response = await wrapped(*args, **kwargs) span.log( output=_decisions_output( - _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + _get_field(response, "answers"), + kwargs.get("questions", args[2] if len(args) > 2 else []), + payload.get("input"), ), metrics=_parse_metrics_from_usage(_get_field(response, "usage")), ) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 3d4538ee2..4d885b980 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2115,23 +2115,43 @@ def _decisions_input(state: Any, questions: Any) -> dict[str, Any]: return {"state": state, "questions": []} -def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: +def _decisions_output(answers: Any, questions: Any, span_input: Any = None) -> dict[str, Any]: try: if isinstance(answers, Mapping): - return _question_span_output(answers) + return _question_span_output(answers, span_input) questions = [_try_to_dict(question) for question in questions] + identifiers = _decision_question_ids(questions) + named_questions = { + question["name"]: (identifier, question) + for identifier, question in zip(identifiers, questions) + if question.get("name") == identifier + } + answers = [_try_to_dict(answer) for answer in answers or []] + named_answer_ids = { + answer["name"] for answer in answers if isinstance(answer, dict) and answer.get("name") in named_questions + } + remaining_questions = iter( + (identifier, question) + for identifier, question in zip(identifiers, questions) + if identifier not in named_answer_ids + ) normalized = [] - # OpenAI guarantees answer order, including refusals and unnamed questions. - # Use the request's IDs so all answers remain linked to their questions. - for identifier, question, answer in zip(_decision_question_ids(questions), questions, answers or []): - answer = _try_to_dict(answer) + # Reserve named matches before pairing the remaining answers and + # questions in order, so reordering cannot duplicate a question ID. + for answer in answers: if not isinstance(answer, dict): continue + match = named_questions.pop(answer.get("name"), None) + if match is None: + match = next(remaining_questions, None) + if match is None: + continue + identifier, question = match kind = answer.get("type") payload = {"id": identifier, "type": "boolean" if kind == "predicate" else kind} - for key in ("probability", "score", "confidence"): + for key in ("noul", "probability", "score", "confidence", "box", "image_path"): if key in answer: - payload[key] = answer[key] + payload[key] = _try_to_dict(answer[key]) if key == "box" else answer[key] if kind == "choice": keys = _decision_choice_keys(question) if "choice" in answer: @@ -2148,7 +2168,7 @@ def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: } payload["legend"] = {str(item["value"]): item["label"] for item in answer["probabilities"]} normalized.append(payload) - return _question_span_output(normalized) + return _question_span_output(normalized, span_input) except Exception: _log.debug("Failed to normalize Decisions answers", exc_info=True) return {"answers": []} @@ -2185,11 +2205,11 @@ def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: ) -def _log_decisions_response(span: Span, response: Any, questions: Any): +def _log_decisions_response(span: Span, response: Any, questions: Any, span_input: Any): response = _try_to_dict(response) if isinstance(response, dict): span.log( - output=_decisions_output(response.get("answers"), questions or []), + output=_decisions_output(response.get("answers"), questions or [], span_input), metrics=_parse_metrics_from_usage(response.get("usage")), ) @@ -2198,25 +2218,27 @@ def _decisions_create_wrapper(wrapped, instance, args, kwargs): if _is_async_callable(wrapped): async def call(): + params = _decisions_span_params(kwargs) with start_span( **merge_dicts( dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_params(kwargs), + params, ) ) as span: response = await wrapped(*args, **kwargs) - _log_decisions_response(span, response, kwargs.get("questions")) + _log_decisions_response(span, response, kwargs.get("questions"), params.get("input")) return response return call() + params = _decisions_span_params(kwargs) with start_span( **merge_dicts( dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), - _decisions_span_params(kwargs), + params, ) ) as span: response = wrapped(*args, **kwargs) - _log_decisions_response(span, response, kwargs.get("questions")) + _log_decisions_response(span, response, kwargs.get("questions"), params.get("input")) return response diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py index 4376f41d7..19c50b1d3 100644 --- a/py/src/braintrust/integrations/question.py +++ b/py/src/braintrust/integrations/question.py @@ -4,14 +4,76 @@ helpers. Named maps become arrays linked by ID; arrays already carry their IDs. """ -from collections.abc import Mapping +import logging +from collections.abc import Callable, Mapping from typing import Any -from .utils import _try_to_dict +from braintrust.logger import Attachment + +from .utils import _materialize_attachment, _try_to_dict _QUESTION_FIELDS = ("type", "instructions", "criteria") -_ANSWER_FIELDS = ("type", "noul", "probability", "choice", "score", "confidence", "legend", "probabilities") +_ANSWER_FIELDS = ( + "type", + "noul", + "probability", + "choice", + "score", + "confidence", + "legend", + "probabilities", + "box", + "image_path", +) +_ATTACHMENT_TYPES = ("braintrust_attachment", "external_attachment", "inline_attachment") +_log = logging.getLogger(__name__) + + +def _decision_media(value: Any, media_transform: Callable[[Any], Any] | None = None) -> Any: + """Replace inline media in the logged payload without changing request values.""" + if isinstance(value, Mapping) and value.get("type") in _ATTACHMENT_TYPES: + return value + if media_transform is not None: + transformed = media_transform(value) + if transformed is not value: + return transformed + if isinstance(value, Mapping): + return {key: _decision_media(item, media_transform) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_decision_media(item, media_transform) for item in value] + if isinstance(value, str) and value.startswith("data:"): + resolved = _materialize_attachment(value) + if resolved is not None: + return resolved.attachment + return value + + +def _decision_image_paths(value: Any, path: str = "") -> list[str]: + """Find image occurrences, retaining repeated locations and JSON Pointer escaping.""" + if isinstance(value, Attachment): + return [path] if value.reference.get("content_type", "").startswith("image/") else [] + if isinstance(value, Mapping): + if value.get("type") in _ATTACHMENT_TYPES: + return [path] if str(value.get("content_type", "")).startswith("image/") else [] + if value.get("type") == "input_image" and isinstance(value.get("image_url"), str): + return [path + "/image_url"] + if value.get("type") == "image_url" and isinstance(value.get("image_url"), Mapping): + return [path + "/image_url/url"] + return [ + image_path + for key, item in value.items() + for image_path in _decision_image_paths(item, path + "/" + str(key).replace("~", "~0").replace("/", "~1")) + ] + if isinstance(value, (list, tuple)): + return [ + image_path + for index, item in enumerate(value) + for image_path in _decision_image_paths(item, f"{path}/{index}") + ] + if isinstance(value, str) and value.startswith("data:image/"): + return [path] + return [] def _question_items(values: Any, fields: tuple[str, ...]) -> Any: @@ -21,13 +83,48 @@ def _question_items(values: Any, fields: tuple[str, ...]) -> Any: for identifier, value in values.items(): value = _try_to_dict(value) if isinstance(value, dict): - items.append({**{key: value[key] for key in fields if key in value}, "id": identifier}) + payload = {key: value[key] for key in fields if key in value} + if "box" in payload: + payload["box"] = _try_to_dict(payload["box"]) + items.append({**payload, "id": identifier}) return items -def _question_span_input(state: Any, questions: Any) -> dict[str, Any]: - return {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} +def _question_span_input( + state: Any, questions: Any, *, media_transform: Callable[[Any], Any] | None = None +) -> dict[str, Any]: + payload = {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} + try: + return { + "state": _decision_media(state, media_transform), + "questions": [ + { + key: _decision_media(value, media_transform) if key in ("instructions", "criteria") else value + for key, value in question.items() + } + for question in payload["questions"] + ] + if isinstance(payload["questions"], list) + else payload["questions"], + } + except Exception: + # Abort the whole rewrite if a conversion fails; retain all original media. + _log.debug("Failed to capture decision media", exc_info=True) + return payload -def _question_span_output(answers: Any) -> dict[str, Any]: - return {"answers": _question_items(answers, _ANSWER_FIELDS)} +def _question_span_output(answers: Any, span_input: Any = None) -> dict[str, Any]: + items = _question_items(answers, _ANSWER_FIELDS) + if not isinstance(items, list) or not any( + isinstance(answer, dict) and answer.get("type") == "box" and "image_path" not in answer for answer in items + ): + return {"answers": items} + image_paths = _decision_image_paths(span_input) + if len(image_paths) == 1: + items = [ + {**answer, "image_path": image_paths[0]} + if isinstance(answer, dict) and answer.get("type") == "box" and "image_path" not in answer + else answer + for answer in items + ] + return {"answers": items} diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py index 4742cf95a..4481adfb5 100644 --- a/py/src/braintrust/integrations/test_decisions.py +++ b/py/src/braintrust/integrations/test_decisions.py @@ -6,6 +6,8 @@ from braintrust.integrations.litellm.tracing import _parse_metrics_from_usage as parse_litellm_usage from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage +from braintrust.integrations.question import _question_span_input +from braintrust.integrations.typesafe.tracing import _image_payload from braintrust.test_helpers import assert_dict_matches @@ -195,13 +197,98 @@ def test_decisions_refusals_and_allowlists(): def test_decisions_system_one_maps_match_typesafe_structure(): - questions = {"positive": {"type": "noul", "instructions": "Is this positive?", "secret": "omit"}} - answers = {"positive": {"type": "noul", "noul": 0.9, "secret": "omit"}} + questions = {"present": {"type": "noul", "instructions": "Is the target present?", "secret": "omit"}} + answers = {"present": {"type": "noul", "noul": 0.01, "secret": "omit"}} + box = {"top": 0, "bottom": 1, "left": 0.123456789, "right": 0.9} + for identifier, optional_fields in ( + ("zero", {"confidence": 0}), + ("null", {"confidence": None}), + ("absent", {}), + ): + questions[identifier] = {"type": "box", "instructions": {"target": "label"}} + answers[identifier] = {"type": "box", "box": box, "image_path": "/state/photos/1", **optional_fields} + # Map keys link answers to questions, independently of return order. + answers = dict(reversed(answers.items())) assert _decisions_input("state", questions) == { "state": "state", - "questions": [{"id": "positive", "type": "noul", "instructions": "Is this positive?"}], + "questions": [ + {"id": "present", "type": "noul", "instructions": "Is the target present?"}, + *({"id": identifier, **questions[identifier]} for identifier in ("zero", "null", "absent")), + ], + } + assert _decisions_output(answers, questions) == { + "answers": [ + *({"id": identifier, **answers[identifier]} for identifier in ("absent", "null", "zero")), + {"id": "present", "type": "noul", "noul": 0.01}, + ] + } + + +@pytest.mark.parametrize("name", ["present", None], ids=["named", "unnamed"]) +def test_decisions_named_answers_match_questions_when_reordered(name): + questions = [ + {"type": "box", "name": "bounds", "instructions": "The target"}, + {"type": "predicate", "instructions": "Is it present?", **({"name": name} if name else {})}, + ] + box = {"top": -0.1, "bottom": 0, "left": 0.8, "right": 0.2} + assert _decisions_output( + [{"type": "predicate", "name": name, "probability": 0.01}, {"type": "box", "name": "bounds", "box": box}], + questions, + ) == { + "answers": [ + {"id": name or "question_1", "type": "boolean", "probability": 0.01}, + {"id": "bounds", "type": "box", "box": box}, + ] + } + + +def test_decisions_nested_media_and_existing_references(): + image = decisions_image_input()[0]["content"][1]["image_url"] + reference = {"type": "external_attachment", "url": "https://example.com/image.png", "content_type": "image/png"} + questions = {"bounds": {"type": "box", "instructions": {"image": image, "text": "target"}}} + span_input = _decisions_input({"photos": [reference, image], "remote": "https://example.com/a.png"}, questions) + assert span_input["state"]["photos"][0] == reference + assert isinstance(span_input["state"]["photos"][1], Attachment) + assert isinstance(span_input["questions"][0]["instructions"]["image"], Attachment) + assert span_input["state"]["remote"] == "https://example.com/a.png" + assert questions["bounds"]["instructions"]["image"] == image + + +def test_decisions_media_transform_handles_serialized_images(): + data_url = decisions_image_input()[0]["content"][1]["image_url"] + image = {"type": "image", "format": "png", "data_base64": data_url.split(",", 1)[1]} + questions = {"bounds": {"type": "box", "instructions": "The target"}} + span_input = _question_span_input({"photo": image}, questions, media_transform=_image_payload) + attachment = span_input["state"]["photo"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image["data_base64"]) + box = {"top": 0, "bottom": 1, "left": 0, "right": 1} + assert _decisions_output({"bounds": {"type": "box", "box": box}}, questions, span_input) == { + "answers": [{"id": "bounds", "type": "box", "box": box, "image_path": "/state/photo"}] + } + assert image["data_base64"] == data_url.split(",", 1)[1] + + +def test_decisions_media_failure_keeps_entire_payload(monkeypatch): + image = decisions_image_input()[0]["content"][1]["image_url"] + from braintrust.integrations import question + + materialize = question._materialize_attachment + calls = 0 + + def fail_second(value): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("conversion failed") + return materialize(value) + + monkeypatch.setattr(question, "_materialize_attachment", fail_second) + questions = {"bounds": {"type": "box", "instructions": {"image": image}}} + assert _question_span_input({"image": image}, questions) == { + "state": {"image": image}, + "questions": [{"id": "bounds", **questions["bounds"]}], } - assert _decisions_output(answers, questions) == {"answers": [{"id": "positive", "type": "noul", "noul": 0.9}]} @pytest.mark.parametrize("parse_usage", (parse_openai_usage, parse_litellm_usage), ids=("openai", "litellm")) diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml new file mode 100644 index 000000000..0c0bd4d51 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_auto_typesafe.yaml @@ -0,0 +1,45 @@ +interactions: +- request: + body: '{"state":"The package arrived intact and on time.","model":"jev-latest","questions":{"positive":{"type":"noul","instructions":"Is + this feedback positive?"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '157' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"positive":{"type":"noul","noul":0.97}},"usage":{"input_tokens":279,"output_tokens":20}}' + headers: + content-length: + - '121' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:11 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '77' + x-typesafe-request-id: + - req_01a0c993f3ff71eeb69776ee462670ac + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml new file mode 100644 index 000000000..65cedc0d6 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_setup_typesafe_system_one_async.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"state":{"message":"I was charged twice. Please refund the duplicate charge + today."},"model":"jev-latest","questions":{"category":{"type":"choice","instructions":"Which + team should handle this?","criteria":{"billing":null,"technical":null,"other":null}},"urgency":{"type":"score","instructions":"How + urgent is this request?","criteria":["routine","soon","urgent"]},"duplicate_charge":{"type":"noul","instructions":{"question":"Does + the customer report a duplicate charge?"}}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '477' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"category":{"type":"choice","choice":"billing","confidence":1.0,"probabilities":{"technical":0.0,"billing":1.0,"other":0.0}},"urgency":{"type":"score","score":1.78,"confidence":0.67,"legend":{"0":"routine","1":"soon","2":"urgent"},"probabilities":{"0":0.0,"1":0.22,"2":0.78}},"duplicate_charge":{"type":"noul","noul":0.99}},"usage":{"input_tokens":370,"output_tokens":70}}' + headers: + content-length: + - '405' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:10 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '132' + x-typesafe-request-id: + - req_01a0c993f0b57238bb5b0d6f611be2bf + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml new file mode 100644 index 000000000..d75e6c0aa --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/0.7.4/test_wrap_typesafe_system_one_sync.yaml @@ -0,0 +1,45 @@ +interactions: +- request: + body: '{"state":"The package arrived intact and on time.","model":"jev-latest","questions":{"positive":{"type":"noul","instructions":"Is + this feedback positive?","not_allowlisted":"secret"}}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '184' + content-type: + - application/json + host: + - api.typesafe.ai + user-agent: + - typesafe-sdk/0.7.0 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.0 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.13.0","answers":{"positive":{"type":"noul","noul":0.97}},"usage":{"input_tokens":279,"output_tokens":20}}' + headers: + content-length: + - '121' + content-type: + - application/json + date: + - Tue, 22 Sep 2026 14:45:10 GMT + server: + - istio-envoy + x-envoy-upstream-service-time: + - '101' + x-typesafe-request-id: + - req_01a0c993ef287b88a5f5929d703f4eb0 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml new file mode 100644 index 000000000..24b4cf860 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[async].yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: !!binary | + LS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0NjhjOGY3MGNlMQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zL34iOlsiPGltYWdlOjIzZDNlMTdkOGUzMTRiYzQ4MmZiZjAw + YzcxNTYzZTYwPiJdLCJkZXNjcmlwdGlvbiI6IkEgdGVzdCBpbWFnZSJ9LCJtb2RlbCI6Impldi0x + LjE0LjAiLCJxdWVzdGlvbnMiOnsiYm91bmRzIjp7InR5cGUiOiJib3giLCJpbnN0cnVjdGlvbnMi + OiJUaGUgbWFpbiBvYmplY3QgaW4gYHBob3Rvcy9+WzBdYCJ9LCJwcmVzZW50Ijp7InR5cGUiOiJu + b3VsIiwiaW5zdHJ1Y3Rpb25zIjoiSXMgdGhlcmUgYSBtYWluIG9iamVjdCBpbiB0aGUgaW1hZ2U/ + In0sImNhdGVnb3J5Ijp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJXaGF0IGlzIGlu + IHRoZSBpbWFnZT8iLCJjcml0ZXJpYSI6eyJvYmplY3QiOm51bGwsInRleHQiOm51bGx9fSwic2l6 + ZSI6eyJ0eXBlIjoic2NvcmUiLCJpbnN0cnVjdGlvbnMiOiJIb3cgbXVjaCBvZiB0aGUgaW1hZ2Ug + aXMgb2NjdXBpZWQ/IiwiY3JpdGVyaWEiOlsibGl0dGxlIiwibW9zdCJdfX0sImFzc2V0cyI6eyI8 + aW1hZ2U6MjNkM2UxN2Q4ZTMxNGJjNDgyZmJmMDBjNzE1NjNlNjA+Ijp7InR5cGUiOiJpbWFnZSIs + InBhcnQiOiJhc3NldC0wIn19fQ0KLS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0NjhjOGY3MGNlMQ0K + Q29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9 + ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAAB + GAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS02YjU5ZWQ2MDQzZjQ0ZDEyYmE4YTY0Njhj + OGY3MGNlMS0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '2348' + content-type: + - multipart/form-data; boundary=6b59ed6043f44d12ba8a6468c8f70ce1 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"bounds":{"type":"box","box":{"top":0.04,"bottom":0.97,"left":0.13,"right":0.75},"confidence":0.24},"present":{"type":"noul","noul":0.47},"category":{"type":"choice","choice":"object","confidence":0.9,"probabilities":{"object":0.95,"text":0.05}},"size":{"type":"score","score":0.54,"confidence":0.08,"legend":{"0":"little","1":"most"},"probabilities":{"0":0.46,"1":0.54}}},"usage":{"input_tokens":610,"output_tokens":77}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b0b61409290eff-YYZ + connection: + - keep-alive + content-length: + - '454' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 17:58:19 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '46' + x-typesafe-request-id: + - req_01a140b708a5708fae9b3420db044484 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml new file mode 100644 index 000000000..5b6f600eb --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_boxes_and_images[sync].yaml @@ -0,0 +1,98 @@ +interactions: +- request: + body: !!binary | + LS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1YjQxNjRjNQ0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zL34iOlsiPGltYWdlOjBkNTU5OWNkMGFkMjQ1ZWI5MmQ0NDU5 + N2ZjMTkxNmM1PiJdLCJkZXNjcmlwdGlvbiI6IkEgdGVzdCBpbWFnZSJ9LCJtb2RlbCI6Impldi0x + LjE0LjAiLCJxdWVzdGlvbnMiOnsiYm91bmRzIjp7InR5cGUiOiJib3giLCJpbnN0cnVjdGlvbnMi + OiJUaGUgbWFpbiBvYmplY3QgaW4gYHBob3Rvcy9+WzBdYCJ9LCJwcmVzZW50Ijp7InR5cGUiOiJu + b3VsIiwiaW5zdHJ1Y3Rpb25zIjoiSXMgdGhlcmUgYSBtYWluIG9iamVjdCBpbiB0aGUgaW1hZ2U/ + In0sImNhdGVnb3J5Ijp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJXaGF0IGlzIGlu + IHRoZSBpbWFnZT8iLCJjcml0ZXJpYSI6eyJvYmplY3QiOm51bGwsInRleHQiOm51bGx9fSwic2l6 + ZSI6eyJ0eXBlIjoic2NvcmUiLCJpbnN0cnVjdGlvbnMiOiJIb3cgbXVjaCBvZiB0aGUgaW1hZ2Ug + aXMgb2NjdXBpZWQ/IiwiY3JpdGVyaWEiOlsibGl0dGxlIiwibW9zdCJdfX0sImFzc2V0cyI6eyI8 + aW1hZ2U6MGQ1NTk5Y2QwYWQyNDVlYjkyZDQ0NTk3ZmMxOTE2YzU+Ijp7InR5cGUiOiJpbWFnZSIs + InBhcnQiOiJhc3NldC0wIn19fQ0KLS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1YjQxNjRjNQ0K + Q29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9 + ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAAB + GAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8 + EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBI + EOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNB + gEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQB + HgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS1iNGY3ODNlMmM5NWE0NGM5ODkzN2E2ODk1 + YjQxNjRjNS0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '2348' + content-type: + - multipart/form-data; boundary=b4f783e2c95a44c98937a6895b4164c5 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"bounds":{"type":"box","box":{"top":0.05,"bottom":0.97,"left":0.14,"right":0.69},"confidence":0.25},"present":{"type":"noul","noul":0.51},"category":{"type":"choice","choice":"object","confidence":0.88,"probabilities":{"object":0.94,"text":0.06}},"size":{"type":"score","score":0.59,"confidence":0.18,"legend":{"0":"little","1":"most"},"probabilities":{"0":0.41,"1":0.59}}},"usage":{"input_tokens":608,"output_tokens":77}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b0b612be89aac7-YYZ + connection: + - keep-alive + content-length: + - '455' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 17:58:19 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '56' + x-typesafe-request-id: + - req_01a140b707ce747a90cf04ef22708fcb + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml new file mode 100644 index 000000000..6d4052e16 --- /dev/null +++ b/py/src/braintrust/integrations/typesafe/cassettes/latest/test_typesafe_nested_image_assets.yaml @@ -0,0 +1,122 @@ +interactions: +- request: + body: !!binary | + LS1mMzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjog + Zm9ybS1kYXRhOyBuYW1lPSJyZXF1ZXN0Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29u + DQoNCnsic3RhdGUiOnsicGhvdG9zIjpbImltYWdlIiwiaW5zcGVjdCBpbWFnZV8yIl19LCJtb2Rl + bCI6Impldi0xLjE0LjAiLCJxdWVzdGlvbnMiOnsiaW1hZ2VfMiI6eyJ0eXBlIjoiYm94IiwiaW5z + dHJ1Y3Rpb25zIjp7ImltYWdlIjoiaW1hZ2VfMiIsInRhcmdldCI6IlRoZSBibGFjayBzcXVhcmUi + fX0sIm1hdGNoIjp7InR5cGUiOiJjaG9pY2UiLCJpbnN0cnVjdGlvbnMiOiJDaG9vc2UgdGhlIG1h + dGNoaW5nIGltYWdlIiwiY3JpdGVyaWEiOnsic3F1YXJlIjoiaW1hZ2VfMiIsIm5vbmUiOm51bGx9 + fX0sImFzc2V0cyI6eyJpbWFnZSI6eyJ0eXBlIjoiaW1hZ2UiLCJwYXJ0IjoiYXNzZXQtMCJ9LCJp + bWFnZV8yIjp7InR5cGUiOiJpbWFnZSIsInBhcnQiOiJhc3NldC0xIn19fQ0KLS1mMzU1ZDcyNTdi + MzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBu + YW1lPSJhc3NldC0wIjsgZmlsZW5hbWU9ImFzc2V0LTAiDQpDb250ZW50LVR5cGU6IGltYWdlL3Bu + Zw0KDQqJUE5HDQoaCgAAAA1JSERSAAABGAAAARgIAgAAAAjsftsAAAWHSURBVHic7dOBCQNADAOx + 33/pdoiaOhhpghgu7wP87LUPgAUeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ + 4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GA + R4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEe + CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgk + CPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeCAI8EAR4JAjwS + BHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ + 4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GA + R4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEe + CQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5ErkJggg0KLS1m + MzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMA0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9y + bS1kYXRhOyBuYW1lPSJhc3NldC0xIjsgZmlsZW5hbWU9ImFzc2V0LTEiDQpDb250ZW50LVR5cGU6 + IGltYWdlL3BuZw0KDQqJUE5HDQoaCgAAAA1JSERSAAABGAAAARgIAgAAAAjsftsAAAWHSURBVHic + 7dOBCQNADAOx33/pdoiaOhhpghgu7wP87LUPgAUeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkC + PBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjw + SBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAj + QYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8E + AR4JAjwSBHgkCPBIEDD4SI/z2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7i + vHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYj + eYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuT + OK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/d + SN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7i + JM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5r + N5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3 + OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz + 2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3k + LU7ivHYjeYuTOK/dSN7iJM5rN5K3OInz2o3kLU7ivHYjeYuTOK/dSN7iJM5rN5I3OAn+zyNBgEeC + AI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkC + PBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAjQYBHggCPBAEeCQI8EgR4JAjw + SBDgkSDAI0GAR4IAjwQBHgkCPBIEeCQI8EgQ4JEgwCNBgEeCAI8EAR4JAjwSBHgkCPBIEOCRIMAj + QYBHggCPBAEeCQI8EgR4JAjwSBDgkSDAI0GAR4IAjwQBHgkCPBIEfAFb2IcjR+brDgAAAABJRU5E + rkJggg0KLS1mMzU1ZDcyNTdiMzM0YWMwYjQ3NzhmOGNmYTVmYTJlMC0tDQo= + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '3749' + content-type: + - multipart/form-data; boundary=f355d7257b334ac0b4778f8cfa5fa2e0 + host: + - api.typesafe.ai + typesafe-danger-zone-images: + - 'true' + user-agent: + - typesafe-sdk/0.7.5 + x-typesafe-runtime: + - python/3.14.7 (darwin; arm64) + x-typesafe-sdk: + - typesafe-sdk/0.7.5 + method: POST + uri: https://api.typesafe.ai/v1/systemone + response: + body: + string: '{"model":"jev-1.14.0","answers":{"image_2":{"type":"box","box":{"top":0.2,"bottom":0.97,"left":0.13,"right":0.75},"confidence":0.16},"match":{"type":"choice","choice":"square","confidence":0.96,"probabilities":{"square":0.98,"none":0.02}}},"usage":{"input_tokens":1352,"output_tokens":49}}' + headers: + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b2b31e8ec9ac72-YYZ + connection: + - keep-alive + content-length: + - '289' + content-type: + - application/json + date: + - Thu, 15 Oct 2026 23:45:50 GMT + server: + - cloudflare + transfer-encoding: + - chunked + x-envoy-upstream-service-time: + - '151' + x-typesafe-request-id: + - req_01a141f52f337112bdff11dcbe39f5b2 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index b12816527..be76da339 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -1,11 +1,14 @@ import os +from pathlib import Path import pytest -from braintrust import logger +import typesafe_sdk +from braintrust import Attachment, logger from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.integrations.typesafe import setup_typesafe, wrap_typesafe from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger +from pydantic import BaseModel from typesafe_sdk import AsyncTypeSafeClient, Choice, Noul, Score, TypeSafeClient, TypeSafeError @@ -126,3 +129,129 @@ async def test_setup_typesafe_system_one_async(memory_logger): def test_auto_instrument_typesafe_subprocess(): verify_autoinstrument_script("test_auto_typesafe.py") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.vcr +async def test_typesafe_boxes_and_images(memory_logger, is_async): + if not hasattr(typesafe_sdk, "Box"): + pytest.skip("Box and Image require TypeSafe SDK >=0.7.5") + + class CustomImage(typesafe_sdk.Image): + pass + + class CustomAnswers(BaseModel): + bounds: typesafe_sdk.BoxAnswer + present: typesafe_sdk.NoulAnswer + category: typesafe_sdk.ChoiceAnswer + size: typesafe_sdk.ScoreAnswer + + class CustomResponse(BaseModel): + model: str + usage: typesafe_sdk.Usage + answers: CustomAnswers + + image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" + image = CustomImage(image_file) + state = {"photos/~": [image], "description": "A test image"} + questions = { + "bounds": typesafe_sdk.Box(instructions="The main object in `photos/~[0]`"), + "present": Noul(instructions="Is there a main object in the image?"), + "category": Choice(instructions="What is in the image?", criteria={"object": None, "text": None}), + "size": Score(instructions="How much of the image is occupied?", criteria=["little", "most"]), + } + assert setup_typesafe() + client_type = AsyncTypeSafeClient if is_async else TypeSafeClient + client = client_type(api_key=os.environ["TYPESAFE_API_KEY"], compression=False) + kwargs = dict( + model="jev-1.14.0", + state=state, + questions=questions, + extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, + ) + if is_async: + kwargs["response_model"] = CustomResponse + async with client: + response = await client.system_one(**kwargs) + assert isinstance(response, CustomResponse) + assert isinstance(response.answers, CustomAnswers) + response_answers = {identifier: getattr(response.answers, identifier) for identifier in questions} + else: + with client: + response = client.system_one(**kwargs) + response_answers = response.answers + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + _assert_span(span, question_ids=list(questions)) + assert span["metadata"]["model"] == "jev-1.14.0" + attachment = span["input"]["state"]["photos/~"][0] + assert isinstance(attachment, Attachment) + assert attachment.data == image_file.read_bytes() + assert attachment.reference["content_type"] == "image/png" + assert state["photos/~"][0] is image + assert span["input"]["questions"][0] == { + "id": "bounds", + "type": "box", + "instructions": questions["bounds"].instructions, + } + answers = {answer["id"]: answer for answer in span["output"]["answers"]} + assert answers["bounds"] == { + "id": "bounds", + "type": "box", + "box": response_answers["bounds"].box.model_dump(), + "confidence": response_answers["bounds"].confidence, + "image_path": "/state/photos~1~0/0", + } + assert answers["present"]["noul"] == response_answers["present"].noul + assert answers["size"]["score"] == response_answers["size"].score + assert "scores" not in span + assert "classifications" not in span + + +@pytest.mark.vcr +def test_typesafe_nested_image_assets(memory_logger): + if not hasattr(typesafe_sdk, "Box"): + pytest.skip("Box and Image require TypeSafe SDK >=0.7.5") + + image_file = Path(__file__).parents[2] / "fixtures" / "decision-box.png" + image = typesafe_sdk.Image(image_file) + state = {"photos": ["image", "inspect image_2"]} + questions = { + # IDs are text, even when they happen to match an asset reference. + "image_2": typesafe_sdk.Box(instructions={"image": image, "target": "The black square"}), + "match": Choice(instructions="Choose the matching image", criteria={"square": image, "none": None}), + } + assert setup_typesafe() + with TypeSafeClient(api_key=os.environ["TYPESAFE_API_KEY"], compression=False) as client: + response = client.system_one( + model="jev-1.14.0", + state=state, + questions=questions, + assets={"image": image, "image_2": image}, + extra_headers={"TypeSafe-Danger-Zone-Images": "true"}, + ) + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + _assert_span(span, question_ids=list(questions)) + captured = span["input"] + photos = captured["state"]["photos"] + assert photos[1] == ["inspect ", photos[0]] + images = [ + photos[0], + photos[1][1], + captured["questions"][0]["instructions"]["image"], + captured["questions"][1]["criteria"]["square"], + ] + assert all(isinstance(item, Attachment) and item.data == image_file.read_bytes() for item in images) + answers = {answer["id"]: answer for answer in span["output"]["answers"]} + assert answers["image_2"]["box"] == response.answers["image_2"].box.model_dump() + # Multiple image occurrences have no structured selector in TypeSafe; + # instruction text must not be used to guess their association. + assert "image_path" not in answers["image_2"] + assert state == {"photos": ["image", "inspect image_2"]} + assert questions["image_2"].instructions == {"image": image, "target": "The black square"} diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index f628bda25..4d478a31d 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -1,5 +1,8 @@ """TypeSafe-specific tracing helpers.""" +import os +import re +import sys from collections.abc import Mapping from typing import Any @@ -9,7 +12,12 @@ _question_span_input, _question_span_output, ) -from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span +from braintrust.integrations.utils import ( + _log_and_end_span, + _log_error_and_end_span, + _materialize_attachment, + _try_to_dict, +) from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute @@ -71,10 +79,73 @@ def _request_parts(instance: Any, args: Any, kwargs: dict[str, Any]) -> tuple[An return state, questions, model -def _span_input(state: Any, questions: Any) -> dict[str, Any]: +def _image_payload(value: Any) -> Any: + # Only interpret TypeSafe's Image objects as files/buffers. Arbitrary JSON + # strings in the request must never be treated as local paths. + image_type = getattr(sys.modules.get("typesafe_sdk"), "Image", ()) + if isinstance(value, image_type): + data = value.data + if isinstance(data, os.PathLike): + with open(data, "rb") as image_file: + data = image_file.read() + else: + data = bytes(data) + mime_type = ( + "image/png" + if data.startswith(b"\x89PNG\r\n\x1a\n") + else "image/jpeg" + if data.startswith(b"\xff\xd8\xff") + else None + ) + if mime_type is not None: + resolved = _materialize_attachment(data, mime_type=mime_type) + if resolved is not None: + return resolved.attachment + if isinstance(value, Mapping) and "data_base64" in value: + image_format = value.get("format") + if image_format in ("png", "jpg", "jpeg") and isinstance(value["data_base64"], str): + resolved = _materialize_attachment( + value["data_base64"], mime_type="image/png" if image_format == "png" else "image/jpeg" + ) + if resolved is not None: + return resolved.attachment + return value + + +def _span_input(state: Any, questions: Any, assets: Mapping[str, Any]) -> dict[str, Any]: if isinstance(questions, Mapping): questions = {identifier: _question_payload(value) for identifier, value in questions.items()} - return _question_span_input(state, questions) + references = [re.escape(key) for key in assets if isinstance(key, str) and key] + references.sort(key=len, reverse=True) + pattern = re.compile("(" + "|".join(references) + ")") if references else None + captured_images: dict[int, Any] = {} + + def capture(value): + key = id(value) + if key in captured_images: + return captured_images[key] + captured = _image_payload(value) + if captured is not value: + captured_images[key] = captured + return captured + + def resolve(reference): + asset = assets[reference] + captured = capture(asset) + return reference if captured is asset else captured + + def media_transform(value): + if isinstance(value, str) and assets: + if value in assets: + return resolve(value) + # TypeSafe expands references inside text as well as whole values. + if pattern is not None: + parts = pattern.split(value) + if len(parts) > 1: + return [resolve(part) if part in assets else part for part in parts if part] + return capture(value) + + return _question_span_input(state, questions, media_transform=media_transform) def _token_metrics(response: Any) -> dict[str, int]: @@ -91,14 +162,14 @@ def _token_metrics(response: Any) -> dict[str, int]: return metrics -def _log_response(span: Any, response: Any) -> None: - answers = getattr(response, "answers", None) +def _log_response(span: Any, response: Any, span_input: Any) -> None: + answers = _try_to_dict(getattr(response, "answers", None)) if isinstance(answers, Mapping): answers = {identifier: _answer_payload(value) for identifier, value in answers.items()} response_model = getattr(response, "model", None) _log_and_end_span( span, - output=_question_span_output(answers), + output=_question_span_output(answers, span_input), metrics=_token_metrics(response), metadata={"model": response_model} if isinstance(response_model, str) else None, ) @@ -109,16 +180,26 @@ def _start_system_one_span(instance: Any, args: Any, kwargs: dict[str, Any]): metadata = {"provider": "typesafe"} if isinstance(model, str): metadata["model"] = model + assets = dict(kwargs.get("assets") or {}) + attached = getattr(instance, "_attached", None) + if attached is not None: + assets.update(attached.snapshot()) + extra_body = kwargs.get("extra_body") + if isinstance(extra_body, Mapping) and isinstance(extra_body.get("assets"), Mapping): + assets = dict(extra_body["assets"]) + # Images may also be referenced in text by their SDK-generated reference. + assets.update({str(image): image for image in list(assets.values()) if not isinstance(image, Mapping)}) + span_input = _span_input(state, questions, assets) return start_span( name="typesafe.systemOne", type=SpanTypeAttribute.DECISION, - input=_span_input(state, questions), + input=span_input, metadata=metadata, - ) + ), span_input def _system_one_wrapper(wrapped, instance, args, kwargs): - span = _start_system_one_span(instance, args, kwargs) + span, span_input = _start_system_one_span(instance, args, kwargs) try: response = wrapped(*args, **kwargs) @@ -126,12 +207,12 @@ def _system_one_wrapper(wrapped, instance, args, kwargs): _log_error_and_end_span(span, error) raise - _log_response(span, response) + _log_response(span, response, span_input) return response async def _async_system_one_wrapper(wrapped, instance, args, kwargs): - span = _start_system_one_span(instance, args, kwargs) + span, span_input = _start_system_one_span(instance, args, kwargs) try: response = await wrapped(*args, **kwargs) @@ -139,5 +220,5 @@ async def _async_system_one_wrapper(wrapped, instance, args, kwargs): _log_error_and_end_span(span, error) raise - _log_response(span, response) + _log_response(span, response, span_input) return response