diff --git a/py/src/braintrust/integrations/instructor/cassettes/1.11.0/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml b/py/src/braintrust/integrations/instructor/cassettes/1.11.0/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml new file mode 100644 index 000000000..488cb32b9 --- /dev/null +++ b/py/src/braintrust/integrations/instructor/cassettes/1.11.0/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Extract Grace, age 45."}],"model":"gpt-4o-mini","stream":true,"tool_choice":{"type":"function","function":{"name":"PartialPerson"}},"tools":[{"type":"function","function":{"name":"PartialPerson","description":"Correctly + extracted `PartialPerson` with all the required parameters with correct types","parameters":{"properties":{"name":{"description":"The + person''s name","title":"Name","type":"string"},"age":{"description":"The person''s + age","title":"Age","type":"integer"}},"required":["age","name"],"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '559' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_I2fwY1uN1ofFXL4ZgpYzBDDT","type":"function","function":{"name":"PartialPerson","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"9QSzSrjjCRMh5zB"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"ECHgUygz3ii"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"name"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"eM9ysbyg4X"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"GhVl3sB68"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Grace"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"gqhKRgmoW"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\",\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"evwCjhJ56"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"age"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"PbS2SRGmlCO"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"Qen2EtOGLod"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"45"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"eHhInggNSW1h"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"YRL4rgYv66qHc"} + + + data: {"id":"chatcmpl-EZPTv89i3vv8hoVEZc5ErscY9glP2","object":"chat.completion.chunk","created":1792108139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"1G"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b2b7c0aba619cf-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Thu, 15 Oct 2026 23:49:00 GMT + openai-processing-ms: + - '281' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_450da74ac2b64aad8f822cb75184a561 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/instructor/cassettes/latest/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml b/py/src/braintrust/integrations/instructor/cassettes/latest/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml new file mode 100644 index 000000000..5427c0059 --- /dev/null +++ b/py/src/braintrust/integrations/instructor/cassettes/latest/TestInstructorOpenAISpans.test_instructor_openai_create_partial_logs_final_snapshot.yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Extract Grace, age 45."}],"model":"gpt-4o-mini","stream":true,"tool_choice":{"type":"function","function":{"name":"PartialPerson"}},"tools":[{"type":"function","function":{"name":"PartialPerson","description":"Correctly + extracted `PartialPerson` with all the required parameters with correct types","parameters":{"properties":{"name":{"description":"The + person''s name","title":"Name","type":"string"},"age":{"description":"The person''s + age","title":"Age","type":"integer"}},"required":["age","name"],"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '559' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_zeVRmcwo8wy7oFjsSsjwdYWJ","type":"function","function":{"name":"PartialPerson","arguments":""}}],"refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"TAtNE0iXbbbwkDu"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"DWOtRLxMnO3"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"name"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"Z1ZsjZ9CLb"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"tAJNTn5F6"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Grace"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"2SjGbCXCu"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\",\""}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"sjwP3op2W"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"age"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"OOvZ5cxm885"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"SRnlg3oVqsD"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"45"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"hSTToF8Uurw0"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}],"obfuscation":"wm5d3PPhCbkNx"} + + + data: {"id":"chatcmpl-EZPTBbOghFpPGuHq4xrcAD3QDNQTI","object":"chat.completion.chunk","created":1792108093,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_2113251dd4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"hP"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4b2b69a7b28ac7c-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Thu, 15 Oct 2026 23:48:13 GMT + openai-processing-ms: + - '339' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3b5a65f672274660a58bd6c631b6e024 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/instructor/test_instructor.py b/py/src/braintrust/integrations/instructor/test_instructor.py index 89ef740bd..d6853d695 100644 --- a/py/src/braintrust/integrations/instructor/test_instructor.py +++ b/py/src/braintrust/integrations/instructor/test_instructor.py @@ -188,6 +188,32 @@ def test_instructor_openai_retries_then_succeeds(self, setup_logger, memory_logg for k in ("tokens", "prompt_tokens", "completion_tokens", "total_tokens"): assert k not in parent_metrics + @pytest.mark.vcr + def test_instructor_openai_create_partial_logs_final_snapshot(self, setup_logger, memory_logger): + """Partial snapshots are yielded to callers, but only the final one is logged.""" + from braintrust import wrap_openai + + client = wrap_openai(_make_openai_client()) + patched = instructor.from_openai(client, mode=instructor.Mode.TOOLS) + + result = list( + patched.chat.completions.create_partial( + model="gpt-4o-mini", + response_model=Person, + max_retries=3, + messages=[{"role": "user", "content": "Extract Grace, age 45."}], + ) + ) + assert len(result) > 1, "create_partial should yield progressive snapshots" + assert _dump(result[-1]) == {"name": "Grace", "age": 45} + + spans = _all_spans(memory_logger) + task_spans = find_spans_by_type(spans, SpanTypeAttribute.TASK) + assert len(task_spans) == 1 + parent = task_spans[0] + assert parent["span_attributes"]["name"] == "instructor.create_partial" + assert _dump(parent["output"]) == _dump(result[-1]) + class TestInstructorAutoInstrumentSubprocess: """auto_instrument() must instrument Instructor in a fresh subprocess too.""" diff --git a/py/src/braintrust/integrations/instructor/tracing.py b/py/src/braintrust/integrations/instructor/tracing.py index 5f110bf87..28fb7786f 100644 --- a/py/src/braintrust/integrations/instructor/tracing.py +++ b/py/src/braintrust/integrations/instructor/tracing.py @@ -14,8 +14,8 @@ - ``name`` : ``instructor.create`` (or ``.create_with_completion`` etc.) - ``input`` : ``{"response_model": , "mode": , "messages": [...]}`` - ``output``: the extracted Pydantic model (or list of models for - iterable/partial helpers). This is Instructor's product and is not - present on any provider child span. + iterable helpers). Partial helpers log only their final snapshot. This is + Instructor's product and is not present on any provider child span. - ``metadata``: ``model``, ``provider``, ``response_model`` (class name), ``mode``, ``max_retries``, ``retry_count``, ``validation_errors``. The ``model`` / ``provider`` pair carries attribution for the framework span @@ -310,7 +310,7 @@ async def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: return _wrapper -def _make_sync_iterable_wrapper(span_name: str): +def _make_sync_iterable_wrapper(span_name: str, *, final_only: bool = False): def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: response_model = _extract_response_model(args, kwargs) max_retries = kwargs.get("max_retries", 3) @@ -328,7 +328,11 @@ def _iterate() -> Iterator[Any]: try: result = wrapped(*args, **kwargs) for item in result: - output.append(_extract_output(item)) + extracted = _extract_output(item) + if final_only: + output = extracted + else: + output.append(extracted) yield item except Exception as exc: span.log( @@ -360,7 +364,7 @@ def _iterate() -> Iterator[Any]: return _wrapper -def _make_async_iterable_wrapper(span_name: str): +def _make_async_iterable_wrapper(span_name: str, *, final_only: bool = False): def _wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: response_model = _extract_response_model(args, kwargs) max_retries = kwargs.get("max_retries", 3) @@ -380,7 +384,11 @@ async def _iterate() -> AsyncIterator[Any]: if inspect.isawaitable(result): result = await result async for item in result: - output.append(_extract_output(item)) + extracted = _extract_output(item) + if final_only: + output = extracted + else: + output.append(extracted) yield item except Exception as exc: span.log( @@ -416,10 +424,10 @@ async def _iterate() -> AsyncIterator[Any]: # patcher targets in ``patchers.py``. _create_wrapper = _make_sync_wrapper("instructor.create") _create_with_completion_wrapper = _make_sync_wrapper("instructor.create_with_completion") -_create_partial_wrapper = _make_sync_iterable_wrapper("instructor.create_partial") +_create_partial_wrapper = _make_sync_iterable_wrapper("instructor.create_partial", final_only=True) _create_iterable_wrapper = _make_sync_iterable_wrapper("instructor.create_iterable") _async_create_wrapper = _make_async_wrapper("instructor.create") _async_create_with_completion_wrapper = _make_async_wrapper("instructor.create_with_completion") -_async_create_partial_wrapper = _make_async_iterable_wrapper("instructor.create_partial") +_async_create_partial_wrapper = _make_async_iterable_wrapper("instructor.create_partial", final_only=True) _async_create_iterable_wrapper = _make_async_iterable_wrapper("instructor.create_iterable")