diff --git a/.changeset/wise-sheep-hear.md b/.changeset/wise-sheep-hear.md new file mode 100644 index 000000000..33fc0f9be --- /dev/null +++ b/.changeset/wise-sheep-hear.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(ai-sdk): Trace code mode diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index f6a06afae..69f7bfe12 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -622,11 +622,11 @@ }, { "variantKey": "ai-sdk-v7", - "label": "v7 pinned (including evaluate)" + "label": "v7 pinned" }, { "variantKey": "ai-sdk-v7-latest", - "label": "v7 latest (including decide)" + "label": "v7 latest" } ] }, diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7-latest.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7-latest.cassette.json index 007df2780..b76952bec 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7-latest.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7-latest.cassette.json @@ -4594,6 +4594,417 @@ }, "status": 200 } + }, + { + "callIndex": 0, + "id": "e57f1128145d3780", + "matchKey": "POST api.openai.com/v1/responses", + "recordedAt": "2026-10-18T16:46:22.409Z", + "request": { + "body": { + "kind": "json", + "value": { + "input": [ + { + "content": [ + { + "text": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 120, + "model": "gpt-4.1-nano", + "tool_choice": { + "name": "code", + "type": "function" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "name": "code", + "parameters": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": ["js"], + "type": "object" + }, + "strict": false, + "type": "function" + }, + { + "name": "host", + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/responses" + }, + "response": { + "body": { + "kind": "json", + "value": { + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792341982, + "created_at": 1792341981, + "error": null, + "frequency_penalty": 0, + "id": "resp_038f6758e4471371006ad4f7ddcfd087d28f3a3d5669f549cd", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 120, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4.1-nano-2025-04-14", + "moderation": null, + "object": "response", + "output": [ + { + "arguments": "{\"js\":\"return await tools.host({value: 6});\"}", + "call_id": "call_w8gnHKlZG0WX33Jv3OJrQHzo", + "id": "fc_038f6758e4471371006ad4f7de1f7c87d2a1be3071cefbb7b8", + "name": "code", + "status": "completed", + "type": "function_call" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "name": "code", + "type": "function" + }, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "name": "code", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": ["js"], + "type": "object" + }, + "strict": false, + "type": "function" + }, + { + "description": null, + "name": "host", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "user": null + } + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a4c904c99c8ec799-CDG", + "content-encoding": "gzip", + "content-type": "application/json", + "date": "Sun, 18 Oct 2026 16:46:22 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "519", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999545", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_0a4a917d777044e499b1650c7580520d" + }, + "status": 200 + } + }, + { + "callIndex": 1, + "id": "9c1a26e35ca50c0a", + "matchKey": "POST api.openai.com/v1/responses", + "recordedAt": "2026-10-18T16:46:23.376Z", + "request": { + "body": { + "kind": "json", + "value": { + "input": [ + { + "content": [ + { + "text": "Call host with value 8.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 80, + "model": "gpt-4.1-nano", + "tool_choice": { + "name": "host", + "type": "function" + }, + "tools": [ + { + "name": "host", + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/responses" + }, + "response": { + "body": { + "kind": "json", + "value": { + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792341982, + "created_at": 1792341982, + "error": null, + "frequency_penalty": 0, + "id": "resp_0961ece96c334425006ad4f7de8f1c87d2a90e7ff34d8f8512", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 80, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4.1-nano-2025-04-14", + "moderation": null, + "object": "response", + "output": [ + { + "arguments": "{\"value\":8}", + "call_id": "call_QzSMISGv66wDZomsxmeGHuj0", + "id": "fc_0961ece96c334425006ad4f7ded9e887d293bf8e0899d775ea", + "name": "host", + "status": "completed", + "type": "function_call" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "name": "host", + "type": "function" + }, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [ + { + "description": null, + "name": "host", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "user": null + } + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a4c904ce484ac799-CDG", + "content-encoding": "gzip", + "content-type": "application/json", + "date": "Sun, 18 Oct 2026 16:46:23 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "739", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29998", + "x-ratelimit-remaining-tokens": "149999267", + "x-ratelimit-reset-requests": "4ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_ba6e575047d24e1287b4aeba2826165d" + }, + "status": 200 + } } ], "meta": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7.cassette.json index c28412424..f1b3a7c25 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v7.cassette.json @@ -5089,6 +5089,417 @@ }, "status": 200 } + }, + { + "callIndex": 0, + "id": "e57f1128145d3780", + "matchKey": "POST api.openai.com/v1/responses", + "recordedAt": "2026-10-18T16:45:50.211Z", + "request": { + "body": { + "kind": "json", + "value": { + "input": [ + { + "content": [ + { + "text": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 120, + "model": "gpt-4.1-nano", + "tool_choice": { + "name": "code", + "type": "function" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "name": "code", + "parameters": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": ["js"], + "type": "object" + }, + "strict": false, + "type": "function" + }, + { + "name": "host", + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/responses" + }, + "response": { + "body": { + "kind": "json", + "value": { + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792341950, + "created_at": 1792341949, + "error": null, + "frequency_penalty": 0, + "id": "resp_0373705039ac686f006ad4f7bd7e9087d2afdd784f2cf20c95", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 120, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4.1-nano-2025-04-14", + "moderation": null, + "object": "response", + "output": [ + { + "arguments": "{\"js\":\"return await tools.host({value: 6});\"}", + "call_id": "call_n1c7KKF3Fu15k8cAoBTu6HFV", + "id": "fc_0373705039ac686f006ad4f7bddbf887d2a0465a5836c934e0", + "name": "code", + "status": "completed", + "type": "function_call" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "name": "code", + "type": "function" + }, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "name": "code", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": ["js"], + "type": "object" + }, + "strict": false, + "type": "function" + }, + { + "description": null, + "name": "host", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "user": null + } + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a4c903ff8e0641da-CDG", + "content-encoding": "gzip", + "content-type": "application/json", + "date": "Sun, 18 Oct 2026 16:45:50 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "637", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999545", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_f0911934c00548c9ae2705dd3fc264c2" + }, + "status": 200 + } + }, + { + "callIndex": 1, + "id": "9c1a26e35ca50c0a", + "matchKey": "POST api.openai.com/v1/responses", + "recordedAt": "2026-10-18T16:45:50.871Z", + "request": { + "body": { + "kind": "json", + "value": { + "input": [ + { + "content": [ + { + "text": "Call host with value 8.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 80, + "model": "gpt-4.1-nano", + "tool_choice": { + "name": "host", + "type": "function" + }, + "tools": [ + { + "name": "host", + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.openai.com/v1/responses" + }, + "response": { + "body": { + "kind": "json", + "value": { + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1792341950, + "created_at": 1792341950, + "error": null, + "frequency_penalty": 0, + "id": "resp_090e29521c0692b0006ad4f7be5a7087d29bcb12803b222c48", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 80, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4.1-nano-2025-04-14", + "moderation": null, + "object": "response", + "output": [ + { + "arguments": "{\"value\":8}", + "call_id": "call_OuYX2gwPvBeu86HKruD54Hwx", + "id": "fc_090e29521c0692b0006ad4f7bea94c87d28b3c02ccb55447c1", + "name": "host", + "status": "completed", + "type": "function_call" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "name": "host", + "type": "function" + }, + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [ + { + "description": null, + "name": "host", + "output_schema": null, + "parameters": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": ["value"], + "type": "object" + }, + "strict": false, + "type": "function" + } + ], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "user": null + } + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "a4c90405198141da-CDG", + "content-encoding": "gzip", + "content-type": "application/json", + "date": "Sun, 18 Oct 2026 16:45:50 GMT", + "openai-organization": "braintrust-data", + "openai-processing-ms": "448", + "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "[REDACTED]", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29998", + "x-ratelimit-remaining-tokens": "149999267", + "x-ratelimit-reset-requests": "4ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a586d1797c664d1cba854701bdec0be1" + }, + "status": 200 + } } ], "meta": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json index af9b827b0..6f084756d 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.json @@ -9644,6 +9644,2830 @@ "operation": "workflow-agent-stream-prompt", "testRunId": "" } + }, + { + "name": "ai-sdk-code-mode-auto-hook-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-auto-hook-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-auto-hook-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-auto-hook-direct", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-factory-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 7 + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 7});" + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-factory", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-generation-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 120, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "inputSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + }, + "name": "code", + "type": "function" + }, + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 240, + "tokens": 255 + } + }, + { + "name": "code", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 6 + }, + "output": { + "value": 12 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "code" + } + } + ], + "input": { + "maxOutputTokens": 120, + "maxRetries": 2, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "code", + "type": "tool" + }, + "tools": { + "code": { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + } + } + }, + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 80, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 6, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "tokens": 51 + } + }, + { + "name": "host", + "type": "tool", + "children": [], + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "host" + } + } + ], + "input": { + "maxOutputTokens": 80, + "maxRetries": 2, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-generation", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-combined-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-combined-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-combined-direct", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-factory-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 7 + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 7});" + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-factory", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-generation-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 120, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "inputSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + }, + "name": "code", + "type": "function" + }, + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 240, + "tokens": 255 + } + }, + { + "name": "code", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 6 + }, + "output": { + "value": 12 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "code" + } + } + ], + "input": { + "maxOutputTokens": 120, + "maxRetries": 2, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "code", + "type": "tool" + }, + "tools": { + "code": { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + } + } + }, + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 80, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 6, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "tokens": 51 + } + }, + { + "name": "host", + "type": "tool", + "children": [], + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "host" + } + } + ], + "input": { + "maxOutputTokens": 80, + "maxRetries": 2, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "code-mode-combined-generation", + "testRunId": "" + } } ], "metadata": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt index 0e2905a27..a1158281c 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-auto-hook.span-tree.txt @@ -8291,499 +8291,3361 @@ span_tree: │ metrics: { │ "duration_ms": 0 │ } - └── ai-sdk-workflow-agent-stream-prompt-operation + ├── ai-sdk-workflow-agent-stream-prompt-operation + │ metadata: { + │ "operation": "workflow-agent-stream-prompt", + │ "testRunId": "" + │ } + │ └── WorkflowAgent.stream [function] + │ input: { + │ "messages": [ + │ { + │ "content": "What's the weather in Paris?", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 70 + │ }, + │ "inputTokens": 70, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 70, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 78 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 1, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 109 + │ }, + │ "inputTokens": 109, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 109, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 117 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 2, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 148 + │ }, + │ "inputTokens": 148, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 148, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 156 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 3, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 187 + │ }, + │ "inputTokens": 187, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 187, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 195 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [], + │ "totalUsage": { + │ "inputTokens": 514, + │ "outputTokens": 32, + │ "totalTokens": 546 + │ }, + │ "usage": { + │ "inputTokens": 514, + │ "outputTokens": 32, + │ "totalTokens": 546 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": { + │ "maxOutputTokens": 96, + │ "maxRetries": 2, + │ "temperature": 0, + │ "toolChoice": "required" + │ }, + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 70 + │ │ }, + │ │ "inputTokens": 70, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 8 + │ │ }, + │ │ "outputTokens": 8, + │ │ "raw": { + │ │ "input_tokens": 70, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 78 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0 + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 70, + │ │ "tokens": 78 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather" + │ │ } + │ │ metrics: { + │ │ "duration_ms": 0 + │ │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 109 + │ │ }, + │ │ "inputTokens": 109, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 8 + │ │ }, + │ │ "outputTokens": 8, + │ │ "raw": { + │ │ "input_tokens": 109, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 117 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0 + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 109, + │ │ "tokens": 117 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather" + │ │ } + │ │ metrics: { + │ │ "duration_ms": 0 + │ │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 148 + │ │ }, + │ │ "inputTokens": 148, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 8 + │ │ }, + │ │ "outputTokens": 8, + │ │ "raw": { + │ │ "input_tokens": 148, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 156 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0 + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 148, + │ │ "tokens": 156 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather" + │ │ } + │ │ metrics: { + │ │ "duration_ms": 0 + │ │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 187 + │ │ }, + │ │ "inputTokens": 187, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 8 + │ │ }, + │ │ "outputTokens": 8, + │ │ "raw": { + │ │ "input_tokens": 187, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "totalTokens": 195 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0 + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 187, + │ │ "tokens": 195 + │ │ } + │ └── get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather" + │ } + │ metrics: { + │ "duration_ms": 0 + │ } + ├── ai-sdk-code-mode-auto-hook-sequential-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-parallel-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-nested-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-host-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-auto-hook-code-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-auto-hook-caught-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-auto-hook-stream-output-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-concurrent-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-direct-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-direct", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return 42;" + │ } + │ output: 42 + ├── ai-sdk-code-mode-auto-hook-factory-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-factory", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 7});" + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 7 + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-generation-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-generation", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 120, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "toolChoice": { + │ │ "toolName": "code", + │ │ "type": "tool" + │ │ }, + │ │ "tools": { + │ │ "code": { + │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + │ │ "inputSchema": { + │ │ "jsonSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "js": { + │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "js" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ }, + │ │ "host": { + │ │ "inputSchema": { + │ │ "jsonSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ }, + │ │ "label": "original receiver" + │ │ } + │ │ } + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "value": 12 + │ │ } + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4.1-nano-2025-04-14", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "value": 12 + │ │ } + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4.1-nano-2025-04-14", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "toolResults": [ + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ ├── doGenerate [llm] + │ │ │ input: { + │ │ │ "maxOutputTokens": 120, + │ │ │ "messages": [ + │ │ │ { + │ │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ │ "role": "user" + │ │ │ } + │ │ │ ], + │ │ │ "model": { + │ │ │ "modelId": "gpt-4.1-nano", + │ │ │ "provider": "openai.responses" + │ │ │ }, + │ │ │ "tools": [ + │ │ │ { + │ │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + │ │ │ "inputSchema": { + │ │ │ "additionalProperties": false, + │ │ │ "properties": { + │ │ │ "js": { + │ │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ │ "type": "string" + │ │ │ } + │ │ │ }, + │ │ │ "required": [ + │ │ │ "js" + │ │ │ ], + │ │ │ "type": "object" + │ │ │ }, + │ │ │ "name": "code", + │ │ │ "type": "function" + │ │ │ }, + │ │ │ { + │ │ │ "inputSchema": { + │ │ │ "additionalProperties": false, + │ │ │ "properties": { + │ │ │ "value": { + │ │ │ "type": "number" + │ │ │ } + │ │ │ }, + │ │ │ "required": [ + │ │ │ "value" + │ │ │ ], + │ │ │ "type": "object" + │ │ │ }, + │ │ │ "name": "host", + │ │ │ "type": "function" + │ │ │ } + │ │ │ ] + │ │ │ } + │ │ │ output: { + │ │ │ "content": [ + │ │ │ { + │ │ │ "input": { + │ │ │ "js": "return await tools.host({value: 6});" + │ │ │ }, + │ │ │ "providerMetadata": { + │ │ │ "openai": { + │ │ │ "itemId": "" + │ │ │ } + │ │ │ }, + │ │ │ "toolCallId": "", + │ │ │ "toolName": "code", + │ │ │ "type": "tool-call" + │ │ │ } + │ │ │ ], + │ │ │ "finishReason": "tool-calls", + │ │ │ "providerMetadata": { + │ │ │ "openai": { + │ │ │ "responseId": "", + │ │ │ "serviceTier": "default" + │ │ │ } + │ │ │ }, + │ │ │ "response": {}, + │ │ │ "usage": { + │ │ │ "inputTokenDetails": { + │ │ │ "cacheReadTokens": 0, + │ │ │ "cacheWriteTokens": 0, + │ │ │ "noCacheTokens": 240 + │ │ │ }, + │ │ │ "inputTokens": 240, + │ │ │ "outputTokenDetails": { + │ │ │ "reasoningTokens": 0, + │ │ │ "textTokens": 15 + │ │ │ }, + │ │ │ "outputTokens": 15, + │ │ │ "raw": { + │ │ │ "input_tokens": 240, + │ │ │ "input_tokens_details": { + │ │ │ "cache_write_tokens": 0, + │ │ │ "cached_tokens": 0 + │ │ │ }, + │ │ │ "output_tokens": 15, + │ │ │ "output_tokens_details": { + │ │ │ "reasoning_tokens": 0 + │ │ │ }, + │ │ │ "total_tokens": 255 + │ │ │ }, + │ │ │ "totalTokens": 255 + │ │ │ } + │ │ │ } + │ │ │ metadata: { + │ │ │ "braintrust": { + │ │ │ "integration_name": "ai-sdk", + │ │ │ "sdk_language": "typescript" + │ │ │ }, + │ │ │ "model": "gpt-4.1-nano", + │ │ │ "provider": "openai.responses" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 15, + │ │ │ "prompt_cache_creation_tokens": 0, + │ │ │ "prompt_cached_tokens": 0, + │ │ │ "prompt_tokens": 240, + │ │ │ "tokens": 255 + │ │ │ } + │ │ └── code [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 6});" + │ │ } + │ │ output: { + │ │ "value": 12 + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 6 + │ │ } + │ │ output: { + │ │ "value": 12 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 80, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "Call host with value 8.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "toolChoice": { + │ "toolName": "host", + │ "type": "tool" + │ }, + │ "tools": { + │ "host": { + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "value": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "value" + │ ], + │ "type": "object" + │ } + │ }, + │ "label": "original receiver" + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 16 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 16 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 80, + │ │ "messages": [ + │ │ { + │ │ "content": "Call host with value 8.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "host", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "value": 8 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "host", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 45 + │ │ }, + │ │ "inputTokens": 45, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 6 + │ │ }, + │ │ "outputTokens": 6, + │ │ "raw": { + │ │ "input_tokens": 45, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 6, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 51 + │ │ }, + │ │ "totalTokens": 51 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 6, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 45, + │ │ "tokens": 51 + │ │ } + │ └── host [tool] + │ input: { + │ "value": 8 + │ } + │ output: { + │ "value": 16 + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "toolCallId": "", + │ "toolName": "host" + │ } + ├── ai-sdk-code-mode-combined-sequential-operation + │ metadata: { + │ "operation": "code-mode-combined-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-parallel-operation + │ metadata: { + │ "operation": "code-mode-combined-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-nested-operation + │ metadata: { + │ "operation": "code-mode-combined-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-host-error-operation + │ metadata: { + │ "operation": "code-mode-combined-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-combined-code-error-operation + │ metadata: { + │ "operation": "code-mode-combined-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-combined-caught-error-operation + │ metadata: { + │ "operation": "code-mode-combined-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-combined-stream-output-operation + │ metadata: { + │ "operation": "code-mode-combined-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-concurrent-operation + │ metadata: { + │ "operation": "code-mode-combined-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-direct-operation + │ metadata: { + │ "operation": "code-mode-combined-direct", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return 42;" + │ } + │ output: 42 + ├── ai-sdk-code-mode-combined-factory-operation + │ metadata: { + │ "operation": "code-mode-combined-factory", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 7});" + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 7 + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + └── ai-sdk-code-mode-combined-generation-operation metadata: { - "operation": "workflow-agent-stream-prompt", + "operation": "code-mode-combined-generation", "testRunId": "" } - └── WorkflowAgent.stream [function] + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 120, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "toolChoice": { + │ "toolName": "code", + │ "type": "tool" + │ }, + │ "tools": { + │ "code": { + │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "js": { + │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "js" + │ ], + │ "type": "object" + │ } + │ } + │ }, + │ "host": { + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "value": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "value" + │ ], + │ "type": "object" + │ } + │ }, + │ "label": "original receiver" + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 12 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 12 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 120, + │ │ "messages": [ + │ │ { + │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "js": { + │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "js" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "code", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "host", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 15, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 240, + │ │ "tokens": 255 + │ │ } + │ └── code [tool] + │ input: { + │ "js": "return await tools.host({value: 6});" + │ } + │ output: { + │ "value": 12 + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "toolCallId": "", + │ "toolName": "code" + │ } + │ └── host [function] + │ input: { + │ "value": 6 + │ } + │ output: { + │ "value": 12 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + └── generateText [function] input: { + "maxOutputTokens": 80, + "maxRetries": 2, "messages": [ { - "content": "What's the weather in Paris?", + "content": "Call host with value 8.", "role": "user" } - ] + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } } output: { "content": [ { "input": { - "location": "Paris, France" + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } }, - "toolCallId": "", - "toolName": "get_weather", + "toolCallId": "", + "toolName": "host", "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" } ], "finishReason": "tool-calls", "providerMetadata": { "openai": { - "responseId": "", + "responseId": "", "serviceTier": "default" } }, - "request": { - "messages": [] - }, "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", "timestamp": "" }, + "responseMessages": "", "steps": [ { "callId": "", "content": [ { "input": { - "location": "Paris, France" + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } }, "toolCallId": "", - "toolName": "get_weather", + "toolName": "host", "type": "tool-call" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 0, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 70 - }, - "inputTokens": 70, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 70, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } }, - "totalTokens": 78 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ { + "dynamic": false, "input": { - "location": "Paris, France" + "value": 8 }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 1, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 109 - }, - "inputTokens": 109, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 109, - "input_tokens_details": { - "cached_tokens": 0 + "output": { + "value": 16 }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 117 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" + "providerMetadata": { + "openai": { + "itemId": "" + } }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" + "toolCallId": "", + "toolName": "host", + "type": "tool-result" } ], "finishReason": "tool-calls", "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" }, "providerMetadata": { "openai": { - "responseId": "", + "responseId": "", "serviceTier": "default" } }, - "request": { - "messages": [] - }, "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", "timestamp": "" }, "runtimeContext": {}, - "stepNumber": 2, + "stepNumber": 0, "toolsContext": {}, "usage": { "inputTokenDetails": { "cacheReadTokens": 0, - "noCacheTokens": 148 + "cacheWriteTokens": 0, + "noCacheTokens": 45 }, - "inputTokens": 148, + "inputTokens": 45, "outputTokenDetails": { "reasoningTokens": 0, - "textTokens": 8 + "textTokens": 6 }, - "outputTokens": 8, + "outputTokens": 6, "raw": { - "input_tokens": 148, + "input_tokens": 45, "input_tokens_details": { + "cache_write_tokens": 0, "cached_tokens": 0 }, - "output_tokens": 8, + "output_tokens": 6, "output_tokens_details": { "reasoning_tokens": 0 - } + }, + "total_tokens": 51 }, - "totalTokens": 156 + "totalTokens": 51 }, "warnings": [] - }, + } + ], + "text": "", + "toolCalls": [ { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" + "input": { + "value": 8 }, "providerMetadata": { "openai": { - "responseId": "", - "serviceTier": "default" + "itemId": "" } }, - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 3, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 187 - }, - "inputTokens": 187, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 187, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 195 - }, - "warnings": [] + "toolCallId": "", + "toolName": "host", + "type": "tool-call" } ], - "text": "", - "toolCalls": [ + "toolResults": [ { + "dynamic": false, "input": { - "location": "Paris, France" + "value": 8 }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" } ], - "toolResults": [], "totalUsage": { - "inputTokens": 514, - "outputTokens": 32, - "totalTokens": 546 + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 }, "usage": { - "inputTokens": 514, - "outputTokens": 32, - "totalTokens": 546 + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 }, "warnings": [] } - metadata: { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "maxOutputTokens": 96, - "maxRetries": 2, - "temperature": 0, - "toolChoice": "required" - }, - "provider": "openai.responses", - "tools": { - "get_weather": { - "description": "Get the weather for a location", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - } - } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" } ├── doGenerate [llm] │ input: { + │ "maxOutputTokens": 80, │ "messages": [ │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], + │ "content": "Call host with value 8.", │ "role": "user" │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } │ ], - │ "finishReason": "tool-calls", - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "noCacheTokens": 70 - │ }, - │ "inputTokens": 70, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 8 - │ }, - │ "outputTokens": 8, - │ "raw": { - │ "input_tokens": 70, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ }, - │ "totalTokens": 78 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "maxOutputTokens": 96, - │ "temperature": 0 + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" │ }, - │ "provider": "openai.responses", │ "tools": [ │ { - │ "description": "Get the weather for a location", │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", │ "additionalProperties": false, │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" + │ "value": { + │ "type": "number" │ } │ }, │ "required": [ - │ "location" + │ "value" │ ], │ "type": "object" - │ } - │ } - │ ] - │ } - │ metrics: { - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 70, - │ "tokens": 78 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" + │ }, + │ "name": "host", + │ "type": "function" │ } │ ] │ } @@ -8791,416 +11653,51 @@ span_tree: │ "content": [ │ { │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "finishReason": "tool-calls", - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "noCacheTokens": 109 - │ }, - │ "inputTokens": 109, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 8 - │ }, - │ "outputTokens": 8, - │ "raw": { - │ "input_tokens": 109, - │ "input_tokens_details": { - │ "cached_tokens": 0 + │ "value": 8 │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ }, - │ "totalTokens": 117 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "maxOutputTokens": 96, - │ "temperature": 0 - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ } - │ ] - │ } - │ metrics: { - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 109, - │ "tokens": 117 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", + │ "toolCallId": "", + │ "toolName": "host", │ "type": "tool-call" │ } │ ], │ "finishReason": "tool-calls", - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "noCacheTokens": 148 - │ }, - │ "inputTokens": 148, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 8 - │ }, - │ "outputTokens": 8, - │ "raw": { - │ "input_tokens": 148, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ }, - │ "totalTokens": 156 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "maxOutputTokens": 96, - │ "temperature": 0 - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" │ } - │ ] - │ } - │ metrics: { - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 148, - │ "tokens": 156 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" │ }, - │ "toolCallId": "", - │ "toolName": "get_weather" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── doGenerate [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "finishReason": "tool-calls", │ "response": {}, │ "usage": { │ "inputTokenDetails": { │ "cacheReadTokens": 0, - │ "noCacheTokens": 187 + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 │ }, - │ "inputTokens": 187, + │ "inputTokens": 45, │ "outputTokenDetails": { │ "reasoningTokens": 0, - │ "textTokens": 8 + │ "textTokens": 6 │ }, - │ "outputTokens": 8, + │ "outputTokens": 6, │ "raw": { - │ "input_tokens": 187, + │ "input_tokens": 45, │ "input_tokens_details": { + │ "cache_write_tokens": 0, │ "cached_tokens": 0 │ }, - │ "output_tokens": 8, + │ "output_tokens": 6, │ "output_tokens_details": { │ "reasoning_tokens": 0 - │ } + │ }, + │ "total_tokens": 51 │ }, - │ "totalTokens": 195 + │ "totalTokens": 51 │ } │ } │ metadata: { @@ -9208,46 +11705,22 @@ span_tree: │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "maxOutputTokens": 96, - │ "temperature": 0 - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ } - │ ] + │ "model": "gpt-4.1-nano", + │ "provider": "openai.responses" │ } │ metrics: { - │ "completion_tokens": 8, + │ "completion_tokens": 6, + │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 187, - │ "tokens": 195 + │ "prompt_tokens": 45, + │ "tokens": 51 │ } - └── get_weather [tool] + └── host [tool] input: { - "location": "Paris, France" + "value": 8 } output: { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 + "value": 16 } metadata: { "braintrust": { @@ -9255,8 +11728,5 @@ span_tree: "sdk_language": "typescript" }, "toolCallId": "", - "toolName": "get_weather" - } - metrics: { - "duration_ms": 0 + "toolName": "host" } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json index 0cdad11a1..477a35876 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.json @@ -10080,6 +10080,380 @@ "operation": "workflow-agent-stream-prompt", "testRunId": "" } + }, + { + "name": "ai-sdk-code-mode-wrapped-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-wrapped-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-wrapped-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-wrapped-direct", + "testRunId": "" + } } ], "metadata": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt index 45fd47c40..7ff01c5e2 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-explicit.span-tree.txt @@ -8511,1188 +8511,1433 @@ span_tree: │ metadata: { │ "toolCallId": "" │ } - └── ai-sdk-workflow-agent-stream-prompt-operation + ├── ai-sdk-workflow-agent-stream-prompt-operation + │ metadata: { + │ "operation": "workflow-agent-stream-prompt", + │ "testRunId": "" + │ } + │ └── WorkflowAgent.stream [function] + │ input: { + │ "prompt": "What's the weather in Paris?", + │ "system": "You are a helpful weather assistant." + │ } + │ output: { + │ "messages": [ + │ { + │ "content": "You are a helpful weather assistant.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "What's the weather in Paris?", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "dynamicToolCalls": [], + │ "dynamicToolResults": [], + │ "files": [], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "reasoning": [], + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "sources": [], + │ "staticToolCalls": [], + │ "staticToolResults": [], + │ "stepNumber": 0, + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [], + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 70 + │ }, + │ "inputTokens": 70, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 70, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 78 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "dynamicToolCalls": [], + │ "dynamicToolResults": [], + │ "files": [], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "reasoning": [], + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "sources": [], + │ "staticToolCalls": [], + │ "staticToolResults": [], + │ "stepNumber": 1, + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [], + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 109 + │ }, + │ "inputTokens": 109, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 109, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 117 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "dynamicToolCalls": [], + │ "dynamicToolResults": [], + │ "files": [], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "reasoning": [], + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "sources": [], + │ "staticToolCalls": [], + │ "staticToolResults": [], + │ "stepNumber": 2, + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [], + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 148 + │ }, + │ "inputTokens": 148, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 148, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 156 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "dynamicToolCalls": [], + │ "dynamicToolResults": [], + │ "files": [], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "gpt-4o-mini-2024-07-18" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "reasoning": [], + │ "request": { + │ "messages": [] + │ }, + │ "response": { + │ "messages": [], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "sources": [], + │ "staticToolCalls": [], + │ "staticToolResults": [], + │ "stepNumber": 3, + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [], + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 187 + │ }, + │ "inputTokens": 187, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 187, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "totalTokens": 195 + │ }, + │ "warnings": [] + │ } + │ ], + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": { + │ "maxOutputTokens": 96, + │ "stopWhen": "[Function]", + │ "temperature": 0, + │ "toolChoice": "required" + │ }, + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doStream [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 70, + │ │ "total": 70 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 70, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "includeRawChunks": false, + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ } + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 70, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 78 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ ├── doStream [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 109, + │ │ "total": 109 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 109, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "includeRawChunks": false, + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ } + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 109, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 117 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ ├── doStream [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 148, + │ │ "total": 148 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 148, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "includeRawChunks": false, + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ } + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 148, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 156 + │ │ } + │ ├── get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ ├── doStream [llm] + │ │ input: { + │ │ "messages": [ + │ │ { + │ │ "content": "You are a helpful weather assistant.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "What's the weather in Paris?", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperatureC": 22 + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 187, + │ │ "total": 187 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 187, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "options": { + │ │ "includeRawChunks": false, + │ │ "maxOutputTokens": 96, + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ } + │ │ }, + │ │ "provider": "openai.responses", + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 187, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 195 + │ │ } + │ └── get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperatureC": 22 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-sequential-operation + │ metadata: { + │ "operation": "code-mode-wrapped-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-parallel-operation + │ metadata: { + │ "operation": "code-mode-wrapped-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-nested-operation + │ metadata: { + │ "operation": "code-mode-wrapped-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-host-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-wrapped-code-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-wrapped-caught-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-wrapped-stream-output-operation + │ metadata: { + │ "operation": "code-mode-wrapped-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-concurrent-operation + │ metadata: { + │ "operation": "code-mode-wrapped-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + └── ai-sdk-code-mode-wrapped-direct-operation metadata: { - "operation": "workflow-agent-stream-prompt", + "operation": "code-mode-wrapped-direct", "testRunId": "" } - └── WorkflowAgent.stream [function] + └── codeMode [tool] input: { - "prompt": "What's the weather in Paris?", - "system": "You are a helpful weather assistant." - } - output: { - "messages": [ - { - "content": "You are a helpful weather assistant.", - "role": "system" - }, - { - "content": [ - { - "text": "What's the weather in Paris?", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "json", - "value": { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "json", - "value": { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "json", - "value": { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "json", - "value": { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "steps": [ - { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "dynamicToolCalls": [], - "dynamicToolResults": [], - "files": [], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "reasoning": [], - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "sources": [], - "staticToolCalls": [], - "staticToolResults": [], - "stepNumber": 0, - "text": "", - "toolCalls": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "toolResults": [], - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 70 - }, - "inputTokens": 70, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 70, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 78 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "dynamicToolCalls": [], - "dynamicToolResults": [], - "files": [], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "reasoning": [], - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "sources": [], - "staticToolCalls": [], - "staticToolResults": [], - "stepNumber": 1, - "text": "", - "toolCalls": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "toolResults": [], - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 109 - }, - "inputTokens": 109, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 109, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 117 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "dynamicToolCalls": [], - "dynamicToolResults": [], - "files": [], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "reasoning": [], - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "sources": [], - "staticToolCalls": [], - "staticToolResults": [], - "stepNumber": 2, - "text": "", - "toolCalls": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "toolResults": [], - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 148 - }, - "inputTokens": 148, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 148, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 156 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "dynamicToolCalls": [], - "dynamicToolResults": [], - "files": [], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "gpt-4o-mini-2024-07-18" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "reasoning": [], - "request": { - "messages": [] - }, - "response": { - "messages": [], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "sources": [], - "staticToolCalls": [], - "staticToolResults": [], - "stepNumber": 3, - "text": "", - "toolCalls": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "toolResults": [], - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "noCacheTokens": 187 - }, - "inputTokens": 187, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 8 - }, - "outputTokens": 8, - "raw": { - "input_tokens": 187, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 8, - "output_tokens_details": { - "reasoning_tokens": 0 - } - }, - "totalTokens": 195 - }, - "warnings": [] - } - ], - "toolCalls": [ - { - "input": { - "location": "Paris, France" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "toolResults": [ - { - "input": { - "location": "Paris, France" - }, - "output": { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ] - } - metadata: { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "maxOutputTokens": 96, - "stopWhen": "[Function]", - "temperature": 0, - "toolChoice": "required" - }, - "provider": "openai.responses", - "tools": { - "get_weather": { - "description": "Get the weather for a location", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - } - } + "js": "return 42;" } - ├── doStream [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ } - │ output: { - │ "finishReason": { - │ "unified": "tool-calls" - │ }, - │ "text": "", - │ "toolCalls": [ - │ { - │ "input": "{\"location\":\"Paris, France\"}", - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 0, - │ "noCache": 70, - │ "total": 70 - │ }, - │ "outputTokens": { - │ "reasoning": 0, - │ "text": 8, - │ "total": 8 - │ }, - │ "raw": { - │ "input_tokens": 70, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "finish_reason": { - │ "unified": "tool-calls" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "includeRawChunks": false, - │ "maxOutputTokens": 96, - │ "temperature": 0, - │ "toolChoice": { - │ "type": "required" - │ } - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ }, - │ "name": "get_weather", - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 70, - │ "reasoning_tokens": 0, - │ "tokens": 78 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "toolCallId": "" - │ } - ├── doStream [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ] - │ } - │ output: { - │ "finishReason": { - │ "unified": "tool-calls" - │ }, - │ "text": "", - │ "toolCalls": [ - │ { - │ "input": "{\"location\":\"Paris, France\"}", - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 0, - │ "noCache": 109, - │ "total": 109 - │ }, - │ "outputTokens": { - │ "reasoning": 0, - │ "text": 8, - │ "total": 8 - │ }, - │ "raw": { - │ "input_tokens": 109, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "finish_reason": { - │ "unified": "tool-calls" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "includeRawChunks": false, - │ "maxOutputTokens": 96, - │ "temperature": 0, - │ "toolChoice": { - │ "type": "required" - │ } - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ }, - │ "name": "get_weather", - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 109, - │ "reasoning_tokens": 0, - │ "tokens": 117 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "toolCallId": "" - │ } - ├── doStream [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ] - │ } - │ output: { - │ "finishReason": { - │ "unified": "tool-calls" - │ }, - │ "text": "", - │ "toolCalls": [ - │ { - │ "input": "{\"location\":\"Paris, France\"}", - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 0, - │ "noCache": 148, - │ "total": 148 - │ }, - │ "outputTokens": { - │ "reasoning": 0, - │ "text": 8, - │ "total": 8 - │ }, - │ "raw": { - │ "input_tokens": 148, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "finish_reason": { - │ "unified": "tool-calls" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "includeRawChunks": false, - │ "maxOutputTokens": 96, - │ "temperature": 0, - │ "toolChoice": { - │ "type": "required" - │ } - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ }, - │ "name": "get_weather", - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 148, - │ "reasoning_tokens": 0, - │ "tokens": 156 - │ } - ├── get_weather [tool] - │ input: { - │ "location": "Paris, France" - │ } - │ output: { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ metadata: { - │ "toolCallId": "" - │ } - ├── doStream [llm] - │ input: { - │ "messages": [ - │ { - │ "content": "You are a helpful weather assistant.", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "What's the weather in Paris?", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "location": "Paris, France" - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "json", - │ "value": { - │ "condition": "sunny", - │ "location": "Paris, France", - │ "temperatureC": 22 - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ] - │ } - │ output: { - │ "finishReason": { - │ "unified": "tool-calls" - │ }, - │ "text": "", - │ "toolCalls": [ - │ { - │ "input": "{\"location\":\"Paris, France\"}", - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_weather", - │ "type": "tool-call" - │ } - │ ], - │ "usage": { - │ "inputTokens": { - │ "cacheRead": 0, - │ "noCache": 187, - │ "total": 187 - │ }, - │ "outputTokens": { - │ "reasoning": 0, - │ "text": 8, - │ "total": 8 - │ }, - │ "raw": { - │ "input_tokens": 187, - │ "input_tokens_details": { - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 8, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ } - │ } - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "finish_reason": { - │ "unified": "tool-calls" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "options": { - │ "includeRawChunks": false, - │ "maxOutputTokens": 96, - │ "temperature": 0, - │ "toolChoice": { - │ "type": "required" - │ } - │ }, - │ "provider": "openai.responses", - │ "tools": [ - │ { - │ "description": "Get the weather for a location", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "location": { - │ "description": "The city and country", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ }, - │ "name": "get_weather", - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 8, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 187, - │ "reasoning_tokens": 0, - │ "tokens": 195 - │ } - └── get_weather [tool] - input: { - "location": "Paris, France" - } - output: { - "condition": "sunny", - "location": "Paris, France", - "temperatureC": 22 - } - metadata: { - "toolCallId": "" - } + output: 42 diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json index 0064c6eba..89b7ee922 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.json @@ -8599,6 +8599,2830 @@ "operation": "agent-tool-loop", "testRunId": "" } + }, + { + "name": "ai-sdk-code-mode-auto-hook-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-auto-hook-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-auto-hook-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-auto-hook-direct", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-factory-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 7 + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 7});" + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-factory", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-auto-hook-generation-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 120, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "inputSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + }, + "name": "code", + "type": "function" + }, + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 240, + "tokens": 255 + } + }, + { + "name": "code", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 6 + }, + "output": { + "value": 12 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "code" + } + } + ], + "input": { + "maxOutputTokens": 120, + "maxRetries": 2, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "code", + "type": "tool" + }, + "tools": { + "code": { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + } + } + }, + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 80, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 6, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "tokens": 51 + } + }, + { + "name": "host", + "type": "tool", + "children": [], + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "host" + } + } + ], + "input": { + "maxOutputTokens": 80, + "maxRetries": 2, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "code-mode-auto-hook-generation", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-combined-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-combined-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-combined-direct", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-factory-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 7 + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 7});" + }, + "output": { + "value": 14 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-combined-factory", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-combined-generation-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 120, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + "inputSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + }, + "name": "code", + "type": "function" + }, + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 240, + "tokens": 255 + } + }, + { + "name": "code", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 6 + }, + "output": { + "value": 12 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "code" + } + } + ], + "input": { + "maxOutputTokens": 120, + "maxRetries": 2, + "messages": [ + { + "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "code", + "type": "tool" + }, + "tools": { + "code": { + "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "js": { + "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + "type": "string" + } + }, + "required": [ + "js" + ], + "type": "object" + } + } + }, + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 12 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "js": "return await tools.host({value: 6});" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "js": "return await tools.host({value: 6});" + }, + "output": { + "value": 12 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "code", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 240 + }, + "inputTokens": 240, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 15 + }, + "outputTokens": 15, + "raw": { + "input_tokens": 240, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 15, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 255 + }, + "totalTokens": 255 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "maxOutputTokens": 80, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "tools": [ + { + "inputSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "name": "host", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "metrics": { + "completion_tokens": 6, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "tokens": 51 + } + }, + { + "name": "host", + "type": "tool", + "children": [], + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "toolCallId": "", + "toolName": "host" + } + } + ], + "input": { + "maxOutputTokens": 80, + "maxRetries": 2, + "messages": [ + { + "content": "Call host with value 8.", + "role": "user" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" + } + }, + "label": "original receiver" + } + } + }, + "output": { + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "callId": "", + "content": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "messages": [ + { + "content": [ + { + "input": { + "value": 8 + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4.1-nano-2025-04-14", + "timestamp": "" + }, + "runtimeContext": {}, + "stepNumber": 0, + "toolsContext": {}, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "value": 8 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "value": 8 + }, + "output": { + "value": 16 + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "totalUsage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "usage": { + "inputTokenDetails": { + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4.1-nano", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "code-mode-combined-generation", + "testRunId": "" + } } ], "metadata": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt index f672f7d1e..96713015d 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-auto-hook.span-tree.txt @@ -6914,188 +6914,3452 @@ span_tree: │ "prompt_tokens": 27, │ "tokens": 31 │ } - └── ai-sdk-agent-tool-loop-operation + ├── ai-sdk-agent-tool-loop-operation + │ metadata: { + │ "operation": "agent-tool-loop", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ "maxOutputTokens": 256, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "I need help comparing laptop prices before checkout.", + │ "role": "user" + │ }, + │ { + │ "content": "I can compare store prices and apply a discount code.", + │ "role": "assistant" + │ }, + │ { + │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "temperature": 0, + │ "tools": { + │ "apply_discount": { + │ "description": "Apply a discount code to a total", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "discountCode": { + │ "description": "The discount code to apply", + │ "type": "string" + │ }, + │ "total": { + │ "description": "The original total", + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "discountCode", + │ "total" + │ ], + │ "type": "object" + │ } + │ }, + │ "get_store_price": { + │ "description": "Get the price of an item at a store", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "item": { + │ "description": "The item to price", + │ "type": "string" + │ }, + │ "store": { + │ "description": "The store name", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "item", + │ "store" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ }, + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 173 + │ }, + │ "inputTokens": 173, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 173, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 232 + │ }, + │ "totalTokens": 232 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 1, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 260 + │ }, + │ "inputTokens": 260, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 21 + │ }, + │ "outputTokens": 21, + │ "raw": { + │ "input_tokens": 260, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 21, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 281 + │ }, + │ "totalTokens": 281 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 2, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 305 + │ }, + │ "inputTokens": 305, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 305, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 364 + │ }, + │ "totalTokens": 364 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "toolCalls": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 738 + │ }, + │ "inputTokens": 738, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 139 + │ }, + │ "outputTokens": 139, + │ "totalTokens": 877 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 738 + │ }, + │ "inputTokens": 738, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 139 + │ }, + │ "outputTokens": 139, + │ "totalTokens": 877 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ │ "maxOutputTokens": 256, + │ │ "messages": [ + │ │ { + │ │ "content": "I need help comparing laptop prices before checkout.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "I can compare store prices and apply a discount code.", + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the price of an item at a store", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "item": { + │ │ "description": "The item to price", + │ │ "type": "string" + │ │ }, + │ │ "store": { + │ │ "description": "The store name", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "item", + │ │ "store" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_store_price", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "description": "Apply a discount code to a total", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "discountCode": { + │ │ "description": "The discount code to apply", + │ │ "type": "string" + │ │ }, + │ │ "total": { + │ │ "description": "The original total", + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "discountCode", + │ │ "total" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "apply_discount", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 173 + │ │ }, + │ │ "inputTokens": 173, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 59 + │ │ }, + │ │ "outputTokens": 59, + │ │ "raw": { + │ │ "input_tokens": 173, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 59, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 232 + │ │ }, + │ │ "totalTokens": 232 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 59, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 173, + │ │ "tokens": 232 + │ │ } + │ ├── get_store_price [tool] + │ │ input: { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ } + │ │ output: "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price" + │ │ } + │ ├── get_store_price [tool] + │ │ input: { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ } + │ │ output: "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price" + │ │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ │ "maxOutputTokens": 256, + │ │ "messages": [ + │ │ { + │ │ "content": "I need help comparing laptop prices before checkout.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "I can compare store prices and apply a discount code.", + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-result" + │ │ }, + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the price of an item at a store", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "item": { + │ │ "description": "The item to price", + │ │ "type": "string" + │ │ }, + │ │ "store": { + │ │ "description": "The store name", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "item", + │ │ "store" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_store_price", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "description": "Apply a discount code to a total", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "discountCode": { + │ │ "description": "The discount code to apply", + │ │ "type": "string" + │ │ }, + │ │ "total": { + │ │ "description": "The original total", + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "discountCode", + │ │ "total" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "apply_discount", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "discountCode": "SAVE20", + │ │ "total": 999 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "apply_discount", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 260 + │ │ }, + │ │ "inputTokens": 260, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 21 + │ │ }, + │ │ "outputTokens": 21, + │ │ "raw": { + │ │ "input_tokens": 260, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 21, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 281 + │ │ }, + │ │ "totalTokens": 281 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 21, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 260, + │ │ "tokens": 281 + │ │ } + │ ├── apply_discount [tool] + │ │ input: { + │ │ "discountCode": "SAVE20", + │ │ "total": 999 + │ │ } + │ │ output: "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "apply_discount" + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ "maxOutputTokens": 256, + │ "messages": [ + │ { + │ "content": "I need help comparing laptop prices before checkout.", + │ "role": "user" + │ }, + │ { + │ "content": "I can compare store prices and apply a discount code.", + │ "role": "assistant" + │ }, + │ { + │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the price of an item at a store", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "item": { + │ "description": "The item to price", + │ "type": "string" + │ }, + │ "store": { + │ "description": "The store name", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "item", + │ "store" + │ ], + │ "type": "object" + │ }, + │ "name": "get_store_price", + │ "type": "function" + │ }, + │ { + │ "description": "Apply a discount code to a total", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "discountCode": { + │ "description": "The discount code to apply", + │ "type": "string" + │ }, + │ "total": { + │ "description": "The original total", + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "discountCode", + │ "total" + │ ], + │ "type": "object" + │ }, + │ "name": "apply_discount", + │ "type": "function" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 305 + │ }, + │ "inputTokens": 305, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 305, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 364 + │ }, + │ "totalTokens": 364 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_tokens": 59, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 305, + │ "tokens": 364 + │ } + ├── ai-sdk-code-mode-auto-hook-sequential-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-parallel-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-nested-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-host-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-auto-hook-code-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-auto-hook-caught-error-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-auto-hook-stream-output-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-concurrent-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-direct-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-direct", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return 42;" + │ } + │ output: 42 + ├── ai-sdk-code-mode-auto-hook-factory-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-factory", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 7});" + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 7 + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-auto-hook-generation-operation + │ metadata: { + │ "operation": "code-mode-auto-hook-generation", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 120, + │ │ "maxRetries": 2, + │ │ "messages": [ + │ │ { + │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "toolChoice": { + │ │ "toolName": "code", + │ │ "type": "tool" + │ │ }, + │ │ "tools": { + │ │ "code": { + │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + │ │ "inputSchema": { + │ │ "jsonSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "js": { + │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "js" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ } + │ │ }, + │ │ "host": { + │ │ "inputSchema": { + │ │ "jsonSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ } + │ │ }, + │ │ "label": "original receiver" + │ │ } + │ │ } + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "value": 12 + │ │ } + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4.1-nano-2025-04-14", + │ │ "timestamp": "" + │ │ }, + │ │ "responseMessages": "", + │ │ "steps": [ + │ │ { + │ │ "callId": "", + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "json", + │ │ "value": { + │ │ "value": 12 + │ │ } + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4.1-nano-2025-04-14", + │ │ "timestamp": "" + │ │ }, + │ │ "runtimeContext": {}, + │ │ "stepNumber": 0, + │ │ "toolsContext": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "", + │ │ "toolCalls": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "toolResults": [ + │ │ { + │ │ "dynamic": false, + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "output": { + │ │ "value": 12 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "totalUsage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ ├── doGenerate [llm] + │ │ │ input: { + │ │ │ "maxOutputTokens": 120, + │ │ │ "messages": [ + │ │ │ { + │ │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ │ "role": "user" + │ │ │ } + │ │ │ ], + │ │ │ "model": { + │ │ │ "modelId": "gpt-4.1-nano", + │ │ │ "provider": "openai.responses" + │ │ │ }, + │ │ │ "tools": [ + │ │ │ { + │ │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + │ │ │ "inputSchema": { + │ │ │ "additionalProperties": false, + │ │ │ "properties": { + │ │ │ "js": { + │ │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ │ "type": "string" + │ │ │ } + │ │ │ }, + │ │ │ "required": [ + │ │ │ "js" + │ │ │ ], + │ │ │ "type": "object" + │ │ │ }, + │ │ │ "name": "code", + │ │ │ "type": "function" + │ │ │ }, + │ │ │ { + │ │ │ "inputSchema": { + │ │ │ "additionalProperties": false, + │ │ │ "properties": { + │ │ │ "value": { + │ │ │ "type": "number" + │ │ │ } + │ │ │ }, + │ │ │ "required": [ + │ │ │ "value" + │ │ │ ], + │ │ │ "type": "object" + │ │ │ }, + │ │ │ "name": "host", + │ │ │ "type": "function" + │ │ │ } + │ │ │ ] + │ │ │ } + │ │ │ output: { + │ │ │ "content": [ + │ │ │ { + │ │ │ "input": { + │ │ │ "js": "return await tools.host({value: 6});" + │ │ │ }, + │ │ │ "providerMetadata": { + │ │ │ "openai": { + │ │ │ "itemId": "" + │ │ │ } + │ │ │ }, + │ │ │ "toolCallId": "", + │ │ │ "toolName": "code", + │ │ │ "type": "tool-call" + │ │ │ } + │ │ │ ], + │ │ │ "finishReason": "tool-calls", + │ │ │ "providerMetadata": { + │ │ │ "openai": { + │ │ │ "responseId": "", + │ │ │ "serviceTier": "default" + │ │ │ } + │ │ │ }, + │ │ │ "response": {}, + │ │ │ "usage": { + │ │ │ "inputTokenDetails": { + │ │ │ "cacheReadTokens": 0, + │ │ │ "cacheWriteTokens": 0, + │ │ │ "noCacheTokens": 240 + │ │ │ }, + │ │ │ "inputTokens": 240, + │ │ │ "outputTokenDetails": { + │ │ │ "reasoningTokens": 0, + │ │ │ "textTokens": 15 + │ │ │ }, + │ │ │ "outputTokens": 15, + │ │ │ "raw": { + │ │ │ "input_tokens": 240, + │ │ │ "input_tokens_details": { + │ │ │ "cache_write_tokens": 0, + │ │ │ "cached_tokens": 0 + │ │ │ }, + │ │ │ "output_tokens": 15, + │ │ │ "output_tokens_details": { + │ │ │ "reasoning_tokens": 0 + │ │ │ }, + │ │ │ "total_tokens": 255 + │ │ │ }, + │ │ │ "totalTokens": 255 + │ │ │ } + │ │ │ } + │ │ │ metadata: { + │ │ │ "braintrust": { + │ │ │ "integration_name": "ai-sdk", + │ │ │ "sdk_language": "typescript" + │ │ │ }, + │ │ │ "model": "gpt-4.1-nano", + │ │ │ "provider": "openai.responses" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 15, + │ │ │ "prompt_cache_creation_tokens": 0, + │ │ │ "prompt_cached_tokens": 0, + │ │ │ "prompt_tokens": 240, + │ │ │ "tokens": 255 + │ │ │ } + │ │ └── code [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 6});" + │ │ } + │ │ output: { + │ │ "value": 12 + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 6 + │ │ } + │ │ output: { + │ │ "value": 12 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 80, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "Call host with value 8.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "toolChoice": { + │ "toolName": "host", + │ "type": "tool" + │ }, + │ "tools": { + │ "host": { + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "value": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "value" + │ ], + │ "type": "object" + │ } + │ }, + │ "label": "original receiver" + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 16 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 16 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "value": 8 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "value": 8 + │ }, + │ "output": { + │ "value": 16 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "host", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 51 + │ }, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 80, + │ │ "messages": [ + │ │ { + │ │ "content": "Call host with value 8.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "host", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "value": 8 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "host", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 45 + │ │ }, + │ │ "inputTokens": 45, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 6 + │ │ }, + │ │ "outputTokens": 6, + │ │ "raw": { + │ │ "input_tokens": 45, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 6, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 51 + │ │ }, + │ │ "totalTokens": 51 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 6, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 45, + │ │ "tokens": 51 + │ │ } + │ └── host [tool] + │ input: { + │ "value": 8 + │ } + │ output: { + │ "value": 16 + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "toolCallId": "", + │ "toolName": "host" + │ } + ├── ai-sdk-code-mode-combined-sequential-operation + │ metadata: { + │ "operation": "code-mode-combined-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-parallel-operation + │ metadata: { + │ "operation": "code-mode-combined-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-nested-operation + │ metadata: { + │ "operation": "code-mode-combined-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-host-error-operation + │ metadata: { + │ "operation": "code-mode-combined-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-combined-code-error-operation + │ metadata: { + │ "operation": "code-mode-combined-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-combined-caught-error-operation + │ metadata: { + │ "operation": "code-mode-combined-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-combined-stream-output-operation + │ metadata: { + │ "operation": "code-mode-combined-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-concurrent-operation + │ metadata: { + │ "operation": "code-mode-combined-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-combined-direct-operation + │ metadata: { + │ "operation": "code-mode-combined-direct", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return 42;" + │ } + │ output: 42 + ├── ai-sdk-code-mode-combined-factory-operation + │ metadata: { + │ "operation": "code-mode-combined-factory", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 7});" + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 7 + │ } + │ output: { + │ "value": 14 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + └── ai-sdk-code-mode-combined-generation-operation metadata: { - "operation": "agent-tool-loop", + "operation": "code-mode-combined-generation", "testRunId": "" } + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 120, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "toolChoice": { + │ "toolName": "code", + │ "type": "tool" + │ }, + │ "tools": { + │ "code": { + │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\nNo host tools. Do not call `tools.*`.", + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "js": { + │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "js" + │ ], + │ "type": "object" + │ } + │ } + │ }, + │ "host": { + │ "inputSchema": { + │ "jsonSchema": { + │ "additionalProperties": false, + │ "properties": { + │ "value": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "value" + │ ], + │ "type": "object" + │ } + │ }, + │ "label": "original receiver" + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 12 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "json", + │ "value": { + │ "value": 12 + │ } + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4.1-nano-2025-04-14", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "js": "return await tools.host({value: 6});" + │ }, + │ "output": { + │ "value": 12 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "code", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 240 + │ }, + │ "inputTokens": 240, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 15 + │ }, + │ "outputTokens": 15, + │ "raw": { + │ "input_tokens": 240, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 15, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 255 + │ }, + │ "totalTokens": 255 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4.1-nano", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "maxOutputTokens": 120, + │ │ "messages": [ + │ │ { + │ │ "content": "Call code with exactly this JavaScript: return await tools.host({value: 6});", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Execute code-mode TypeScript in an isolated sandbox.\n\nPut the full program in `js`; top-level `await`/`return` work. Return a JSON-serializable result.\nCall host tools only as async `tools.name(input)`; await each or use `Promise.all` for independent calls.\nUse exact names/types below. `JSON.parse`/`JSON.stringify` are available.\nFetch: `fetch` is not available.\n\nTools:\n```ts\ndeclare const tools: {\n host: (input: { value: number; }) => Promise;\n};\n```\n\nTool call examples:\n```ts\nconst result = await tools.host({\"value\":1});\nreturn result;\n```", + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "js": { + │ │ "description": "Code-mode TypeScript source to execute. The code-mode context lists the available global `tools` API, input types, and call examples.", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "js" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "code", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "inputSchema": { + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "value": { + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "value" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "host", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "js": "return await tools.host({value: 6});" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "code", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 240 + │ │ }, + │ │ "inputTokens": 240, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 15 + │ │ }, + │ │ "outputTokens": 15, + │ │ "raw": { + │ │ "input_tokens": 240, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 15, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 255 + │ │ }, + │ │ "totalTokens": 255 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4.1-nano", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 15, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 240, + │ │ "tokens": 255 + │ │ } + │ └── code [tool] + │ input: { + │ "js": "return await tools.host({value: 6});" + │ } + │ output: { + │ "value": 12 + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "toolCallId": "", + │ "toolName": "code" + │ } + │ └── host [function] + │ input: { + │ "value": 6 + │ } + │ output: { + │ "value": 12 + │ } + │ metadata: { + │ "toolCallId": "" + │ } └── generateText [function] input: { - "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - "maxOutputTokens": 256, + "maxOutputTokens": 80, "maxRetries": 2, "messages": [ { - "content": "I need help comparing laptop prices before checkout.", - "role": "user" - }, - { - "content": "I can compare store prices and apply a discount code.", - "role": "assistant" - }, - { - "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + "content": "Call host with value 8.", "role": "user" - } - ], - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "temperature": 0, - "tools": { - "apply_discount": { - "description": "Apply a discount code to a total", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "discountCode": { - "description": "The discount code to apply", - "type": "string" - }, - "total": { - "description": "The original total", - "type": "number" - } - }, - "required": [ - "discountCode", - "total" - ], - "type": "object" - } - }, - "get_store_price": { - "description": "Get the price of an item at a store", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "item": { - "description": "The item to price", - "type": "string" - }, - "store": { - "description": "The store name", - "type": "string" - } - }, - "required": [ - "item", - "store" - ], - "type": "object" - } - } - } - } - output: { - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreA" - }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" + } + ], + "model": { + "modelId": "gpt-4.1-nano", + "provider": "openai.responses" + }, + "toolChoice": { + "toolName": "host", + "type": "tool" + }, + "tools": { + "host": { + "inputSchema": { + "jsonSchema": { + "additionalProperties": false, + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ], + "type": "object" } }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, + "label": "original receiver" + } + } + } + output: { + "content": [ { "input": { - "discountCode": "SAVE20", - "total": 999 + "value": 8 }, "providerMetadata": { "openai": { - "itemId": "" + "itemId": "" } }, - "toolCallId": "", - "toolName": "apply_discount", + "toolCallId": "", + "toolName": "host", "type": "tool-call" }, { "dynamic": false, "input": { - "discountCode": "SAVE20", - "total": 999 + "value": 8 }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", - "providerMetadata": { - "openai": { - "itemId": "" - } + "output": { + "value": 16 }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - }, - { "providerMetadata": { "openai": { - "itemId": "" + "itemId": "" } }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" + "toolCallId": "", + "toolName": "host", + "type": "tool-result" } ], - "finishReason": "stop", + "finishReason": "tool-calls", "providerMetadata": { "openai": { - "responseId": "", + "responseId": "", "serviceTier": "default" } }, @@ -7104,19 +10368,44 @@ span_tree: { "content": [ { + "input": { + "value": 8 + }, "providerOptions": { "openai": { - "itemId": "" + "itemId": "" } }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" + "toolCallId": "", + "toolName": "host", + "type": "tool-call" } ], "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "json", + "value": { + "value": 16 + } + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "host", + "type": "tool-result" + } + ], + "role": "tool" } ], - "modelId": "gpt-4o-mini-2024-07-18", + "modelId": "gpt-4.1-nano-2025-04-14", "timestamp": "" }, "responseMessages": "", @@ -7126,8 +10415,7 @@ span_tree: "content": [ { "input": { - "item": "laptop", - "store": "StoreA" + "value": 8 }, "providerMetadata": { "openai": { @@ -7135,64 +10423,35 @@ span_tree: } }, "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", + "toolName": "host", "type": "tool-call" }, { "dynamic": false, "input": { - "item": "laptop", - "store": "StoreA" + "value": 8 + }, + "output": { + "value": 16 }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", "providerMetadata": { "openai": { "itemId": "" } }, "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", + "toolName": "host", "type": "tool-result" } ], "finishReason": "tool-calls", "model": { - "modelId": "gpt-4o-mini-2024-07-18", + "modelId": "gpt-4.1-nano", "provider": "openai.responses" }, "providerMetadata": { "openai": { - "responseId": "", + "responseId": "", "serviceTier": "default" } }, @@ -7202,8 +10461,7 @@ span_tree: "content": [ { "input": { - "item": "laptop", - "store": "StoreA" + "value": 8 }, "providerOptions": { "openai": { @@ -7211,21 +10469,7 @@ span_tree: } }, "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", + "toolName": "host", "type": "tool-call" } ], @@ -7235,8 +10479,10 @@ span_tree: "content": [ { "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + "type": "json", + "value": { + "value": 16 + } }, "providerOptions": { "openai": { @@ -7244,28 +10490,14 @@ span_tree: } }, "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", + "toolName": "host", "type": "tool-result" } ], "role": "tool" } ], - "modelId": "gpt-4o-mini-2024-07-18", + "modelId": "gpt-4.1-nano-2025-04-14", "timestamp": "" }, "runtimeContext": {}, @@ -7275,318 +10507,63 @@ span_tree: "inputTokenDetails": { "cacheReadTokens": 0, "cacheWriteTokens": 0, - "noCacheTokens": 173 - }, - "inputTokens": 173, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 59 - }, - "outputTokens": 59, - "raw": { - "input_tokens": 173, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 59, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 232 - }, - "totalTokens": 232 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 1, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 260 - }, - "inputTokens": 260, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 21 - }, - "outputTokens": 21, - "raw": { - "input_tokens": 260, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 21, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 281 - }, - "totalTokens": 281 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "finishReason": "stop", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "role": "assistant" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 2, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 305 + "noCacheTokens": 45 }, - "inputTokens": 305, + "inputTokens": 45, "outputTokenDetails": { "reasoningTokens": 0, - "textTokens": 59 + "textTokens": 6 }, - "outputTokens": 59, + "outputTokens": 6, "raw": { - "input_tokens": 305, + "input_tokens": 45, "input_tokens_details": { "cache_write_tokens": 0, "cached_tokens": 0 }, - "output_tokens": 59, + "output_tokens": 6, "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 364 - }, - "totalTokens": 364 - }, - "warnings": [] - } - ], - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "toolCalls": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerMetadata": { - "openai": { - "itemId": "" - } + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" + "warnings": [] } ], - "toolResults": [ + "text": "", + "toolCalls": [ { - "dynamic": false, "input": { - "item": "laptop", - "store": "StoreA" + "value": 8 }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", "providerMetadata": { "openai": { "itemId": "" } }, "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, + "toolName": "host", + "type": "tool-call" + } + ], + "toolResults": [ { "dynamic": false, "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } + "value": 8 }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "discountCode": "SAVE20", - "total": 999 + "output": { + "value": 16 }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", "providerMetadata": { "openai": { - "itemId": "" + "itemId": "" } }, - "toolCallId": "", - "toolName": "apply_discount", + "toolCallId": "", + "toolName": "host", "type": "tool-result" } ], @@ -7594,29 +10571,53 @@ span_tree: "inputTokenDetails": { "cacheReadTokens": 0, "cacheWriteTokens": 0, - "noCacheTokens": 738 + "noCacheTokens": 45 }, - "inputTokens": 738, + "inputTokens": 45, "outputTokenDetails": { "reasoningTokens": 0, - "textTokens": 139 + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 }, - "outputTokens": 139, - "totalTokens": 877 + "totalTokens": 51 }, "usage": { "inputTokenDetails": { "cacheReadTokens": 0, "cacheWriteTokens": 0, - "noCacheTokens": 738 + "noCacheTokens": 45 }, - "inputTokens": 738, + "inputTokens": 45, "outputTokenDetails": { "reasoningTokens": 0, - "textTokens": 139 + "textTokens": 6 }, - "outputTokens": 139, - "totalTokens": 877 + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 51 + }, + "totalTokens": 51 }, "warnings": [] } @@ -7625,328 +10626,37 @@ span_tree: "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "model": "gpt-4o-mini-2024-07-18", + "model": "gpt-4.1-nano", "provider": "openai.responses" } ├── doGenerate [llm] │ input: { - │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - │ "maxOutputTokens": 256, - │ "messages": [ - │ { - │ "content": "I need help comparing laptop prices before checkout.", - │ "role": "user" - │ }, - │ { - │ "content": "I can compare store prices and apply a discount code.", - │ "role": "assistant" - │ }, - │ { - │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ }, - │ "temperature": 0, - │ "tools": [ - │ { - │ "description": "Get the price of an item at a store", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "item": { - │ "description": "The item to price", - │ "type": "string" - │ }, - │ "store": { - │ "description": "The store name", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "item", - │ "store" - │ ], - │ "type": "object" - │ }, - │ "name": "get_store_price", - │ "type": "function" - │ }, - │ { - │ "description": "Apply a discount code to a total", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "discountCode": { - │ "description": "The discount code to apply", - │ "type": "string" - │ }, - │ "total": { - │ "description": "The original total", - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "discountCode", - │ "total" - │ ], - │ "type": "object" - │ }, - │ "name": "apply_discount", - │ "type": "function" - │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreA" - │ }, - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ }, - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreB" - │ }, - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ } - │ ], - │ "finishReason": "tool-calls", - │ "providerMetadata": { - │ "openai": { - │ "responseId": "", - │ "serviceTier": "default" - │ } - │ }, - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "cacheWriteTokens": 0, - │ "noCacheTokens": 173 - │ }, - │ "inputTokens": 173, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 59 - │ }, - │ "outputTokens": 59, - │ "raw": { - │ "input_tokens": 173, - │ "input_tokens_details": { - │ "cache_write_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 59, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ }, - │ "total_tokens": 232 - │ }, - │ "totalTokens": 232 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ } - │ metrics: { - │ "completion_tokens": 59, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 173, - │ "tokens": 232 - │ } - ├── get_store_price [tool] - │ input: { - │ "item": "laptop", - │ "store": "StoreA" - │ } - │ output: "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price" - │ } - ├── get_store_price [tool] - │ input: { - │ "item": "laptop", - │ "store": "StoreB" - │ } - │ output: "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price" - │ } - ├── doGenerate [llm] - │ input: { - │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - │ "maxOutputTokens": 256, + │ "maxOutputTokens": 80, │ "messages": [ │ { - │ "content": "I need help comparing laptop prices before checkout.", - │ "role": "user" - │ }, - │ { - │ "content": "I can compare store prices and apply a discount code.", - │ "role": "assistant" - │ }, - │ { - │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ "content": "Call host with value 8.", │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreA" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ }, - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreB" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "text", - │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-result" - │ }, - │ { - │ "output": { - │ "type": "text", - │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" │ } │ ], │ "model": { - │ "modelId": "gpt-4o-mini-2024-07-18", + │ "modelId": "gpt-4.1-nano", │ "provider": "openai.responses" │ }, - │ "temperature": 0, │ "tools": [ │ { - │ "description": "Get the price of an item at a store", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "item": { - │ "description": "The item to price", - │ "type": "string" - │ }, - │ "store": { - │ "description": "The store name", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "item", - │ "store" - │ ], - │ "type": "object" - │ }, - │ "name": "get_store_price", - │ "type": "function" - │ }, - │ { - │ "description": "Apply a discount code to a total", │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", │ "additionalProperties": false, │ "properties": { - │ "discountCode": { - │ "description": "The discount code to apply", - │ "type": "string" - │ }, - │ "total": { - │ "description": "The original total", + │ "value": { │ "type": "number" │ } │ }, │ "required": [ - │ "discountCode", - │ "total" + │ "value" │ ], │ "type": "object" │ }, - │ "name": "apply_discount", + │ "name": "host", │ "type": "function" │ } │ ] @@ -7955,8 +10665,7 @@ span_tree: │ "content": [ │ { │ "input": { - │ "discountCode": "SAVE20", - │ "total": 999 + │ "value": 8 │ }, │ "providerMetadata": { │ "openai": { @@ -7964,7 +10673,7 @@ span_tree: │ } │ }, │ "toolCallId": "", - │ "toolName": "apply_discount", + │ "toolName": "host", │ "type": "tool-call" │ } │ ], @@ -7980,27 +10689,27 @@ span_tree: │ "inputTokenDetails": { │ "cacheReadTokens": 0, │ "cacheWriteTokens": 0, - │ "noCacheTokens": 260 + │ "noCacheTokens": 45 │ }, - │ "inputTokens": 260, + │ "inputTokens": 45, │ "outputTokenDetails": { │ "reasoningTokens": 0, - │ "textTokens": 21 + │ "textTokens": 6 │ }, - │ "outputTokens": 21, + │ "outputTokens": 6, │ "raw": { - │ "input_tokens": 260, + │ "input_tokens": 45, │ "input_tokens_details": { │ "cache_write_tokens": 0, │ "cached_tokens": 0 │ }, - │ "output_tokens": 21, + │ "output_tokens": 6, │ "output_tokens_details": { │ "reasoning_tokens": 0 │ }, - │ "total_tokens": 281 + │ "total_tokens": 51 │ }, - │ "totalTokens": 281 + │ "totalTokens": 51 │ } │ } │ metadata: { @@ -8008,267 +10717,28 @@ span_tree: │ "integration_name": "ai-sdk", │ "sdk_language": "typescript" │ }, - │ "model": "gpt-4o-mini-2024-07-18", + │ "model": "gpt-4.1-nano", │ "provider": "openai.responses" │ } │ metrics: { - │ "completion_tokens": 21, + │ "completion_tokens": 6, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 260, - │ "tokens": 281 + │ "prompt_tokens": 45, + │ "tokens": 51 │ } - ├── apply_discount [tool] - │ input: { - │ "discountCode": "SAVE20", - │ "total": 999 - │ } - │ output: "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "apply_discount" - │ } - └── doGenerate [llm] + └── host [tool] input: { - "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - "maxOutputTokens": 256, - "messages": [ - { - "content": "I need help comparing laptop prices before checkout.", - "role": "user" - }, - { - "content": "I can compare store prices and apply a discount code.", - "role": "assistant" - }, - { - "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - "role": "user" - }, - { - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "temperature": 0, - "tools": [ - { - "description": "Get the price of an item at a store", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "item": { - "description": "The item to price", - "type": "string" - }, - "store": { - "description": "The store name", - "type": "string" - } - }, - "required": [ - "item", - "store" - ], - "type": "object" - }, - "name": "get_store_price", - "type": "function" - }, - { - "description": "Apply a discount code to a total", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "discountCode": { - "description": "The discount code to apply", - "type": "string" - }, - "total": { - "description": "The original total", - "type": "number" - } - }, - "required": [ - "discountCode", - "total" - ], - "type": "object" - }, - "name": "apply_discount", - "type": "function" - } - ] + "value": 8 } output: { - "content": [ - { - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "finishReason": "stop", - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 305 - }, - "inputTokens": 305, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 59 - }, - "outputTokens": 59, - "raw": { - "input_tokens": 305, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 59, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 364 - }, - "totalTokens": 364 - } + "value": 16 } metadata: { "braintrust": { "integration_name": "ai-sdk", "sdk_language": "typescript" }, - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - } - metrics: { - "completion_tokens": 59, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 305, - "tokens": 364 + "toolCallId": "", + "toolName": "host" } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json index 0064c6eba..d5fe4c910 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.json @@ -8599,6 +8599,380 @@ "operation": "agent-tool-loop", "testRunId": "" } + }, + { + "name": "ai-sdk-code-mode-wrapped-sequential-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 4 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-sequential", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-parallel-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "output": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "parallel", + "type": "function", + "children": [], + "input": { + "value": 2 + }, + "output": { + "value": 2 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + }, + "output": [ + { + "value": 1 + }, + { + "value": 2 + } + ], + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-parallel", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-nested-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "nested", + "type": "function", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "value": 3 + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.nested({value: 3});" + }, + "output": { + "value": 6 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-nested", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-host-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "return await tools.fail({value: 1});" + }, + "metadata": { + "toolCallId": "" + }, + "error": "Host tool failed." + } + ], + "metadata": { + "operation": "code-mode-wrapped-host-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-code-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "throw new Error(\"code failure\");" + }, + "metadata": { + "toolCallId": "" + }, + "error": "code failure" + } + ], + "metadata": { + "operation": "code-mode-wrapped-code-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-caught-error-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "fail", + "type": "function", + "children": [], + "input": { + "value": 1 + }, + "metadata": { + "toolCallId": "" + }, + "error": "host failure" + } + ], + "input": { + "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + }, + "output": "recovered", + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-caught-error", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-stream-output-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "stream", + "type": "function", + "children": [], + "input": { + "value": 9 + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.stream({value: 9});" + }, + "output": { + "value": 9 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-stream-output", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-concurrent-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 4 + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 4});" + }, + "output": { + "value": 8 + }, + "metadata": { + "toolCallId": "" + } + }, + { + "name": "codeMode", + "type": "tool", + "children": [ + { + "name": "host", + "type": "function", + "children": [], + "input": { + "value": 5 + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "input": { + "js": "return await tools.host({value: 5});" + }, + "output": { + "value": 10 + }, + "metadata": { + "toolCallId": "" + } + } + ], + "metadata": { + "operation": "code-mode-wrapped-concurrent", + "testRunId": "" + } + }, + { + "name": "ai-sdk-code-mode-wrapped-direct-operation", + "children": [ + { + "name": "codeMode", + "type": "tool", + "children": [], + "input": { + "js": "return 42;" + }, + "output": 42 + } + ], + "metadata": { + "operation": "code-mode-wrapped-direct", + "testRunId": "" + } } ], "metadata": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt index f672f7d1e..e027a0353 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v7-latest-explicit.span-tree.txt @@ -6914,1361 +6914,1606 @@ span_tree: │ "prompt_tokens": 27, │ "tokens": 31 │ } - └── ai-sdk-agent-tool-loop-operation + ├── ai-sdk-agent-tool-loop-operation + │ metadata: { + │ "operation": "agent-tool-loop", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ "maxOutputTokens": 256, + │ "maxRetries": 2, + │ "messages": [ + │ { + │ "content": "I need help comparing laptop prices before checkout.", + │ "role": "user" + │ }, + │ { + │ "content": "I can compare store prices and apply a discount code.", + │ "role": "assistant" + │ }, + │ { + │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "temperature": 0, + │ "tools": { + │ "apply_discount": { + │ "description": "Apply a discount code to a total", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "discountCode": { + │ "description": "The discount code to apply", + │ "type": "string" + │ }, + │ "total": { + │ "description": "The original total", + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "discountCode", + │ "total" + │ ], + │ "type": "object" + │ } + │ }, + │ "get_store_price": { + │ "description": "Get the price of an item at a store", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "item": { + │ "description": "The item to price", + │ "type": "string" + │ }, + │ "store": { + │ "description": "The store name", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "item", + │ "store" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ }, + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 0, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 173 + │ }, + │ "inputTokens": 173, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 173, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 232 + │ }, + │ "totalTokens": 232 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 1, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 260 + │ }, + │ "inputTokens": 260, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 21 + │ }, + │ "outputTokens": 21, + │ "raw": { + │ "input_tokens": 260, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 21, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 281 + │ }, + │ "totalTokens": 281 + │ }, + │ "warnings": [] + │ }, + │ { + │ "callId": "", + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "runtimeContext": {}, + │ "stepNumber": 2, + │ "toolsContext": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 305 + │ }, + │ "inputTokens": 305, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 305, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 364 + │ }, + │ "totalTokens": 364 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "toolCalls": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 738 + │ }, + │ "inputTokens": 738, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 139 + │ }, + │ "outputTokens": 139, + │ "totalTokens": 877 + │ }, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 738 + │ }, + │ "inputTokens": 738, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 139 + │ }, + │ "outputTokens": 139, + │ "totalTokens": 877 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ │ "maxOutputTokens": 256, + │ │ "messages": [ + │ │ { + │ │ "content": "I need help comparing laptop prices before checkout.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "I can compare store prices and apply a discount code.", + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the price of an item at a store", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "item": { + │ │ "description": "The item to price", + │ │ "type": "string" + │ │ }, + │ │ "store": { + │ │ "description": "The store name", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "item", + │ │ "store" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_store_price", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "description": "Apply a discount code to a total", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "discountCode": { + │ │ "description": "The discount code to apply", + │ │ "type": "string" + │ │ }, + │ │ "total": { + │ │ "description": "The original total", + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "discountCode", + │ │ "total" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "apply_discount", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 173 + │ │ }, + │ │ "inputTokens": 173, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 59 + │ │ }, + │ │ "outputTokens": 59, + │ │ "raw": { + │ │ "input_tokens": 173, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 59, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 232 + │ │ }, + │ │ "totalTokens": 232 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 59, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 173, + │ │ "tokens": 232 + │ │ } + │ ├── get_store_price [tool] + │ │ input: { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ } + │ │ output: "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price" + │ │ } + │ ├── get_store_price [tool] + │ │ input: { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ } + │ │ output: "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price" + │ │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ │ "maxOutputTokens": 256, + │ │ "messages": [ + │ │ { + │ │ "content": "I need help comparing laptop prices before checkout.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "I can compare store prices and apply a discount code.", + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreA" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ }, + │ │ { + │ │ "input": { + │ │ "item": "laptop", + │ │ "store": "StoreB" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-result" + │ │ }, + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_store_price", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "model": { + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ }, + │ │ "temperature": 0, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the price of an item at a store", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "item": { + │ │ "description": "The item to price", + │ │ "type": "string" + │ │ }, + │ │ "store": { + │ │ "description": "The store name", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "item", + │ │ "store" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_store_price", + │ │ "type": "function" + │ │ }, + │ │ { + │ │ "description": "Apply a discount code to a total", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "discountCode": { + │ │ "description": "The discount code to apply", + │ │ "type": "string" + │ │ }, + │ │ "total": { + │ │ "description": "The original total", + │ │ "type": "number" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "discountCode", + │ │ "total" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "apply_discount", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "discountCode": "SAVE20", + │ │ "total": 999 + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "apply_discount", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": {}, + │ │ "usage": { + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 260 + │ │ }, + │ │ "inputTokens": 260, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 21 + │ │ }, + │ │ "outputTokens": 21, + │ │ "raw": { + │ │ "input_tokens": 260, + │ │ "input_tokens_details": { + │ │ "cache_write_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 21, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ }, + │ │ "total_tokens": 281 + │ │ }, + │ │ "totalTokens": 281 + │ │ } + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 21, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 260, + │ │ "tokens": 281 + │ │ } + │ ├── apply_discount [tool] + │ │ input: { + │ │ "discountCode": "SAVE20", + │ │ "total": 999 + │ │ } + │ │ output: "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "apply_discount" + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", + │ "maxOutputTokens": 256, + │ "messages": [ + │ { + │ "content": "I need help comparing laptop prices before checkout.", + │ "role": "user" + │ }, + │ { + │ "content": "I can compare store prices and apply a discount code.", + │ "role": "assistant" + │ }, + │ { + │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreA" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ }, + │ { + │ "input": { + │ "item": "laptop", + │ "store": "StoreB" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ }, + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_store_price", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "discountCode": "SAVE20", + │ "total": 999 + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "apply_discount", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "model": { + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ }, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the price of an item at a store", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "item": { + │ "description": "The item to price", + │ "type": "string" + │ }, + │ "store": { + │ "description": "The store name", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "item", + │ "store" + │ ], + │ "type": "object" + │ }, + │ "name": "get_store_price", + │ "type": "function" + │ }, + │ { + │ "description": "Apply a discount code to a total", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "discountCode": { + │ "description": "The discount code to apply", + │ "type": "string" + │ }, + │ "total": { + │ "description": "The original total", + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "discountCode", + │ "total" + │ ], + │ "type": "object" + │ }, + │ "name": "apply_discount", + │ "type": "function" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": {}, + │ "usage": { + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 305 + │ }, + │ "inputTokens": 305, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 59 + │ }, + │ "outputTokens": 59, + │ "raw": { + │ "input_tokens": 305, + │ "input_tokens_details": { + │ "cache_write_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 59, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 364 + │ }, + │ "totalTokens": 364 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_tokens": 59, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 305, + │ "tokens": 364 + │ } + ├── ai-sdk-code-mode-wrapped-sequential-operation + │ metadata: { + │ "operation": "code-mode-wrapped-sequential", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "const a = await tools.host({value: 2}); return await tools.host(a);" + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── host [function] + │ │ input: { + │ │ "value": 2 + │ │ } + │ │ output: { + │ │ "value": 4 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── host [function] + │ input: { + │ "value": 4 + │ } + │ output: { + │ "value": 8 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-parallel-operation + │ metadata: { + │ "operation": "code-mode-wrapped-parallel", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);" + │ } + │ output: [ + │ { + │ "value": 1 + │ }, + │ { + │ "value": 2 + │ } + │ ] + │ metadata: { + │ "toolCallId": "" + │ } + │ ├── parallel [function] + │ │ input: { + │ │ "value": 1 + │ │ } + │ │ output: { + │ │ "value": 1 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── parallel [function] + │ input: { + │ "value": 2 + │ } + │ output: { + │ "value": 2 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-nested-operation + │ metadata: { + │ "operation": "code-mode-wrapped-nested", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.nested({value: 3});" + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── nested [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 3 + │ } + │ output: { + │ "value": 6 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-host-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-host-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.fail({value: 1});" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "Host tool failed." + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-wrapped-code-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-code-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "throw new Error(\"code failure\");" + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "code failure" + ├── ai-sdk-code-mode-wrapped-caught-error-operation + │ metadata: { + │ "operation": "code-mode-wrapped-caught-error", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "try { await tools.fail({value: 1}); } catch { return \"recovered\"; }" + │ } + │ output: "recovered" + │ metadata: { + │ "toolCallId": "" + │ } + │ └── fail [function] + │ input: { + │ "value": 1 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ error: "host failure" + ├── ai-sdk-code-mode-wrapped-stream-output-operation + │ metadata: { + │ "operation": "code-mode-wrapped-stream-output", + │ "testRunId": "" + │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.stream({value: 9});" + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── stream [function] + │ input: { + │ "value": 9 + │ } + │ output: { + │ "value": 9 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + ├── ai-sdk-code-mode-wrapped-concurrent-operation + │ metadata: { + │ "operation": "code-mode-wrapped-concurrent", + │ "testRunId": "" + │ } + │ ├── codeMode [tool] + │ │ input: { + │ │ "js": "return await tools.host({value: 4});" + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ │ └── host [function] + │ │ input: { + │ │ "value": 4 + │ │ } + │ │ output: { + │ │ "value": 8 + │ │ } + │ │ metadata: { + │ │ "toolCallId": "" + │ │ } + │ └── codeMode [tool] + │ input: { + │ "js": "return await tools.host({value: 5});" + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + │ └── host [function] + │ input: { + │ "value": 5 + │ } + │ output: { + │ "value": 10 + │ } + │ metadata: { + │ "toolCallId": "" + │ } + └── ai-sdk-code-mode-wrapped-direct-operation metadata: { - "operation": "agent-tool-loop", + "operation": "code-mode-wrapped-direct", "testRunId": "" } - └── generateText [function] + └── codeMode [tool] input: { - "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - "maxOutputTokens": 256, - "maxRetries": 2, - "messages": [ - { - "content": "I need help comparing laptop prices before checkout.", - "role": "user" - }, - { - "content": "I can compare store prices and apply a discount code.", - "role": "assistant" - }, - { - "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - "role": "user" - } - ], - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "temperature": 0, - "tools": { - "apply_discount": { - "description": "Apply a discount code to a total", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "discountCode": { - "description": "The discount code to apply", - "type": "string" - }, - "total": { - "description": "The original total", - "type": "number" - } - }, - "required": [ - "discountCode", - "total" - ], - "type": "object" - } - }, - "get_store_price": { - "description": "Get the price of an item at a store", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "item": { - "description": "The item to price", - "type": "string" - }, - "store": { - "description": "The store name", - "type": "string" - } - }, - "required": [ - "item", - "store" - ], - "type": "object" - } - } - } - } - output: { - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreA" - }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - }, - { - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "finishReason": "stop", - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "role": "assistant" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "responseMessages": "", - "steps": [ - { - "callId": "", - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreA" - }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 0, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 173 - }, - "inputTokens": 173, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 59 - }, - "outputTokens": 59, - "raw": { - "input_tokens": 173, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 59, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 232 - }, - "totalTokens": 232 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - }, - { - "dynamic": false, - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "finishReason": "tool-calls", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 1, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 260 - }, - "inputTokens": 260, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 21 - }, - "outputTokens": 21, - "raw": { - "input_tokens": 260, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 21, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 281 - }, - "totalTokens": 281 - }, - "warnings": [] - }, - { - "callId": "", - "content": [ - { - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "finishReason": "stop", - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": { - "messages": [ - { - "content": [ - { - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "role": "assistant" - } - ], - "modelId": "gpt-4o-mini-2024-07-18", - "timestamp": "" - }, - "runtimeContext": {}, - "stepNumber": 2, - "toolsContext": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 305 - }, - "inputTokens": 305, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 59 - }, - "outputTokens": 59, - "raw": { - "input_tokens": 305, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 59, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 364 - }, - "totalTokens": 364 - }, - "warnings": [] - } - ], - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "toolCalls": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - } - ], - "toolResults": [ - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreA" - }, - "output": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "item": "laptop", - "store": "StoreB" - }, - "output": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "dynamic": false, - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "output": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}", - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "totalUsage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 738 - }, - "inputTokens": 738, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 139 - }, - "outputTokens": 139, - "totalTokens": 877 - }, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 738 - }, - "inputTokens": 738, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 139 - }, - "outputTokens": 139, - "totalTokens": 877 - }, - "warnings": [] - } - metadata: { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" + "js": "return 42;" } - ├── doGenerate [llm] - │ input: { - │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - │ "maxOutputTokens": 256, - │ "messages": [ - │ { - │ "content": "I need help comparing laptop prices before checkout.", - │ "role": "user" - │ }, - │ { - │ "content": "I can compare store prices and apply a discount code.", - │ "role": "assistant" - │ }, - │ { - │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - │ "role": "user" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ }, - │ "temperature": 0, - │ "tools": [ - │ { - │ "description": "Get the price of an item at a store", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "item": { - │ "description": "The item to price", - │ "type": "string" - │ }, - │ "store": { - │ "description": "The store name", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "item", - │ "store" - │ ], - │ "type": "object" - │ }, - │ "name": "get_store_price", - │ "type": "function" - │ }, - │ { - │ "description": "Apply a discount code to a total", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "discountCode": { - │ "description": "The discount code to apply", - │ "type": "string" - │ }, - │ "total": { - │ "description": "The original total", - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "discountCode", - │ "total" - │ ], - │ "type": "object" - │ }, - │ "name": "apply_discount", - │ "type": "function" - │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreA" - │ }, - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ }, - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreB" - │ }, - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ } - │ ], - │ "finishReason": "tool-calls", - │ "providerMetadata": { - │ "openai": { - │ "responseId": "", - │ "serviceTier": "default" - │ } - │ }, - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "cacheWriteTokens": 0, - │ "noCacheTokens": 173 - │ }, - │ "inputTokens": 173, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 59 - │ }, - │ "outputTokens": 59, - │ "raw": { - │ "input_tokens": 173, - │ "input_tokens_details": { - │ "cache_write_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 59, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ }, - │ "total_tokens": 232 - │ }, - │ "totalTokens": 232 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ } - │ metrics: { - │ "completion_tokens": 59, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 173, - │ "tokens": 232 - │ } - ├── get_store_price [tool] - │ input: { - │ "item": "laptop", - │ "store": "StoreA" - │ } - │ output: "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price" - │ } - ├── get_store_price [tool] - │ input: { - │ "item": "laptop", - │ "store": "StoreB" - │ } - │ output: "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price" - │ } - ├── doGenerate [llm] - │ input: { - │ "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - │ "maxOutputTokens": 256, - │ "messages": [ - │ { - │ "content": "I need help comparing laptop prices before checkout.", - │ "role": "user" - │ }, - │ { - │ "content": "I can compare store prices and apply a discount code.", - │ "role": "assistant" - │ }, - │ { - │ "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - │ "role": "user" - │ }, - │ { - │ "content": [ - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreA" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ }, - │ { - │ "input": { - │ "item": "laptop", - │ "store": "StoreB" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-call" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "output": { - │ "type": "text", - │ "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-result" - │ }, - │ { - │ "output": { - │ "type": "text", - │ "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - │ }, - │ "providerOptions": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "get_store_price", - │ "type": "tool-result" - │ } - │ ], - │ "role": "tool" - │ } - │ ], - │ "model": { - │ "modelId": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ }, - │ "temperature": 0, - │ "tools": [ - │ { - │ "description": "Get the price of an item at a store", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "item": { - │ "description": "The item to price", - │ "type": "string" - │ }, - │ "store": { - │ "description": "The store name", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "item", - │ "store" - │ ], - │ "type": "object" - │ }, - │ "name": "get_store_price", - │ "type": "function" - │ }, - │ { - │ "description": "Apply a discount code to a total", - │ "inputSchema": { - │ "$schema": "http://json-schema.org/draft-07/schema#", - │ "additionalProperties": false, - │ "properties": { - │ "discountCode": { - │ "description": "The discount code to apply", - │ "type": "string" - │ }, - │ "total": { - │ "description": "The original total", - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "discountCode", - │ "total" - │ ], - │ "type": "object" - │ }, - │ "name": "apply_discount", - │ "type": "function" - │ } - │ ] - │ } - │ output: { - │ "content": [ - │ { - │ "input": { - │ "discountCode": "SAVE20", - │ "total": 999 - │ }, - │ "providerMetadata": { - │ "openai": { - │ "itemId": "" - │ } - │ }, - │ "toolCallId": "", - │ "toolName": "apply_discount", - │ "type": "tool-call" - │ } - │ ], - │ "finishReason": "tool-calls", - │ "providerMetadata": { - │ "openai": { - │ "responseId": "", - │ "serviceTier": "default" - │ } - │ }, - │ "response": {}, - │ "usage": { - │ "inputTokenDetails": { - │ "cacheReadTokens": 0, - │ "cacheWriteTokens": 0, - │ "noCacheTokens": 260 - │ }, - │ "inputTokens": 260, - │ "outputTokenDetails": { - │ "reasoningTokens": 0, - │ "textTokens": 21 - │ }, - │ "outputTokens": 21, - │ "raw": { - │ "input_tokens": 260, - │ "input_tokens_details": { - │ "cache_write_tokens": 0, - │ "cached_tokens": 0 - │ }, - │ "output_tokens": 21, - │ "output_tokens_details": { - │ "reasoning_tokens": 0 - │ }, - │ "total_tokens": 281 - │ }, - │ "totalTokens": 281 - │ } - │ } - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai.responses" - │ } - │ metrics: { - │ "completion_tokens": 21, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 260, - │ "tokens": 281 - │ } - ├── apply_discount [tool] - │ input: { - │ "discountCode": "SAVE20", - │ "total": 999 - │ } - │ output: "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - │ metadata: { - │ "braintrust": { - │ "integration_name": "ai-sdk", - │ "sdk_language": "typescript" - │ }, - │ "toolCallId": "", - │ "toolName": "apply_discount" - │ } - └── doGenerate [llm] - input: { - "instructions": "You are a shopping assistant. Use the tools before answering pricing questions.", - "maxOutputTokens": 256, - "messages": [ - { - "content": "I need help comparing laptop prices before checkout.", - "role": "user" - }, - { - "content": "I can compare store prices and apply a discount code.", - "role": "assistant" - }, - { - "content": "Call get_store_price for a laptop at StoreA and StoreB, then call apply_discount with code SAVE20 for the cheaper price. Finish with the discounted total.", - "role": "user" - }, - { - "content": [ - { - "input": { - "item": "laptop", - "store": "StoreA" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - }, - { - "input": { - "item": "laptop", - "store": "StoreB" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":999,\"store\":\"StoreA\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - }, - { - "output": { - "type": "text", - "value": "{\"item\":\"laptop\",\"price\":1099,\"store\":\"StoreB\"}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_store_price", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "discountCode": "SAVE20", - "total": 999 - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"discountCode\":\"SAVE20\",\"finalTotal\":799.2,\"originalTotal\":999}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "apply_discount", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "model": { - "modelId": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "temperature": 0, - "tools": [ - { - "description": "Get the price of an item at a store", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "item": { - "description": "The item to price", - "type": "string" - }, - "store": { - "description": "The store name", - "type": "string" - } - }, - "required": [ - "item", - "store" - ], - "type": "object" - }, - "name": "get_store_price", - "type": "function" - }, - { - "description": "Apply a discount code to a total", - "inputSchema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "discountCode": { - "description": "The discount code to apply", - "type": "string" - }, - "total": { - "description": "The original total", - "type": "number" - } - }, - "required": [ - "discountCode", - "total" - ], - "type": "object" - }, - "name": "apply_discount", - "type": "function" - } - ] - } - output: { - "content": [ - { - "providerMetadata": { - "openai": { - "itemId": "" - } - }, - "text": "Here are the prices for the laptop:\n\n- **Store A**: $999\n- **Store B**: $1099\n\nThe cheaper price is from **Store A**. After applying the discount code **SAVE20**, the final total is **$799.20**.", - "type": "text" - } - ], - "finishReason": "stop", - "providerMetadata": { - "openai": { - "responseId": "", - "serviceTier": "default" - } - }, - "response": {}, - "usage": { - "inputTokenDetails": { - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "noCacheTokens": 305 - }, - "inputTokens": 305, - "outputTokenDetails": { - "reasoningTokens": 0, - "textTokens": 59 - }, - "outputTokens": 59, - "raw": { - "input_tokens": 305, - "input_tokens_details": { - "cache_write_tokens": 0, - "cached_tokens": 0 - }, - "output_tokens": 59, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 364 - }, - "totalTokens": 364 - } - } - metadata: { - "braintrust": { - "integration_name": "ai-sdk", - "sdk_language": "typescript" - }, - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - } - metrics: { - "completion_tokens": 59, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 305, - "tokens": 364 - } + output: 42 diff --git a/e2e/scenarios/ai-sdk-instrumentation/assertions.ts b/e2e/scenarios/ai-sdk-instrumentation/assertions.ts index 1e971c4b1..c30ef2195 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/assertions.ts @@ -136,7 +136,7 @@ function findModelChildren( }); } -function findModelDescendants( +function findSpanDescendants( capturedEvents: CapturedLogEvent[], parentId: string | undefined, ) { @@ -151,13 +151,9 @@ function findModelDescendants( } } - return capturedEvents.filter((event) => { - const name = event.span.name ?? ""; - return ( - event.span.parentIds.some((id) => descendantIds.has(id)) && - (name === "doGenerate" || name === "doStream") - ); - }); + return capturedEvents.filter((event) => + event.span.parentIds.some((id) => descendantIds.has(id)), + ); } function findParentSpan( @@ -332,7 +328,10 @@ function findAgentGenerateTrace( `${agentSpanName}.generate`, operation?.span.id, ); - const modelChildren = findModelDescendants(events, parent?.span.id); + const modelChildren = findSpanDescendants(events, parent?.span.id).filter( + (event) => + event.span.name === "doGenerate" || event.span.name === "doStream", + ); return { latestChild: latestEvent(modelChildren), @@ -352,7 +351,10 @@ function findAgentStreamTrace( `${agentSpanName}.stream`, operation?.span.id, ); - const modelChildren = findModelDescendants(events, parent?.span.id); + const modelChildren = findSpanDescendants(events, parent?.span.id).filter( + (event) => + event.span.name === "doGenerate" || event.span.name === "doStream", + ); return { latestChild: latestEvent(modelChildren), @@ -665,6 +667,7 @@ function expectAnyMetricGreaterThanZero( export function defineAISDKInstrumentationAssertions(options: { agentSpanName?: AgentSpanName; + codeModeModes: string[]; name: string; runScenario: RunAISDKScenario; sdkMajorVersion: number; @@ -1570,6 +1573,12 @@ export function defineAISDKInstrumentationAssertions(options: { ); } + for (const mode of options.codeModeModes) { + test(`captures code-mode ${mode} execution`, testConfig, () => { + assertAISDKCodeModeInstrumentation(events, mode); + }); + } + test("matches the shared span tree snapshot", testConfig, async () => { await matchSpanTreeSnapshot(events, spanSnapshotPath, { normalize: { @@ -1611,3 +1620,123 @@ function findWorkflowAgentTrace( workflowSpans, }; } + +function assertAISDKCodeModeInstrumentation( + events: CapturedLogEvent[], + mode: string, +) { + const roots = [ + "sequential", + "parallel", + "nested", + "host-error", + "code-error", + "caught-error", + "stream-output", + "concurrent", + "direct", + ...(mode === "wrapped" ? [] : ["factory", "generation"]), + ]; + const root = (name: string) => + findAllSpans(events, `ai-sdk-code-mode-${mode}-${name}-operation`)[0]!; + for (const name of roots) { + expect(root(name)).toBeDefined(); + expect(root(name).span.parentIds).toEqual([ + findLatestSpan(events, ROOT_NAME)!.span.id, + ]); + } + const spans = (name: string, rootName: string) => + findAllSpans(findSpanDescendants(events, root(rootName).span.id), name); + for (const name of roots.filter((n) => n !== "generation")) { + const outer = spans("codeMode", name); + expect(outer).toHaveLength(name === "concurrent" ? 2 : 1); + for (const span of outer) { + expect(span.span.type).toBe("tool"); + expect(span.span.parentIds).toEqual([root(name).span.id]); + expect(span.span.ended).toBe(true); + for (const child of events.filter((event) => + event.span.parentIds.includes(span.span.id), + )) { + expect(child.span.type).toBe("function"); + } + } + } + expect(findAllSpans(events, "unused")).toHaveLength(0); + for (const name of ["sequential", "concurrent"]) { + const host = spans("host", name); + const outer = spans("codeMode", name); + expect(host).toHaveLength(2); + for (const span of host) { + expect(span.span.type).toBe("function"); + expect(span.output).toEqual({ + value: (span.input as { value: number }).value * 2, + }); + const parent = outer.find((s) => s.span.id === span.span.parentIds[0])!; + expect(parent).toBeDefined(); + expect(span.metrics!.start).toBeGreaterThanOrEqual( + parent.metrics!.start as number, + ); + expect(span.metrics!.end).toBeLessThanOrEqual( + parent.metrics!.end as number, + ); + } + if (name === "sequential") + expect(host[1]!.metrics!.start).toBeGreaterThanOrEqual( + host[0]!.metrics!.end as number, + ); + else expect(new Set(host.map((s) => s.span.parentIds[0])).size).toBe(2); + } + const parallel = spans("parallel", "parallel").filter( + (s) => s.span.type === "function", + ); + expect(parallel).toHaveLength(2); + expect( + Math.max(...parallel.map((s) => s.metrics!.start as number)), + ).toBeLessThan(Math.min(...parallel.map((s) => s.metrics!.end as number))); + for (const span of parallel) + expect(span.span.parentIds).toEqual([ + spans("codeMode", "parallel")[0]!.span.id, + ]); + const nested = spans("nested", "nested").find( + (s) => s.span.type === "function", + )!; + expect(nested.span.parentIds).toEqual([ + spans("codeMode", "nested")[0]!.span.id, + ]); + expect(spans("host", "nested")).toHaveLength(1); + expect(spans("host", "nested")[0]!.span.type).toBe("function"); + expect(spans("host", "nested")[0]!.span.parentIds).toEqual([nested.span.id]); + for (const name of ["host-error", "code-error"]) + expect(spans("codeMode", name)[0]!.row.error).toBeTruthy(); + expect(spans("fail", "host-error")[0]!.row.error).toContain("host failure"); + expect(spans("fail", "caught-error")[0]!.row.error).toContain("host failure"); + expect(spans("codeMode", "caught-error")[0]!.row.error).toBeUndefined(); + expect(spans("codeMode", "caught-error")[0]!.output).toBe("recovered"); + expect(spans("stream", "stream-output")[0]!.output).toEqual({ + value: 9, + }); + expect(spans("codeMode", "direct")[0]!.output).toBe(42); + if (mode !== "wrapped") { + const code = spans("code", "generation"); + const host = spans("host", "generation"); + expect(code).toHaveLength(1); + expect(code[0]!.span.type).toBe("tool"); + expect(code[0]!.output).toEqual({ value: 12 }); + expect(spans("codeMode", "generation")).toHaveLength(0); + expect(host).toHaveLength(2); + expect( + host.find((s) => (s.input as { value: number }).value === 6)!.span.type, + ).toBe("function"); + expect( + host.find((s) => (s.input as { value: number }).value === 8)!.span.type, + ).toBe("tool"); + expect( + host.find((s) => (s.input as { value: number }).value === 6)!.span + .parentIds, + ).toEqual([code[0]!.span.id]); + expect( + host.find((s) => (s.input as { value: number }).value === 8)!.span + .parentIds, + ).not.toEqual([code[0]!.span.id]); + } +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/package.json b/e2e/scenarios/ai-sdk-instrumentation/package.json index f6e8b40e5..7962fb571 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/package.json +++ b/e2e/scenarios/ai-sdk-instrumentation/package.json @@ -67,13 +67,17 @@ "ai-sdk-cohere-v7-latest": { "package": "@ai-sdk/cohere", "range": "4" + }, + "code-mode-v1-latest": { + "package": "@ai-sdk/code-mode", + "range": "1" } } } }, "dependencies": { "zod": "3.25.76", - "ai": "7.0.8", + "ai": "7.0.143", "ai-sdk-v3": "npm:ai@3.4.33", "ai-sdk-v3-latest": "npm:ai@3.4.33", "ai-sdk-openai-v3": "npm:@ai-sdk/openai@0.0.62", @@ -106,7 +110,9 @@ "ai-sdk-anthropic-v7-latest": "npm:@ai-sdk/anthropic@4.0.36", "ai-sdk-cohere-v7": "npm:@ai-sdk/cohere@4.0.0", "ai-sdk-cohere-v7-latest": "npm:@ai-sdk/cohere@4.0.25", - "ai-sdk-workflow-v1": "npm:@ai-sdk/workflow@1.0.8" + "ai-sdk-workflow-v1": "npm:@ai-sdk/workflow@1.0.8", + "code-mode-v1": "npm:@ai-sdk/code-mode@1.0.100", + "code-mode-v1-latest": "npm:@ai-sdk/code-mode@1.0.100" }, "pnpm": { "overrides": { diff --git a/e2e/scenarios/ai-sdk-instrumentation/pnpm-lock.yaml b/e2e/scenarios/ai-sdk-instrumentation/pnpm-lock.yaml index e31c2a1ac..cf9e235f4 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/ai-sdk-instrumentation/pnpm-lock.yaml @@ -24,8 +24,8 @@ importers: .: dependencies: ai: - specifier: 7.0.8 - version: 7.0.8(zod@3.25.76) + specifier: 7.0.143 + version: 7.0.143(zod@3.25.76) ai-sdk-anthropic-v5: specifier: npm:@ai-sdk/anthropic@2.0.74 version: '@ai-sdk/anthropic@2.0.74(zod@3.25.76)' @@ -125,6 +125,12 @@ importers: ai-sdk-workflow-v1: specifier: npm:@ai-sdk/workflow@1.0.8 version: '@ai-sdk/workflow@1.0.8(zod@3.25.76)' + code-mode-v1: + specifier: npm:@ai-sdk/code-mode@1.0.100 + version: '@ai-sdk/code-mode@1.0.100(ai@7.0.143(zod@3.25.76))' + code-mode-v1-latest: + specifier: npm:@ai-sdk/code-mode@1.0.100 + version: '@ai-sdk/code-mode@1.0.100(ai@7.0.143(zod@3.25.76))' zod: specifier: 3.25.76 version: 3.25.76 @@ -167,6 +173,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/code-mode@1.0.100': + resolution: {integrity: sha512-PMFuJiNn0UrfqyUMg+mbGvQkBM7ysEKPqbjR5caNepsI35+jMGHg3t2HaITjJ+caKMG2gk/WemykmuMVtKwPfg==} + engines: {node: '>=22.13.0'} + peerDependencies: + ai: 7.0.143 + '@ai-sdk/cohere@2.0.35': resolution: {integrity: sha512-bvdbfO3SUfKq5ziaK6EYy9NMJND65UTcr0Xd1WtyTxZHOfKB3svDI8dIFpKHx7Z9fsQfWLBsbpXIN7scs2stGg==} engines: {node: '>=18'} @@ -233,6 +245,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.116': + resolution: {integrity: sha512-YK2As2GpxmyHi5w6R4FLepiLcs6MbIMxh6nyTMgHaoHKoM5aWEXvKDe/JqdMQ1ivOy3EPJiEBgRI677YTGxfHQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.6': resolution: {integrity: sha512-8GjssUxXeTd8tst2fYXNOFbtNNZFv3sG7aq7wKnQYrU2eB3JFR7np6o4F3Snp/fy/rJZSeH28LvjSWq5wkdL8Q==} engines: {node: '>=22'} @@ -419,6 +437,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.62': + resolution: {integrity: sha512-//6sHa6QNQdM2KHOg5pxp1UMFQB4U8cqdw/kSm9BMr0DLeaFgG3TXZfxhpwRZmxgBpOrzMVZ/gwLXH309+Ortw==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@0.0.23': resolution: {integrity: sha512-oAc49O5+xypVrKM7EUU5P/Y4DUL4JZUWVxhejoAVOTOl3WZUEWsMbP3QZR+TrimQIsS0WR/n9UuF6U0jPdp0tQ==} engines: {node: '>=18'} @@ -487,6 +511,10 @@ packages: resolution: {integrity: sha512-TtWwm6SxFiPomlMIbTQa+yhJte+w4/oCsW+wJrJzMC3MqHpAY9e5dJF1Y4HxyKt3W2Csac0KiI76Ckio11TFYw==} engines: {node: '>=22'} + '@ai-sdk/provider@4.0.29': + resolution: {integrity: sha512-nV7UpLADtKn9f0xfnxVZAeFEKuNkScJv5rOpHL7qeWe2EwEEBa3pd6P0QVK6D6coCJ0E8FyCgoM50AffTYrzDQ==} + engines: {node: '>=22'} + '@ai-sdk/provider@4.0.7': resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} engines: {node: '>=22'} @@ -754,6 +782,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ai@7.0.143: + resolution: {integrity: sha512-y9QNgCK9QRHRCcHEUNoU2vemJ+q2zFX5M9T7xHCMZutfcWCqZPckkAfDbp+KHwqZDiJs5AqoKazj64cb20m5PA==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ai@7.0.8: resolution: {integrity: sha512-vTEKl6fDBZ2IxBXTRaZOajf9W2Ev57Ju8iKtUvqlmDk8Z9BrEP4c22SWJsg1RcWHSFmJMSBa/s5dlUBHUq3YwA==} engines: {node: '>=22'} @@ -864,6 +898,15 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + run@2.1.8: + resolution: {integrity: sha512-B+N1a8grWUEmQwS8QL+o0yoB0sjPitphQ/PaION9j0m/msDxA9j1fxgX6VC4m604/OzT3IAMAZIw7kn8zn1R0A==} + engines: {node: '>=20.19.0'} + peerDependencies: + typescript: '>=5.1.6 <7' + peerDependenciesMeta: + typescript: + optional: true + secure-json-parse@2.7.0: resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} @@ -967,6 +1010,13 @@ snapshots: '@ai-sdk/provider-utils': 5.0.25(zod@3.25.76) zod: 3.25.76 + '@ai-sdk/code-mode@1.0.100(ai@7.0.143(zod@3.25.76))': + dependencies: + ai: 7.0.143(zod@3.25.76) + run: 2.1.8 + transitivePeerDependencies: + - typescript + '@ai-sdk/cohere@2.0.35(zod@3.25.76)': dependencies: '@ai-sdk/provider': 2.0.3 @@ -1038,6 +1088,13 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 3.25.76 + '@ai-sdk/gateway@4.0.116(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.29 + '@ai-sdk/provider-utils': 5.0.62(zod@3.25.76) + '@vercel/oidc': 3.2.0 + zod: 3.25.76 + '@ai-sdk/gateway@4.0.6(zod@3.25.76)': dependencies: '@ai-sdk/provider': 4.0.1 @@ -1255,6 +1312,15 @@ snapshots: undici: 7.29.1 zod: 3.25.76 + '@ai-sdk/provider-utils@5.0.62(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 4.0.29 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.0.8 + undici: 7.29.1 + zod: 3.25.76 + '@ai-sdk/provider@0.0.23': dependencies: json-schema: 0.4.0 @@ -1323,6 +1389,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@4.0.29': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@4.0.7': dependencies: json-schema: 0.4.0 @@ -1629,6 +1699,13 @@ snapshots: '@ai-sdk/provider-utils': 5.0.59(zod@3.25.76) zod: 3.25.76 + ai@7.0.143(zod@3.25.76): + dependencies: + '@ai-sdk/gateway': 4.0.116(zod@3.25.76) + '@ai-sdk/provider': 4.0.29 + '@ai-sdk/provider-utils': 5.0.62(zod@3.25.76) + zod: 3.25.76 + ai@7.0.8(zod@3.25.76): dependencies: '@ai-sdk/gateway': 4.0.6(zod@3.25.76) @@ -1711,6 +1788,8 @@ snapshots: require-from-string@2.0.2: {} + run@2.1.8: {} + secure-json-parse@2.7.0: {} source-map-js@1.2.2: {} diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs index a0f5094ba..ce45d9337 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7-explicit.mjs @@ -36,6 +36,7 @@ runMain(async () => { await runAutoAISDKInstrumentation({ agentClassExport: "ToolLoopAgent", + codeModeModes: ["wrapped"], ai, anthropic, createAnthropic, diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs index f932e1372..4e30ccce7 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.mjs @@ -28,6 +28,7 @@ import { runAutoAISDKInstrumentationOrExit } from "./scenario.impl.mjs"; runAutoAISDKInstrumentationOrExit({ agentClassExport: "ToolLoopAgent", + codeModeModes: ["auto-hook", "combined"], ai, anthropic, createAnthropic, diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts index 4c43ceb67..70b8ebf9f 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.ai-sdk-v7.ts @@ -22,6 +22,7 @@ runMain(async () => { await runAutoAISDKInstrumentation({ agentClassExport: "ToolLoopAgent", + codeModeModes: ["wrapped"], ai, anthropic, createAnthropic, diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.code-mode.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.code-mode.mjs new file mode 100644 index 000000000..cde34d904 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.code-mode.mjs @@ -0,0 +1,185 @@ +import assert from "node:assert/strict"; +import { setTimeout as delay } from "node:timers/promises"; +import { braintrustAISDKTelemetry, wrapAISDK } from "braintrust"; +import * as ai from "ai"; +import { runOperation } from "../../helpers/provider-runtime.mjs"; + +export async function runAISDKCodeMode(mode, packageName) { + const namespace = await import(packageName); + const codeMode = mode === "auto-hook" ? namespace : wrapAISDK(namespace); + const run = codeMode.experimental_runCodeMode; + const root = (name, fn) => + runOperation( + `ai-sdk-code-mode-${mode}-${name}-operation`, + `code-mode-${mode}-${name}`, + fn, + ); + const schema = ai.jsonSchema({ + type: "object", + properties: { value: { type: "number" } }, + required: ["value"], + additionalProperties: false, + }); + const host = { + inputSchema: schema, + label: "original receiver", + async execute({ value }, options) { + assert.equal(this.label, "original receiver"); + assert.ok(options.toolCallId); + await delay(20); + return { value: value * 2 }; + }, + }; + const tools = { + host, + unused: { + inputSchema: schema, + execute() { + throw new Error("unused tool executed"); + }, + }, + fail: { + inputSchema: schema, + execute() { + throw new Error("host failure"); + }, + }, + stream: { + inputSchema: schema, + async *execute({ value }) { + yield { value: 0 }; + await delay(10); + yield { value }; + }, + }, + }; + const originalExecute = host.execute; + const execute = (js, callId = "outer", extraTools = {}) => + run({ + js, + tools: { ...tools, ...extraTools }, + toolExecutionOptions: { toolCallId: callId, messages: [] }, + }); + + await root("sequential", async () => { + assert.deepEqual( + await execute( + "const a = await tools.host({value: 2}); return await tools.host(a);", + ), + { value: 8 }, + ); + }); + await root("parallel", async () => { + let count = 0; + let release; + const bothStarted = new Promise((resolve) => { + release = resolve; + }); + const parallel = { + inputSchema: schema, + async execute({ value }) { + if (++count === 2) release(); + await bothStarted; + await delay(20); + return { value }; + }, + }; + assert.deepEqual( + await execute( + "return await Promise.all([tools.parallel({value: 1}), tools.parallel({value: 2})]);", + "outer", + { parallel }, + ), + [{ value: 1 }, { value: 2 }], + ); + }); + await root("nested", async () => { + const nested = { + inputSchema: schema, + execute({ value }, options) { + return execute( + `return await tools.host({value: ${value}});`, + options.toolCallId, + ); + }, + }; + assert.deepEqual( + await execute("return await tools.nested({value: 3});", "outer", { + nested, + }), + { value: 6 }, + ); + }); + await root("host-error", async () => { + await assert.rejects(execute("return await tools.fail({value: 1});")); + }); + await root("code-error", async () => { + await assert.rejects(execute('throw new Error("code failure");')); + }); + await root("caught-error", async () => { + assert.equal( + await execute( + 'try { await tools.fail({value: 1}); } catch { return "recovered"; }', + ), + "recovered", + ); + }); + await root("stream-output", async () => { + assert.deepEqual(await execute("return await tools.stream({value: 9});"), { + value: 9, + }); + }); + await root("concurrent", async () => { + assert.deepEqual( + await Promise.all([ + execute("return await tools.host({value: 4});"), + execute("return await tools.host({value: 5});"), + ]), + [{ value: 8 }, { value: 10 }], + ); + }); + await root("direct", async () => { + // No call ID or AI SDK generation is required by runCodeMode. + assert.equal(await run({ js: "return 42;", tools: {} }), 42); + }); + if (mode !== "wrapped") { + await root("factory", async () => { + const tool = namespace.experimental_createCodeModeTool({ host }); + assert.deepEqual( + await tool.execute( + { js: "return await tools.host({value: 7});" }, + { toolCallId: "factory", messages: [] }, + ), + { value: 14 }, + ); + }); + } + assert.equal(host.execute, originalExecute); + + if (mode !== "wrapped") { + const { createOpenAI } = await import("ai-sdk-openai-v7-latest"); + const openai = createOpenAI({ baseURL: process.env.OPENAI_BASE_URL }); + if (mode === "combined") ai.registerTelemetry(braintrustAISDKTelemetry()); + await root("generation", async () => { + const code = namespace.experimental_codeModeTool(); + const result = await ai.generateText({ + model: openai("gpt-4.1-nano"), + tools: { code, host }, + experimental_toolCallers: { host: ["code", "AI_SDK_DIRECT_TOOL_CALL"] }, + toolChoice: { type: "tool", toolName: "code" }, + prompt: + "Call code with exactly this JavaScript: return await tools.host({value: 6});", + maxOutputTokens: 120, + }); + assert.deepEqual(result.toolResults[0].output, { value: 12 }); + const direct = await ai.generateText({ + model: openai("gpt-4.1-nano"), + tools: { host }, + toolChoice: { type: "tool", toolName: "host" }, + prompt: "Call host with value 8.", + maxOutputTokens: 80, + }); + assert.deepEqual(direct.toolResults[0].output, { value: 16 }); + }); + } +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs index 802f01602..cb1810ad6 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs @@ -166,6 +166,7 @@ export const AI_SDK_SCENARIO_SPECS = [ openaiModuleName: "ai-sdk-openai-v7", packageName: "ai-sdk-v7", snapshotName: "ai-sdk-v7", + codeModeModuleName: "code-mode-v1", supportsEvaluate: true, supportsEvaluateStringModel: false, anthropicModuleName: "ai-sdk-anthropic-v7", @@ -196,6 +197,7 @@ export const AI_SDK_SCENARIO_SPECS = [ openaiModuleName: "ai-sdk-openai-v7-latest", packageName: "ai-sdk-v7-latest", snapshotName: "ai-sdk-v7-latest", + codeModeModuleName: "code-mode-v1-latest", supportsEvaluate: true, anthropicModuleName: "ai-sdk-anthropic-v7-latest", supportsAgentToolLoop: true, @@ -1134,6 +1136,15 @@ async function runAISDKInstrumentationScenario( openaiModel, }); } + if (process.env.AI_SDK_CODE_MODE_PACKAGE_NAME) { + const { runAISDKCodeMode } = await import("./scenario.code-mode.mjs"); + for (const mode of options.codeModeModes) { + await runAISDKCodeMode( + mode, + process.env.AI_SDK_CODE_MODE_PACKAGE_NAME, + ); + } + } }, flushCount, flushDelayMs, diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts b/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts index 4ebeb9198..b5efdc36f 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts @@ -51,11 +51,17 @@ describeVariants("variants", () => { defineAISDKInstrumentationAssertions({ agentSpanName: scenario.agentSpanName, name: scenario.wrapperTestName ?? "wrapped instrumentation", + codeModeModes: scenario.codeModeModuleName ? ["wrapped"] : [], runScenario: async ({ runScenarioDir }) => { await runScenarioDir({ entry: scenario.wrapperEntry, env: { AI_SDK_PACKAGE_NAME: scenario.packageName, + ...(scenario.codeModeModuleName + ? { + AI_SDK_CODE_MODE_PACKAGE_NAME: scenario.codeModeModuleName, + } + : {}), BRAINTRUST_CAPTURE_ATTACHMENTS: "true", AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName, AI_SDK_SUPPORTS_EVALUATE_STRING_MODEL: String( @@ -113,11 +119,19 @@ describeVariants("variants", () => { defineAISDKInstrumentationAssertions({ agentSpanName: scenario.agentSpanName, name: "auto-hook instrumentation", + codeModeModes: scenario.codeModeModuleName + ? ["auto-hook", "combined"] + : [], runScenario: async ({ runNodeScenarioDir }) => { await runNodeScenarioDir({ entry: scenario.autoEntry, env: { AI_SDK_PACKAGE_NAME: scenario.packageName, + ...(scenario.codeModeModuleName + ? { + AI_SDK_CODE_MODE_PACKAGE_NAME: scenario.codeModeModuleName, + } + : {}), BRAINTRUST_CAPTURE_ATTACHMENTS: "true", AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName, AI_SDK_SUPPORTS_EVALUATE_STRING_MODEL: String( diff --git a/js/src/auto-instrumentations/configs/ai-sdk.ts b/js/src/auto-instrumentations/configs/ai-sdk.ts index c1c42c528..67e4cb0e9 100644 --- a/js/src/auto-instrumentations/configs/ai-sdk.ts +++ b/js/src/auto-instrumentations/configs/ai-sdk.ts @@ -1,6 +1,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { aiSDKChannels, + codeModeChannels, harnessAgentChannels, } from "../../instrumentation/plugins/ai-sdk-interceptors"; @@ -16,6 +17,15 @@ import { * "orchestrion:ai-sdk:generateText" */ export const aiSDKConfigs: InstrumentationConfig[] = [ + { + channelName: codeModeChannels.run.channelName, + module: { + name: "@ai-sdk/code-mode", + versionRange: ">=1.0.100 <2.0.0", + filePath: "dist/run-code-mode.js", + }, + functionQuery: { functionName: "runCodeMode" }, + }, // Older evaluation releases have no telemetry callbacks. Newer versions use // the telemetry dispatcher below, including the renamed decision API. { diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 33d3d291b..366dd9397 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -30,9 +30,10 @@ import type { ProgressReporter } from "./reporters/types"; import { InternalAbortError } from "./util"; import { getEventListeners } from "node:events"; +// SDK initialization can exceed the default hook timeout on Windows CI. beforeAll(() => { configureNode(); -}); +}, 30_000); class NoopProgressReporter implements ProgressReporter { public start() {} diff --git a/js/src/instrumentation/plugins/ai-sdk-code-mode.test.ts b/js/src/instrumentation/plugins/ai-sdk-code-mode.test.ts new file mode 100644 index 000000000..beca0c617 --- /dev/null +++ b/js/src/instrumentation/plugins/ai-sdk-code-mode.test.ts @@ -0,0 +1,327 @@ +import { AsyncResource } from "node:async_hooks"; +import { afterEach, beforeAll, beforeEach, expect, it } from "vitest"; +import { + _exportsForTestingOnly, + NOOP_SPAN, + currentSpan, + initLogger, + startSpan, + withCurrent, +} from "../../logger"; +import { braintrustAISDKTelemetry } from "../../wrappers/ai-sdk/telemetry"; +import { configureNode } from "../../node/config"; +import { aiSDKChannels } from "./ai-sdk-interceptors"; +import { traceAISDKCodeMode } from "./ai-sdk-code-mode"; +import { + AI_SDK_TOOL_CALL_ID, + AI_SDK_TOOL_CAPTURE, + currentAISDKToolSpan, + traceAISDKTool, +} from "./ai-sdk-tool-tracing"; +import { runWithAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; + +configureNode(); +let logger: ReturnType; +beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); +}); +beforeEach(() => { + logger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ projectName: "code-mode-test", projectId: "test-project-id" }); +}); +afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); +}); + +it("binds host callbacks across lost context and isolates concurrent runs with identical call IDs", async () => { + const worker = new AsyncResource("worker-before-execution"); + const parent = startSpan({ name: "parent" }); + const tools = { + host: { + label: "receiver", + needsApproval() { + expect(this).toBe(tools.host); + return false; + }, + execute: function (input: unknown) { + expect(this.label).toBe("receiver"); + expect(currentSpan().spanId).not.toBe(parent.spanId); + return input; + }, + }, + }; + const original = tools.host.execute; + await withCurrent(parent, () => + Promise.all( + [1, 2].map((value) => + traceAISDKCodeMode( + async (params) => { + const options = { toolCallId: "outer:tool-0" }; + expect(params.tools.host.needsApproval).toBeTypeOf("function"); + Reflect.apply( + params.tools.host.needsApproval as Function, + params.tools.host, + [value, options], + ); + return worker.runInAsyncScope(() => + params.tools.host.execute!(value, options), + ); + }, + { + js: String(value), + tools, + toolExecutionOptions: { toolCallId: "outer" }, + }, + ), + ), + ), + ); + expect(tools.host.execute).toBe(original); + parent.end(); + const spans = (await logger.drain()) as Record[]; + const outer = spans.filter((s) => s.span_attributes?.name === "codeMode"); + const hosts = spans.filter((s) => s.span_attributes?.name === "host"); + expect(outer).toHaveLength(2); + expect(hosts).toHaveLength(2); + for (const span of outer) { + expect(span.span_attributes.type).toBe("tool"); + expect(span.span_parents).toEqual([parent.spanId]); + expect(hosts.find((s) => s.input === Number(span.input.js))).toMatchObject({ + span_attributes: { type: "function" }, + span_parents: [span.span_id], + }); + } + worker.emitDestroy(); +}); + +it("reuses the executing tool span and deduplicates manual plus automatic run hooks", async () => { + const result = traceAISDKTool( + () => + traceAISDKCodeMode( + (params) => + traceAISDKCodeMode( + async (inner) => + inner.tools.host.execute!("input", { + toolCallId: "outer:tool-0", + }), + params, + ), + { + js: "source", + tools: { + host: { + execute: (input) => + traceAISDKTool(() => input, { + name: "host", + input, + toolCallId: "outer:tool-0", + }), + }, + }, + toolExecutionOptions: { toolCallId: "outer" }, + }, + ), + { name: "customCodeTool", input: { js: "source" }, toolCallId: "outer" }, + ); + expect(await result).toBe("input"); + const spans = (await logger.drain()) as Record[]; + expect(spans).toHaveLength(2); + const parent = spans.find( + (s) => s.span_attributes?.name === "customCodeTool", + ); + expect(parent!.span_attributes.type).toBe("tool"); + expect(spans.find((s) => s.span_attributes?.name === "host")).toMatchObject({ + span_attributes: { type: "function" }, + span_parents: [parent!.span_id], + output: "input", + }); +}); + +it.each([false, true])( + "preserves synchronous and asynchronous errors (%s)", + async (asyncError) => { + const error = new Error("host failure"); + const run = () => + traceAISDKCodeMode( + (params) => + params.tools.host.execute!({}, { toolCallId: "outer:tool-0" }), + { + js: "source", + tools: { + host: { + execute() { + if (asyncError) return Promise.reject(error); + throw error; + }, + }, + }, + }, + ); + if (asyncError) await expect(run()).rejects.toBe(error); + else expect(run).toThrow(error); + const spans = (await logger.drain()) as Record[]; + expect(spans).toHaveLength(2); + for (const span of spans) expect(span.error).toContain("host failure"); + }, +); + +it("preserves promises, iterator output and context, and suppression", async () => { + const promise = Promise.resolve("value"); + expect(traceAISDKTool(() => promise, { name: "promise", input: 1 })).toBe( + promise, + ); + const iterator = traceAISDKTool( + () => + (async function* () { + expect(currentSpan().spanId).toBeTruthy(); + yield "first"; + yield "last"; + })(), + { name: "stream", input: 2 }, + ); + const chunks = []; + for await (const chunk of iterator) chunks.push(chunk); + expect(chunks).toEqual(["first", "last"]); + await runWithAutoInstrumentationSuppressed(() => + traceAISDKCodeMode(async () => 3, { js: "", tools: {} }), + ); + const spans = (await logger.drain()) as Record[]; + expect(spans).toHaveLength(2); + expect(spans.find((s) => s.span_attributes?.name === "stream")!.output).toBe( + "last", + ); +}); + +it("reuses telemetry ownership and propagates capture flags to host tools", async () => { + const telemetry = braintrustAISDKTelemetry(); + const event = { + callId: "generation", + recordInputs: false, + recordOutputs: false, + }; + const toolCall = { + toolCallId: "outer", + toolName: "code", + input: { js: "private source" }, + }; + telemetry.onStart({ ...event, operationId: "ai.generateText" }); + telemetry.onToolExecutionStart({ ...event, toolCall }); + await telemetry.executeTool({ + toolCallId: "outer", + execute: () => + traceAISDKCodeMode( + async (input) => + input.tools.host.execute!("private input", { + toolCallId: "outer:tool-0", + }), + { + js: "private source", + tools: { host: { execute: () => "private output" } }, + toolExecutionOptions: { toolCallId: "outer" }, + }, + ), + }); + telemetry.onToolExecutionEnd({ + ...event, + toolCall, + toolOutput: { type: "tool-result", output: "private output" }, + }); + telemetry.onEnd({ ...event, operationId: "ai.generateText" }); + const spans = (await logger.drain()) as Record[]; + expect(spans).toHaveLength(3); + for (const span of spans) { + expect(span).not.toHaveProperty("input"); + expect(span).not.toHaveProperty("output"); + } + const code = spans.find((s) => s.span_attributes?.name === "code"); + expect(code!.span_attributes.type).toBe("tool"); + expect(spans.find((s) => s.span_attributes?.name === "host")).toMatchObject({ + span_attributes: { type: "function" }, + span_parents: [code!.span_id], + }); +}); + +it.each([false, true])( + "deduplicates legacy AI SDK tool wrapping and catches sync errors (%s)", + async (fail) => { + const error = new Error("host failed synchronously"); + const host = { + execute: () => { + if (fail) throw error; + return 7; + }, + }; + const params = { + tools: { + host, + code: { + execute: () => + traceAISDKCodeMode( + async (input) => + input.tools.host.execute!({}, { toolCallId: "code:tool-0" }), + { + js: "source", + tools: { host }, + toolExecutionOptions: { toolCallId: "code" }, + }, + ), + }, + }, + }; + await aiSDKChannels.generateText.invoke( + async () => { + const run = () => + Reflect.apply(params.tools.code.execute, params.tools.code, [ + { js: "source" }, + { toolCallId: "code" }, + ]); + if (fail) await expect(run()).rejects.toBe(error); + else expect(await run()).toBe(7); + return { text: "done" }; + }, + undefined, + [params], + {}, + ); + const spans = (await logger.drain()) as Record[]; + expect(spans).toHaveLength(3); + const code = spans.find((s) => s.span_attributes?.name === "code")!; + const child = spans.find((s) => s.span_attributes?.name === "host")!; + expect(code.span_attributes.type).toBe("tool"); + expect(child.span_attributes.type).toBe("function"); + expect(child.span_parents).toEqual([code.span_id]); + if (fail) { + expect(child.error).toContain("host failed synchronously"); + expect(code.error).toContain("host failed synchronously"); + } else expect(child.output).toBe(7); + }, +); + +it("does not treat a no-op span from before initialization as a tool owner", async () => { + Object.defineProperty(NOOP_SPAN, AI_SDK_TOOL_CALL_ID, { + value: "outer", + configurable: true, + }); + Object.defineProperty(NOOP_SPAN, AI_SDK_TOOL_CAPTURE, { + value: { recordInputs: false, recordOutputs: false }, + configurable: true, + }); + try { + expect( + withCurrent(NOOP_SPAN, () => currentAISDKToolSpan("outer")), + ).toBeUndefined(); + withCurrent(NOOP_SPAN, () => + traceAISDKTool(() => "output", { + name: "tool", + input: "input", + toolCallId: "outer", + }), + ); + expect(await logger.drain()).toEqual([ + expect.objectContaining({ input: "input", output: "output" }), + ]); + } finally { + Reflect.deleteProperty(NOOP_SPAN, AI_SDK_TOOL_CAPTURE); + Reflect.deleteProperty(NOOP_SPAN, AI_SDK_TOOL_CALL_ID); + } +}); diff --git a/js/src/instrumentation/plugins/ai-sdk-code-mode.ts b/js/src/instrumentation/plugins/ai-sdk-code-mode.ts new file mode 100644 index 000000000..ca35a9653 --- /dev/null +++ b/js/src/instrumentation/plugins/ai-sdk-code-mode.ts @@ -0,0 +1,70 @@ +import { currentSpan, withCurrent, type Span } from "../../logger"; +import { SpanTypeAttribute } from "../../util"; +import type { AISDKRunCodeModeInput } from "../../vendor-sdk-types/ai-sdk-code-mode"; +import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; +import { currentAISDKToolSpan, traceAISDKTool } from "./ai-sdk-tool-tracing"; + +const CODE_MODE_PARENT = Symbol.for("braintrust.ai-sdk.code-mode-parent"); + +export function traceAISDKCodeMode( + execute: (input: AISDKRunCodeModeInput) => T, + input: AISDKRunCodeModeInput, +): T { + if ( + isAutoInstrumentationSuppressed() || + (input as AISDKRunCodeModeInput & { [CODE_MODE_PARENT]?: Span })[ + CODE_MODE_PARENT + ] + ) { + return execute(input); + } + + const run = () => { + const parent = currentSpan(); + // Bind each execution separately: worker-pool callbacks need not inherit + // the async context of runCodeMode. Never patch the user's shared tools. + const tools = Object.fromEntries( + Object.entries(input.tools).map(([name, tool]) => { + const original = tool.execute; + if (typeof original !== "function") return [name, tool]; + return [ + name, + Object.create(tool, { + execute: { + value: ( + value: unknown, + options: Parameters[1], + ) => + withCurrent(parent, () => + traceAISDKTool( + () => Reflect.apply(original, tool, [value, options]), + { + name, + input: value, + toolCallId: options?.toolCallId, + parent, + type: SpanTypeAttribute.FUNCTION, + }, + ), + ), + }, + ...(typeof tool.needsApproval === "function" + ? { + needsApproval: { value: tool.needsApproval.bind(tool) }, + } + : {}), + }), + ]; + }), + ); + return execute({ ...input, tools, [CODE_MODE_PARENT]: parent }); + }; + const toolCallId = input.toolExecutionOptions?.toolCallId; + return currentAISDKToolSpan(toolCallId) + ? run() + : traceAISDKTool(run, { + name: "codeMode", + input: { js: input.js }, + toolCallId, + }); +} diff --git a/js/src/instrumentation/plugins/ai-sdk-interceptors.ts b/js/src/instrumentation/plugins/ai-sdk-interceptors.ts index 0c20ff138..e628ada93 100644 --- a/js/src/instrumentation/plugins/ai-sdk-interceptors.ts +++ b/js/src/instrumentation/plugins/ai-sdk-interceptors.ts @@ -1,3 +1,4 @@ +import type { AISDKRunCodeModeInput } from "../../vendor-sdk-types/ai-sdk-code-mode"; import { channel, defineInterceptor } from "../core/interceptor-definitions"; import type { @@ -237,3 +238,9 @@ export const harnessAgentChannels = defineInterceptor("@ai-sdk/harness", { channelName: "HarnessAgent.continueStream", }), }); + +export const codeModeChannels = defineInterceptor("@ai-sdk/code-mode", { + run: channel<[AISDKRunCodeModeInput], Promise>({ + channelName: "runCodeMode", + }), +}); diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts b/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts index 2e0d64ba5..36191e8bc 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts +++ b/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts @@ -27,7 +27,10 @@ vi.mock("../../wrappers/ai-sdk/telemetry", () => ({ braintrustAISDKTelemetry: telemetryMocks.braintrustAISDKTelemetry, })); -import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; +import { + BRAINTRUST_AI_SDK_V7_TELEMETRY, + BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY, +} from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; import { serializeAISDKToolsForLogging } from "../../wrappers/ai-sdk/tool-serialization"; import { AISDKPlugin, @@ -612,6 +615,35 @@ describe("AISDKPlugin", () => { await expect(returned).rejects.toBe(rejection); }); + it("defers to explicit telemetry, respecting per-call registry overrides", () => { + const explicit = { [BRAINTRUST_AI_SDK_V7_TELEMETRY]: true }; + vi.stubGlobal("AI_SDK_TELEMETRY_INTEGRATIONS", [explicit]); + try { + plugin.enable(); + const channel = mockChannels.get( + "orchestrion:ai:createTelemetryDispatcher", + ); + for (const integrations of [undefined, explicit, [explicit]]) { + const dispatcher = { onStart: vi.fn() }; + const original = dispatcher.onStart; + channel?.handlers[0]?.end({ + arguments: [{ telemetry: { integrations } }], + result: dispatcher, + }); + expect(dispatcher.onStart).toBe(original); + } + const dispatcher = { onStart: vi.fn() }; + const original = dispatcher.onStart; + channel?.handlers[0]?.end({ + arguments: [{ telemetry: { integrations: [] } }], + result: dispatcher, + }); + expect(dispatcher.onStart).not.toBe(original); + } finally { + vi.unstubAllGlobals(); + } + }); + it("patches each dispatcher once and respects telemetry opt-out", () => { const dispatcher = { onStart: vi.fn(), diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.ts b/js/src/instrumentation/plugins/ai-sdk-plugin.ts index d45e0edb4..a0d30da42 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/ai-sdk-plugin.ts @@ -1,3 +1,5 @@ +import { traceAISDKCodeMode } from "./ai-sdk-code-mode"; +import { currentAISDKToolSpan, traceAISDKTool } from "./ai-sdk-tool-tracing"; import { prepareDecisionAttachments } from "../../decision-attachments"; import type { ReturnOf } from "../core/interceptor-definitions"; import { debugLogger } from "../../debug-logger"; @@ -53,9 +55,13 @@ import type { } from "../../vendor-sdk-types/ai-sdk"; import type { AISDKV7Telemetry, + AISDKV7TelemetryGlobal, AISDKV7TelemetryOptions, } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; -import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; +import { + BRAINTRUST_AI_SDK_V7_TELEMETRY, + BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY, +} from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; import { captureHarnessCreateSessionParent, endHarnessTurn, @@ -87,6 +93,7 @@ import type { ChannelMessage } from "../core/tracing-types"; import { BRAINTRUST_WRAPPED_AI_SDK_MODEL, aiSDKChannels, + codeModeChannels, harnessAgentChannels, } from "./ai-sdk-interceptors"; import { extractTokenMetrics } from "./ai-sdk-metrics"; @@ -215,6 +222,12 @@ export class AISDKPlugin extends BasePlugin { this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS; this.unsubscribers.push( + codeModeChannels.run.intercept((target, receiver, [input]) => + traceAISDKCodeMode( + (params) => Reflect.apply(target, receiver, [params]), + input, + ), + ), interceptAISDKV7TelemetryDispatcher(), interceptAISDKEvaluate(denyOutputPaths), interceptAISDKModelGenerate(denyOutputPaths), @@ -1262,7 +1275,19 @@ function interceptAISDKV7TelemetryDispatcher(): () => void { (target, thisArg, args) => { const dispatcher = Reflect.apply(target, thisArg, args); const telemetryOptions = args[0]?.telemetry; - if (telemetryOptions?.isEnabled !== false) { + // Per-call integrations replace the global registry in AI SDK. Let an + // explicitly registered Braintrust integration own the whole lifecycle. + const integrations = + telemetryOptions?.integrations ?? + (globalThis as AISDKV7TelemetryGlobal).AI_SDK_TELEMETRY_INTEGRATIONS; + const hasBraintrust = ( + Array.isArray(integrations) ? integrations : [integrations] + ).some( + (integration) => + isObject(integration) && + Reflect.get(integration, BRAINTRUST_AI_SDK_V7_TELEMETRY) === true, + ); + if (telemetryOptions?.isEnabled !== false && !hasBraintrust) { try { patchAISDKV7TelemetryDispatcher( dispatcher, @@ -3209,69 +3234,23 @@ function prepareAISDKChildTracing( }; toolRecord[AUTO_PATCHED_TOOL] = state; tool.execute = function executePatched(...args: unknown[]) { - const activeEntry = activeAISDKChildPatchEntry(state.entries); - const result = Reflect.apply(originalExecute, this, args); - if (!activeEntry) { - return result; - } - const executionOptions = args[1]; - const toolCallId = isObject(executionOptions) - ? executionOptions.toolCallId - : undefined; - const toolInput = { - input: serializeToolExecutionInput(args), - ...(typeof toolCallId === "string" ? { metadata: { toolCallId } } : {}), - }; - - if (isAsyncGenerator(result)) { - return (async function* () { - const span = activeEntry.parentSpan.startSpan( - withSpanInstrumentationName( - { - name: activeEntry.name, - spanAttributes: { - type: SpanTypeAttribute.TOOL, - }, - }, - INSTRUMENTATION_NAMES.AI_SDK, - ), - ); - span.log(toolInput); - - try { - let lastValue: unknown; - for await (const value of result) { - lastValue = value; - yield value; - } - span.log({ output: lastValue }); - } catch (error) { - span.log({ error: toLoggedError(error) }); - throw error; - } finally { - span.end(); - } - })(); + const toolCallId = + isObject(executionOptions) && + typeof executionOptions.toolCallId === "string" + ? executionOptions.toolCallId + : undefined; + if (currentAISDKToolSpan(toolCallId)) { + return Reflect.apply(originalExecute, this, args); } - - return activeEntry.parentSpan.traced( - async (span) => { - span.log(toolInput); - const awaitedResult = await result; - span.log({ output: awaitedResult }); - return awaitedResult; - }, - withSpanInstrumentationName( - { - name: activeEntry.name, - spanAttributes: { - type: SpanTypeAttribute.TOOL, - }, - }, - INSTRUMENTATION_NAMES.AI_SDK, - ), - ); + const activeEntry = activeAISDKChildPatchEntry(state.entries); + if (!activeEntry) return Reflect.apply(originalExecute, this, args); + return traceAISDKTool(() => Reflect.apply(originalExecute, this, args), { + name: activeEntry.name, + input: serializeToolExecutionInput(args), + toolCallId, + parent: activeEntry.parentSpan, + }); }; cleanup.push(() => { @@ -4160,17 +4139,6 @@ function extractTextDelta(chunk: AISDKModelStreamChunk): string { return ""; } -function isAsyncGenerator(value: unknown): value is AsyncGenerator { - return ( - value != null && - typeof value === "object" && - typeof (value as AsyncGenerator)[Symbol.asyncIterator] === "function" && - typeof (value as AsyncGenerator).next === "function" && - typeof (value as AsyncGenerator).return === "function" && - typeof (value as AsyncGenerator).throw === "function" - ); -} - /** * Process AI SDK output, omitting specified paths. */ diff --git a/js/src/instrumentation/plugins/ai-sdk-tool-tracing.ts b/js/src/instrumentation/plugins/ai-sdk-tool-tracing.ts new file mode 100644 index 000000000..862a0c0b2 --- /dev/null +++ b/js/src/instrumentation/plugins/ai-sdk-tool-tracing.ts @@ -0,0 +1,102 @@ +import { currentSpan, startSpan, withCurrent, type Span } from "../../logger"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { SpanTypeAttribute } from "../../util"; +import { observeResult } from "../core/observe-result"; +import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; + +// Span-local ownership works across SDK copies without a process-wide call-ID +// map (call IDs are only unique within an execution). +export const AI_SDK_TOOL_CALL_ID = Symbol.for("braintrust.ai-sdk.tool-call-id"); +export const AI_SDK_TOOL_CAPTURE = Symbol.for("braintrust.ai-sdk.tool-capture"); +export type AISDKToolSpan = Span & { + [AI_SDK_TOOL_CALL_ID]?: string; + [AI_SDK_TOOL_CAPTURE]?: { recordInputs?: boolean; recordOutputs?: boolean }; +}; + +export function currentAISDKToolSpan(toolCallId?: string): Span | undefined { + const span = currentSpan() as AISDKToolSpan; + return span.spanId && + toolCallId !== undefined && + span[AI_SDK_TOOL_CALL_ID] === toolCallId + ? span + : undefined; +} + +export function traceAISDKTool( + execute: () => T, + { + name, + input, + toolCallId, + parent, + type = SpanTypeAttribute.TOOL, + }: { + name: string; + input: unknown; + toolCallId?: string; + parent?: Span; + type?: SpanTypeAttribute; + }, +): T { + if (currentAISDKToolSpan(toolCallId)) return execute(); + const args = withSpanInstrumentationName( + { + name, + spanAttributes: { type }, + }, + INSTRUMENTATION_NAMES.AI_SDK, + ); + const span = ( + parent ? parent.startSpan(args) : startSpan(args) + ) as AISDKToolSpan; + span[AI_SDK_TOOL_CALL_ID] = toolCallId; + const captureParent = (parent ?? currentSpan()) as AISDKToolSpan; + const capture = captureParent.spanId + ? captureParent[AI_SDK_TOOL_CAPTURE] + : undefined; + span[AI_SDK_TOOL_CAPTURE] = capture; + span.log({ + ...(capture?.recordInputs === false ? {} : { input }), + metadata: { + ...(toolCallId === undefined ? {} : { toolCallId }), + }, + }); + const fail = (error: unknown) => { + span.log({ error }); + span.end(); + }; + const finish = (output: unknown) => { + if (capture?.recordOutputs !== false) span.log({ output }); + span.end(); + }; + try { + const result = withCurrent(span, execute); + return observeResult( + result, + (output) => { + if (isAsyncIterable(output)) { + let last: unknown; + patchStreamIfNeeded(output, { + shouldCollect: (chunk) => { + last = chunk; + return false; + }, + onComplete: () => finish(last), + onCancel: () => finish(last), + onError: fail, + aroundNext: (next) => withCurrent(span, next), + }); + } else { + finish(output); + } + }, + fail, + ); + } catch (error) { + fail(error); + throw error; + } +} diff --git a/js/src/vendor-sdk-types/ai-sdk-code-mode.ts b/js/src/vendor-sdk-types/ai-sdk-code-mode.ts new file mode 100644 index 000000000..d3f119175 --- /dev/null +++ b/js/src/vendor-sdk-types/ai-sdk-code-mode.ts @@ -0,0 +1,27 @@ +/** The execution surface of @ai-sdk/code-mode 1.x. */ +export interface AISDKCodeModeExecutionOptions { + toolCallId?: string; + [key: string]: unknown; +} + +export interface AISDKCodeModeTool { + needsApproval?: + | boolean + | (( + input: unknown, + options: AISDKCodeModeExecutionOptions, + ) => boolean | PromiseLike); + execute?: (input: unknown, options: AISDKCodeModeExecutionOptions) => unknown; + [key: string]: unknown; +} + +export interface AISDKRunCodeModeInput { + js: string; + tools: Record; + toolExecutionOptions?: AISDKCodeModeExecutionOptions; + [key: string]: unknown; +} + +export type AISDKRunCodeMode = ( + input: AISDKRunCodeModeInput, +) => Promise; diff --git a/js/src/vendor-sdk-types/ai-sdk-v7-telemetry.ts b/js/src/vendor-sdk-types/ai-sdk-v7-telemetry.ts index d349ab337..b1ca126a2 100644 --- a/js/src/vendor-sdk-types/ai-sdk-v7-telemetry.ts +++ b/js/src/vendor-sdk-types/ai-sdk-v7-telemetry.ts @@ -17,6 +17,15 @@ export interface AISDKV7TelemetryOptions { functionId?: string; } +export const BRAINTRUST_AI_SDK_V7_TELEMETRY = Symbol.for( + "braintrust.ai-sdk.v7.telemetry", +); + +/** The registry read by AI SDK's getGlobalTelemetryIntegrations(). */ +export interface AISDKV7TelemetryGlobal { + AI_SDK_TELEMETRY_INTEGRATIONS?: unknown[]; +} + export const BRAINTRUST_AI_SDK_V7_OPERATION_KEY = Symbol.for( "braintrust.ai-sdk.v7.telemetry-operation-key", ); diff --git a/js/src/wrappers/ai-sdk/ai-sdk.ts b/js/src/wrappers/ai-sdk/ai-sdk.ts index 3a339f1a7..86a5d874d 100644 --- a/js/src/wrappers/ai-sdk/ai-sdk.ts +++ b/js/src/wrappers/ai-sdk/ai-sdk.ts @@ -3,6 +3,7 @@ import { SpanTypeAttribute } from "../../../util"; import { aiSDKChannels, + codeModeChannels, BRAINTRUST_WRAPPED_AI_SDK_MODEL, harnessAgentChannels, } from "../../instrumentation/plugins/ai-sdk-interceptors"; @@ -29,6 +30,10 @@ import type { AISDKStreamFunction, AISDKWorkflowAgentClass, } from "../../vendor-sdk-types/ai-sdk"; +import type { + AISDKRunCodeMode, + AISDKRunCodeModeInput, +} from "../../vendor-sdk-types/ai-sdk-code-mode"; import { currentWorkflowAgentWrapperSpan } from "./workflow-agent-context"; interface WrapAISDKOptions { @@ -92,6 +97,10 @@ function isModuleNamespace(obj: unknown): obj is AISDKNamespaceObject { /** * Wraps Vercel AI SDK methods with Braintrust tracing. * + * For code-mode without a loader hook, wrap the code-mode namespace and call + * `wrapAISDK(codeMode).experimental_runCodeMode(...)`. Factory-created tools + * call runCodeMode internally and require auto-instrumentation. + * * @param ai - The AI SDK namespace (e.g., import * as ai from "ai") * @returns AI SDK with Braintrust tracing. * @@ -137,6 +146,17 @@ export function wrapAISDK(aiSDK: T, options: WrapAISDKOptions = {}): T { get(target, prop, receiver) { const original = Reflect.get(target, prop, receiver); switch (prop) { + case "experimental_runCodeMode": + return typeof original === "function" + ? function (this: unknown, input: AISDKRunCodeModeInput) { + return codeModeChannels.run.invoke( + original as AISDKRunCodeMode, + this, + [input], + {}, + ); + } + : original; case "experimental_decide": case "experimental_evaluate": { const evaluate = typedAISDK[prop]; diff --git a/js/src/wrappers/ai-sdk/telemetry.ts b/js/src/wrappers/ai-sdk/telemetry.ts index 12f60a9f2..7fee1fe9d 100644 --- a/js/src/wrappers/ai-sdk/telemetry.ts +++ b/js/src/wrappers/ai-sdk/telemetry.ts @@ -1,4 +1,9 @@ import { debugLogger } from "../../debug-logger"; +import { + AI_SDK_TOOL_CALL_ID, + AI_SDK_TOOL_CAPTURE, + type AISDKToolSpan, +} from "../../instrumentation/plugins/ai-sdk-tool-tracing"; import { isAutoInstrumentationSuppressed } from "../../instrumentation/auto-instrumentation-suppression"; import { SpanTypeAttribute, @@ -41,7 +46,10 @@ import type { AISDKV7Telemetry, AISDKV7TelemetryOptions, } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; -import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; +import { + BRAINTRUST_AI_SDK_V7_TELEMETRY, + BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY, +} from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; import { currentWorkflowAgentWrapperSpan } from "./workflow-agent-context"; import { currentHarnessTurnParent, @@ -537,6 +545,7 @@ export function braintrustAISDKTelemetry(): any { }; return { + [BRAINTRUST_AI_SDK_V7_TELEMETRY]: true, experimental_onEvaluateStart: onDecisionStart, experimental_onEvaluateEnd: onEnd, experimental_onDecideStart: onDecisionStart, @@ -962,6 +971,11 @@ export function braintrustAISDKTelemetry(): any { }, state?.harnessTurnParent, ); + (span as AISDKToolSpan)[AI_SDK_TOOL_CALL_ID] = toolCallId; + (span as AISDKToolSpan)[AI_SDK_TOOL_CAPTURE] = { + recordInputs: shouldRecordInputs(event), + recordOutputs: shouldRecordOutputs(event), + }; toolSpans.set(toolCallId, { operationKey, span }); }); }, @@ -1070,7 +1084,7 @@ export function braintrustAISDKTelemetry(): any { const state = toolSpans.get(toolCallId); return state ? withCurrent(state.span, () => execute()) : execute(); }, - } satisfies AISDKV7Telemetry; + } satisfies AISDKV7Telemetry & { [BRAINTRUST_AI_SDK_V7_TELEMETRY]: true }; } function shouldRecordInputs(event: AISDKV7TelemetryOptions): boolean {