From b8dd0d6ee3fdfcbcc3a08b4236a00239585e6083 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 24 Sep 2026 12:21:52 +0000 Subject: [PATCH 01/10] feat(providers/openai): keep web search queries and sources OpenAI now reports search queries in web_search_call action.queries and deprecated action.query. The metadata only kept query, so callers lost the queries of current responses. action.sources, the URLs a search consulted, is only returned when the request includes web_search_call.action.sources, so requests with the web_search tool now add it. When streaming, response.output_item.done can omit sources or list only some of them; response.completed carries the full list. The web_search result is now emitted at the terminal event with that action data. The TestOpenAIWebSearch cassettes were re-recorded because the request body changed; they show done items without sources. --- providers/openai/responses_language_model.go | 113 +++-- providers/openai/responses_options.go | 13 +- providers/openai/responses_web_search_test.go | 391 ++++++++++++++++++ providertests/openai_web_search_test.go | 27 ++ .../TestOpenAIWebSearch/generate.yaml | 101 +++-- .../testdata/TestOpenAIWebSearch/stream.yaml | 82 ++-- 6 files changed, 624 insertions(+), 103 deletions(-) create mode 100644 providers/openai/responses_web_search_test.go diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index bcb432ebd..358161b65 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -330,14 +330,6 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res params.Truncation = responses.ResponseNewParamsTruncationAuto } - if len(include) > 0 { - includeParams := make([]responses.ResponseIncludable, len(include)) - for i, inc := range include { - includeParams[i] = responses.ResponseIncludable(string(inc)) - } - params.Include = includeParams - } - if modelConfig.isReasoningModel { if call.Temperature != nil { params.Temperature = param.Opt[float64]{} @@ -404,6 +396,20 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res params.ToolChoice = toolChoice } + // web_search_call items only carry action.sources, the full list of + // URLs the search consulted, when the request asks for them. + if hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { + include = append(include, IncludeWebSearchCallActionSources) + } + + if len(include) > 0 { + includeParams := make([]responses.ResponseIncludable, len(include)) + for i, inc := range include { + includeParams[i] = responses.ResponseIncludable(string(inc)) + } + params.Include = includeParams + } + return params, warnings, nil } @@ -887,6 +893,12 @@ func effectiveResponseStore(requested bool, response *responses.Response) bool { return requested && response.JSON.ExtraFields["store"].Raw() != "false" } +func hasResponsesWebSearchTool(tools []responses.ToolUnionParam) bool { + return slices.ContainsFunc(tools, func(tool responses.ToolUnionParam) bool { + return tool.OfWebSearch != nil || tool.OfWebSearchPreview != nil + }) +} + func isResponsesWebSearchToolCall(toolCallPart fantasy.ToolCallPart) bool { return toolCallPart.ToolName == "web_search" || toolCallPart.ToolName == "web_search_preview" @@ -1191,8 +1203,9 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call) // matching the vercel/ai pattern for provider tools. // // Note: source citations come from url_citation annotations - // on the message text (handled in the "message" case above), - // not from the web_search_call action. + // on the message text (handled in the "message" case above). + // The URLs the search consulted stay in the result metadata + // so callers can tell them apart from citations. wsMeta := webSearchCallToMetadata(outputItem.ID, outputItem.Action) content = append(content, fantasy.ToolCallContent{ ProviderExecuted: true, @@ -1330,6 +1343,39 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( sourceStoreEnabled := params.Store.Value return func(yield func(fantasy.StreamPart) bool) { + // pendingWebSearchResults holds web_search results, in call + // order, until a terminal event supplies the final action data. + var pendingWebSearchResults []*WebSearchCallMetadata + // flushWebSearchResults emits the pending web_search results, + // preferring the action data from the terminal response output + // over the data captured when each item finished. + flushWebSearchResults := func(output []responses.ResponseOutputItemUnion) bool { + pending := pendingWebSearchResults + pendingWebSearchResults = nil + for _, meta := range pending { + for _, item := range output { + if item.Type == "web_search_call" && item.ID == meta.ItemID { + meta = webSearchCallToMetadata(item.ID, item.Action) + break + } + } + // Emit a ToolResult so the agent framework includes + // it in round-trip messages. + if !yield(fantasy.StreamPart{ + Type: fantasy.StreamPartTypeToolResult, + ID: meta.ItemID, + ToolCallName: "web_search", + ProviderExecuted: true, + ProviderMetadata: fantasy.ProviderMetadata{ + Name: meta, + }, + }) { + return false + } + } + return true + } + if len(warnings) > 0 { if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeWarnings, @@ -1446,9 +1492,13 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } case "web_search_call": - // Provider-executed web search completed. - // Source citations come from url_citation annotations - // on the streamed message text, not from the action. + // Provider-executed web search completed. Source + // citations come from url_citation annotations on the + // streamed message text. The URLs the search consulted + // stay in the result metadata, and the result waits + // for the terminal event because this item can omit + // action.sources, or list only some of them, while + // response.completed carries the full list. if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputEnd, ID: done.Item.ID, @@ -1463,19 +1513,7 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( }) { return } - // Emit a ToolResult so the agent framework - // includes it in round-trip messages. - if !yield(fantasy.StreamPart{ - Type: fantasy.StreamPartTypeToolResult, - ID: done.Item.ID, - ToolCallName: "web_search", - ProviderExecuted: true, - ProviderMetadata: fantasy.ProviderMetadata{ - Name: webSearchCallToMetadata(done.Item.ID, done.Item.Action), - }, - }) { - return - } + pendingWebSearchResults = append(pendingWebSearchResults, webSearchCallToMetadata(done.Item.ID, done.Item.Action)) case "message": if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeTextEnd, @@ -1638,6 +1676,9 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( responseID = completed.Response.ID finishReason = mapResponsesFinishReason(completed.Response.IncompleteDetails.Reason, hasFunctionCall) usage = responsesUsage(completed.Response) + if !flushWebSearchResults(completed.Response.Output) { + return + } case "response.incomplete": sawTerminalEvent = true @@ -1645,9 +1686,15 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( responseID = incomplete.Response.ID finishReason = mapResponsesFinishReason(incomplete.Response.IncompleteDetails.Reason, hasFunctionCall) usage = responsesUsage(incomplete.Response) + if !flushWebSearchResults(incomplete.Response.Output) { + return + } case "response.failed": failed := event.AsResponseFailed() + if !flushWebSearchResults(nil) { + return + } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeError, Error: responsesFailedStreamError(failed.Response.Error.Message, string(failed.Response.Error.Code)), @@ -1658,6 +1705,9 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( case "error": errorEvent := event.AsError() + if !flushWebSearchResults(nil) { + return + } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeError, Error: responsesErrorStreamError(errorEvent.Message, errorEvent.Code), @@ -1668,6 +1718,12 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } } + // Every emitted web_search tool call keeps its paired result, + // even when the stream ends without a terminal event. + if !flushWebSearchResults(nil) { + return + } + err := stream.Err() if err != nil && !errors.Is(err, io.EOF) { yield(fantasy.StreamPart{ @@ -1770,8 +1826,9 @@ func webSearchCallToMetadata(itemID string, action responses.ResponseOutputItemU meta := &WebSearchCallMetadata{ItemID: itemID} if action.Type != "" { a := &WebSearchAction{ - Type: action.Type, - Query: action.Query, + Type: action.Type, + Queries: action.Queries, + Query: action.Query, } for _, src := range action.Sources { a.Sources = append(a.Sources, WebSearchSource{ diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index 3cd7875fb..fa357433b 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -188,6 +188,10 @@ const ( IncludeFileSearchCallResults IncludeType = "file_search_call.results" // IncludeMessageOutputTextLogprobs includes message output text log probabilities. IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" + // IncludeWebSearchCallActionSources includes the URLs a web search + // consulted in web_search_call action.sources. Requests that carry the + // web_search tool add it automatically. + IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" ) // ServiceTier represents the service tier for OpenAI Responses API. @@ -450,9 +454,14 @@ type WebSearchSource struct { type WebSearchAction struct { // Type is the kind of action: "search", "open_page", or "find". Type string `json:"type"` - // Query is the search query (present when Type is "search"). + // Queries are the search queries (present when Type is "search", + // though OpenAI does not always report them). + Queries []string `json:"queries,omitempty"` + // Query is the single search query that older responses report + // instead of Queries. OpenAI deprecated it in favor of Queries. Query string `json:"query,omitempty"` - // Sources are the results returned by the search. + // Sources are the URLs the search consulted. OpenAI only returns + // them when the request includes web_search_call.action.sources. Sources []WebSearchSource `json:"sources,omitempty"` } diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go new file mode 100644 index 000000000..38e25a71f --- /dev/null +++ b/providers/openai/responses_web_search_test.go @@ -0,0 +1,391 @@ +package openai + +import ( + "context" + "encoding/json" + "slices" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" +) + +// webSearchCallActionCases covers the web_search_call action shapes +// OpenAI returns: current responses with queries and consulted sources, +// older responses with the deprecated single query, and searches that +// report no query at all. +var webSearchCallActionCases = []struct { + name string + action map[string]any + want *WebSearchAction +}{ + { + name: "QueriesAndSources", + action: map[string]any{ + "type": "search", + "queries": []any{"coder agents web search", "openai responses sources"}, + "sources": []any{ + map[string]any{"type": "url", "url": "https://coder.com/docs/ai-coder"}, + map[string]any{"type": "url", "url": "https://developers.openai.com/api/docs/guides/tools-web-search"}, + }, + }, + want: &WebSearchAction{ + Type: "search", + Queries: []string{"coder agents web search", "openai responses sources"}, + Sources: []WebSearchSource{ + {Type: "url", URL: "https://coder.com/docs/ai-coder"}, + {Type: "url", URL: "https://developers.openai.com/api/docs/guides/tools-web-search"}, + }, + }, + }, + { + name: "DeprecatedQuery", + action: map[string]any{ + "type": "search", + "query": "latest AI news", + }, + want: &WebSearchAction{ + Type: "search", + Query: "latest AI news", + }, + }, + { + name: "NoQueries", + action: map[string]any{ + "type": "search", + "sources": []any{ + map[string]any{"type": "url", "url": "https://example.com/weather"}, + }, + }, + want: &WebSearchAction{ + Type: "search", + Sources: []WebSearchSource{ + {Type: "url", URL: "https://example.com/weather"}, + }, + }, + }, +} + +func requireWebSearchCallMetadata(t *testing.T, metadata fantasy.ProviderMetadata, want *WebSearchAction) { + t.Helper() + + wsMeta, ok := metadata[Name].(*WebSearchCallMetadata) + require.True(t, ok, "metadata should be *WebSearchCallMetadata, got %T", metadata[Name]) + require.Equal(t, "ws_01", wsMeta.ItemID) + require.Equal(t, want, wsMeta.Action) + + // Callers persist the metadata as JSON and read it back on later + // turns, so the action fields must survive the round trip. + encoded, err := json.Marshal(metadata) + require.NoError(t, err) + var raw map[string]json.RawMessage + require.NoError(t, json.Unmarshal(encoded, &raw)) + decoded, err := fantasy.UnmarshalProviderMetadata(raw) + require.NoError(t, err) + roundTripped, ok := decoded[Name].(*WebSearchCallMetadata) + require.True(t, ok, "decoded metadata should be *WebSearchCallMetadata, got %T", decoded[Name]) + require.Equal(t, wsMeta, roundTripped) +} + +func requireWebSearchSourcesInclude(t *testing.T, body map[string]any) { + t.Helper() + + include, ok := body["include"].([]any) + require.True(t, ok, "request body should have an include array, got %#v", body["include"]) + count := 0 + for _, value := range include { + if value == string(IncludeWebSearchCallActionSources) { + count++ + } + } + require.Equal(t, 1, count, "include should request web_search_call.action.sources exactly once: %v", include) +} + +func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { + t.Parallel() + + for _, tc := range webSearchCallActionCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server := newMockServer() + defer server.close() + server.response = map[string]any{ + "id": "resp_01", + "object": "response", + "model": "gpt-4.1", + "status": "completed", + "output": []any{ + map[string]any{ + "type": "web_search_call", + "id": "ws_01", + "status": "completed", + "action": tc.action, + }, + map[string]any{ + "type": "message", + "id": "msg_01", + "role": "assistant", + "status": "completed", + "content": []any{ + map[string]any{ + "type": "output_text", + "text": "Answer.", + "annotations": []any{ + map[string]any{ + "type": "url_citation", + "url": "https://example.com/cited", + "title": "Cited", + "start_index": 0, + "end_index": 7, + }, + }, + }, + }, + }, + }, + "usage": map[string]any{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + + model := newResponsesProvider(t, server.server.URL) + resp, err := model.Generate(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + require.Len(t, server.calls, 1) + requireWebSearchSourcesInclude(t, server.calls[0].body) + + var ( + toolResults []fantasy.ToolResultContent + sources []fantasy.SourceContent + ) + for _, c := range resp.Content { + switch v := c.(type) { + case fantasy.ToolResultContent: + toolResults = append(toolResults, v) + case fantasy.SourceContent: + sources = append(sources, v) + } + } + + require.Len(t, toolResults, 1) + require.True(t, toolResults[0].ProviderExecuted) + require.Equal(t, "web_search", toolResults[0].ToolName) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.want) + + // Consulted sources must not become citation sources. + require.Len(t, sources, 1) + require.Equal(t, "https://example.com/cited", sources[0].URL) + }) + } +} + +func TestResponsesStream_WebSearchCallAction(t *testing.T) { + t.Parallel() + + for _, tc := range webSearchCallActionCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + item, err := json.Marshal(map[string]any{ + "type": "web_search_call", + "id": "ws_01", + "status": "completed", + "action": tc.action, + }) + require.NoError(t, err) + + sms := newStreamingMockServer() + defer sms.close() + sms.chunks = []string{ + responsesSSEEvent("response.output_item.added", + `{"type":"response.output_item.added","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"in_progress"}}`), + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":0,"item":`+string(item)+`}`), + responsesSSEEvent("response.output_item.added", + `{"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"in_progress","content":[]}}`), + responsesSSEEvent("response.output_text.delta", + `{"type":"response.output_text.delta","item_id":"msg_01","output_index":1,"content_index":0,"delta":"Answer."}`), + responsesSSEEvent("response.output_text.annotation.added", + `{"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","url":"https://example.com/cited","title":"Cited","start_index":0,"end_index":7},"annotation_index":0,"content_index":0,"item_id":"msg_01","output_index":1,"sequence_number":5}`), + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Answer.","annotations":[]}]}}`), + responsesSSEEvent("response.completed", + `{"type":"response.completed","response":{"id":"resp_01","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`), + } + + model := newResponsesProvider(t, sms.server.URL) + stream, err := model.Stream(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + + var toolResults, sources []fantasy.StreamPart + for part := range stream { + require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) + switch part.Type { + case fantasy.StreamPartTypeToolResult: + toolResults = append(toolResults, part) + case fantasy.StreamPartTypeSource: + sources = append(sources, part) + } + } + require.Len(t, sms.calls, 1) + requireWebSearchSourcesInclude(t, sms.calls[0].body) + + require.Len(t, toolResults, 1) + require.True(t, toolResults[0].ProviderExecuted) + require.Equal(t, "web_search", toolResults[0].ToolCallName) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.want) + + // Consulted sources must not become citation sources. + require.Len(t, sources, 1) + require.Equal(t, "https://example.com/cited", sources[0].URL) + }) + } +} + +// TestResponsesStream_WebSearchCallSourcesFromCompletedResponse covers the +// recorded OpenAI behavior where response.output_item.done omits +// action.sources and only response.completed lists them. +func TestResponsesStream_WebSearchCallSourcesFromCompletedResponse(t *testing.T) { + t.Parallel() + + sms := newStreamingMockServer() + defer sms.close() + sms.chunks = []string{ + responsesSSEEvent("response.output_item.added", + `{"type":"response.output_item.added","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"in_progress","action":{"type":"search"}}}`), + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"query":"tokyo population"}}}`), + responsesSSEEvent("response.output_item.added", + `{"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"in_progress","content":[]}}`), + responsesSSEEvent("response.output_text.delta", + `{"type":"response.output_text.delta","item_id":"msg_01","output_index":1,"content_index":0,"delta":"About 14 million."}`), + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"completed","content":[{"type":"output_text","text":"About 14 million.","annotations":[]}]}}`), + responsesSSEEvent("response.completed", + `{"type":"response.completed","response":{"id":"resp_01","status":"completed","output":[`+ + `{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"query":"tokyo population","sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"},{"type":"url","url":"https://example.com/tokyo"}]}},`+ + `{"type":"message","id":"msg_01","role":"assistant","status":"completed","content":[{"type":"output_text","text":"About 14 million.","annotations":[]}]}`+ + `],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`), + } + + model := newResponsesProvider(t, sms.server.URL) + stream, err := model.Stream(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + + var types []fantasy.StreamPartType + var toolResults []fantasy.StreamPart + for part := range stream { + require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) + types = append(types, part.Type) + if part.Type == fantasy.StreamPartTypeToolResult { + toolResults = append(toolResults, part) + } + } + + // The call keeps its position ahead of the answer text; the + // result follows once the terminal event arrives. + require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolCall), slices.Index(types, fantasy.StreamPartTypeTextDelta)) + require.Less(t, slices.Index(types, fantasy.StreamPartTypeTextDelta), slices.Index(types, fantasy.StreamPartTypeToolResult)) + require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolResult), slices.Index(types, fantasy.StreamPartTypeFinish)) + + require.Len(t, toolResults, 1) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ + Type: "search", + Queries: []string{"tokyo population"}, + Query: "tokyo population", + Sources: []WebSearchSource{ + {Type: "url", URL: "https://www.metro.tokyo.lg.jp/"}, + {Type: "url", URL: "https://example.com/tokyo"}, + }, + }) +} + +func TestResponsesStream_WebSearchCallResultWithoutTerminalEvent(t *testing.T) { + t.Parallel() + + sms := newStreamingMockServer() + defer sms.close() + sms.chunks = []string{ + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"]}}}`), + } + + model := newResponsesProvider(t, sms.server.URL) + stream, err := model.Stream(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + + var toolResults, errs []fantasy.StreamPart + for part := range stream { + switch part.Type { + case fantasy.StreamPartTypeToolResult: + require.Empty(t, errs, "the paired result must precede the stream error") + toolResults = append(toolResults, part) + case fantasy.StreamPartTypeError: + errs = append(errs, part) + } + } + + require.Len(t, errs, 1) + require.Len(t, toolResults, 1) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ + Type: "search", + Queries: []string{"tokyo population"}, + }) +} + +func TestResponsesWebSearchSourcesInclude(t *testing.T) { + t.Parallel() + + t.Run("NotRequestedWithoutWebSearchTool", func(t *testing.T) { + t.Parallel() + + server := newMockServer() + defer server.close() + server.response = mockResponsesWebSearchResponse() + + model := newResponsesProvider(t, server.server.URL) + _, err := model.Generate(context.Background(), fantasy.Call{Prompt: testPrompt}) + require.NoError(t, err) + require.Len(t, server.calls, 1) + + include, _ := server.calls[0].body["include"].([]any) + require.NotContains(t, include, string(IncludeWebSearchCallActionSources)) + }) + + t.Run("KeepsCallerIncludesWithoutDuplicates", func(t *testing.T) { + t.Parallel() + + server := newMockServer() + defer server.close() + server.response = mockResponsesWebSearchResponse() + + model := newResponsesProvider(t, server.server.URL) + _, err := model.Generate(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + ProviderOptions: fantasy.ProviderOptions{ + Name: &ResponsesProviderOptions{ + Include: []IncludeType{ + IncludeReasoningEncryptedContent, + IncludeWebSearchCallActionSources, + }, + }, + }, + }) + require.NoError(t, err) + require.Len(t, server.calls, 1) + requireWebSearchSourcesInclude(t, server.calls[0].body) + require.Contains(t, server.calls[0].body["include"], string(IncludeReasoningEncryptedContent)) + }) +} diff --git a/providertests/openai_web_search_test.go b/providertests/openai_web_search_test.go index 8e8eb50e2..3dbda5268 100644 --- a/providertests/openai_web_search_test.go +++ b/providertests/openai_web_search_test.go @@ -58,6 +58,7 @@ func TestOpenAIWebSearch(t *testing.T) { // Walk the steps and verify web search content was produced. var sources []fantasy.SourceContent var providerToolCalls []fantasy.ToolCallContent + var providerToolResults []fantasy.ToolResultContent for _, step := range result.Steps { for _, c := range step.Content { switch v := c.(type) { @@ -65,6 +66,10 @@ func TestOpenAIWebSearch(t *testing.T) { if v.ProviderExecuted { providerToolCalls = append(providerToolCalls, v) } + case fantasy.ToolResultContent: + if v.ProviderExecuted { + providerToolResults = append(providerToolResults, v) + } case fantasy.SourceContent: sources = append(sources, v) } @@ -73,6 +78,7 @@ func TestOpenAIWebSearch(t *testing.T) { require.NotEmpty(t, providerToolCalls, "should have provider-executed tool calls") require.Equal(t, "web_search", providerToolCalls[0].ToolName) + requireWebSearchActionMetadata(t, providerToolResults) // Sources come from url_citation annotations; the model // may or may not include inline citations so we don't // require them, but if present they should have URLs. @@ -124,5 +130,26 @@ func TestOpenAIWebSearch(t *testing.T) { require.NotEmpty(t, providerToolCalls, "should have provider-executed tool calls") require.Equal(t, "web_search", providerToolCalls[0].ToolName) require.NotEmpty(t, providerToolResults, "should have provider-executed tool results") + requireWebSearchActionMetadata(t, providerToolResults) }) } + +// requireWebSearchActionMetadata checks that web_search results carry the +// search queries and the consulted source URLs, which OpenAI only returns +// when the request includes web_search_call.action.sources. +func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultContent) { + t.Helper() + + require.NotEmpty(t, results, "should have provider-executed tool results") + for _, result := range results { + meta, ok := result.ProviderMetadata[openai.Name].(*openai.WebSearchCallMetadata) + require.True(t, ok, "web_search result metadata should be *openai.WebSearchCallMetadata, got %T", result.ProviderMetadata[openai.Name]) + require.NotNil(t, meta.Action) + require.Equal(t, "search", meta.Action.Type) + require.NotEmpty(t, meta.Action.Queries, "search action should report its queries") + require.NotEmpty(t, meta.Action.Sources, "search action should report its consulted sources") + for _, source := range meta.Action.Sources { + require.NotEmpty(t, source.URL, "consulted source should have a URL") + } + } +} diff --git a/providertests/testdata/TestOpenAIWebSearch/generate.yaml b/providertests/testdata/TestOpenAIWebSearch/generate.yaml index 8591a4402..078f0bd9e 100644 --- a/providertests/testdata/TestOpenAIWebSearch/generate.yaml +++ b/providertests/testdata/TestOpenAIWebSearch/generate.yaml @@ -6,16 +6,16 @@ interactions: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 297 + content_length: 342 host: "" - body: '{"max_output_tokens":4000,"store":false,"input":[{"content":"You are a helpful assistant","role":"system"},{"content":[{"text":"What is the current population of Tokyo? Cite your source.","type":"input_text"}],"role":"user"}],"model":"gpt-4.1","tool_choice":"auto","tools":[{"type":"web_search"}]}' + body: '{"max_output_tokens":4000,"store":false,"include":["web_search_call.action.sources"],"input":[{"content":"You are a helpful assistant","role":"system"},{"content":[{"text":"What is the current population of Tokyo? Cite your source.","type":"input_text"}],"role":"user"}],"model":"gpt-4.1","tool_choice":"auto","tools":[{"type":"web_search"}]}' headers: Accept: - application/json Content-Type: - application/json User-Agent: - - Charm-Fantasy/0.33.1 (https://charm.land/fantasy) + - Charm-Fantasy/0.40.0 (https://charm.land/fantasy) url: https://api.openai.com/v1/responses method: POST response: @@ -23,18 +23,18 @@ interactions: proto_major: 2 proto_minor: 0 content_length: -1 - uncompressed: true body: |- { - "id": "resp_032814a9038cfb27016a3c13acb58c8192925156b4193041ea", + "id": "resp_0dd899f98a0f9334016ab514af517087d28fa5bcd2976919d3", "object": "response", - "created_at": 1782322092, + "created_at": 1790252207, "status": "completed", + "access_programs": null, "background": false, "billing": { "payer": "developer" }, - "completed_at": 1782322099, + "completed_at": 1790252211, "error": null, "frequency_penalty": 0.0, "incomplete_details": null, @@ -45,7 +45,7 @@ interactions: "moderation": null, "output": [ { - "id": "ws_032814a9038cfb27016a3c13ad26608192bf08bba827aa04d2", + "id": "ws_0dd899f98a0f9334016ab514aff05c87d2a7c90547b95a92c3", "type": "web_search_call", "status": "completed", "action": { @@ -53,11 +53,25 @@ interactions: "queries": [ "current population of Tokyo 2024" ], - "query": "current population of Tokyo 2024" + "query": "current population of Tokyo 2024", + "sources": [ + { + "type": "url", + "url": "https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai" + }, + { + "type": "url", + "url": "https://www.tokyoupdates.metro.tokyo.lg.jp/en/brochure2026/pdf/tokyo2026.pdf?utm_source=openai" + }, + { + "type": "url", + "url": "https://www.rns-pdf.londonstockexchange.com/rns/4629B_1-2026-4-22.pdf?utm_source=openai" + } + ] } }, { - "id": "msg_032814a9038cfb27016a3c13b1005481928ac017f3e5b8f7eb", + "id": "msg_0dd899f98a0f9334016ab514b1e92c87d2b5a11e70fa737058", "type": "message", "status": "completed", "content": [ @@ -66,42 +80,28 @@ interactions: "annotations": [ { "type": "url_citation", - "end_index": 474, - "start_index": 367, - "title": "\u4f4f\u6c11\u57fa\u672c\u53f0\u5e33\u306b\u3088\u308b\u4e16\u5e2f\u3068\u4eba\u53e3 7\u5e741\u6708\u73fe\u5728|3\u6708|\u90fd\u5e81\u7dcf\u5408\u30db\u30fc\u30e0\u30da\u30fc\u30b8", - "url": "https://www.metro.tokyo.lg.jp/information/press/2025/03/2025031902?utm_source=openai" + "end_index": 359, + "start_index": 252, + "title": "\u4ee4\u548c7\u5e74\u56fd\u52e2\u8abf\u67fb \u4eba\u53e3\u53ca\u3073\u4e16\u5e2f\u6570\uff08\u901f\u5831\uff09|5\u6708|\u90fd\u5e81\u7dcf\u5408\u30db\u30fc\u30e0\u30da\u30fc\u30b8", + "url": "https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai" }, { "type": "url_citation", - "end_index": 898, - "start_index": 671, - "title": "Fact Check: The Tokyo Metropolitan Government reports an official population of 14,094,034 residents as of September 2025, encompassing the 23 special wards and Tokyo Prefecture | AIME by AInvest", - "url": "https://www.ainvest.com/aime/share/fact-check-tokyo-metropolitan-government-reports-official-population-14094034-residents-september-2025-encompassing-23-special-wards-tokyo-prefecture-54f325/?utm_source=openai" + "end_index": 650, + "start_index": 520, + "title": "Tokyo Basics", + "url": "https://www.tokyoupdates.metro.tokyo.lg.jp/en/brochure2026/pdf/tokyo2026.pdf?utm_source=openai" }, { "type": "url_citation", - "end_index": 1496, - "start_index": 1395, - "title": "Tokyo Population 2026", - "url": "https://worldpopulationreview.com/cities/japan/tokyo?utm_source=openai" - }, - { - "type": "url_citation", - "end_index": 1817, - "start_index": 1590, - "title": "Fact Check: The Tokyo Metropolitan Government reports an official population of 14,094,034 residents as of September 2025, encompassing the 23 special wards and Tokyo Prefecture | AIME by AInvest", - "url": "https://www.ainvest.com/aime/share/fact-check-tokyo-metropolitan-government-reports-official-population-14094034-residents-september-2025-encompassing-23-special-wards-tokyo-prefecture-54f325/?utm_source=openai" - }, - { - "type": "url_citation", - "end_index": 1988, - "start_index": 1881, - "title": "\u4f4f\u6c11\u57fa\u672c\u53f0\u5e33\u306b\u3088\u308b\u4e16\u5e2f\u3068\u4eba\u53e3 7\u5e741\u6708\u73fe\u5728|3\u6708|\u90fd\u5e81\u7dcf\u5408\u30db\u30fc\u30e0\u30da\u30fc\u30b8", - "url": "https://www.metro.tokyo.lg.jp/information/press/2025/03/2025031902?utm_source=openai" + "end_index": 969, + "start_index": 845, + "title": "THE METROPOLIS OF TOKYO", + "url": "https://www.rns-pdf.londonstockexchange.com/rns/4629B_1-2026-4-22.pdf?utm_source=openai" } ], "logprobs": [], - "text": "Here are the most reliable and up-to-date figures for Tokyo\u2019s current population, based on the latest official data:\n\n\u2022 As of **January 1, 2025**, the **Tokyo Metropolis** (equivalent to Tokyo Prefecture, including the 23 special wards, western Tama area, islands, etc.) had a population of **14,002,534** residents, according to the Basic Resident Register (\u4f4f\u6c11\u57fa\u672c\u53f0\u5e33) ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2025/03/2025031902?utm_source=openai)).\n\n\u2022 Meanwhile, the Tokyo Metropolitan Government\u2019s official \u201cPopulation of Tokyo (estimate)\u201d for **September 1, 2025** reports a slightly higher figure of **14,263,372** for the entire Metropolis ([ainvest.com](https://www.ainvest.com/aime/share/fact-check-tokyo-metropolitan-government-reports-official-population-14094034-residents-september-2025-encompassing-23-special-wards-tokyo-prefecture-54f325/?utm_source=openai)).\n\nBecause these values come from the same source (the Tokyo Metropolitan Government), the September 2025 figure is the most recent official estimate. Therefore, the current population of Tokyo Metropolis is approximately **14.26 million**.\n\nIf you're specifically interested in the population of Tokyo\u2019s central **23 special wards** (sometimes referred to as \u201ccity proper\u201d), that number is considerably lower and typically around 9.7 to 10.3 million depending on the source and estimation method ([worldpopulationreview.com](https://worldpopulationreview.com/cities/japan/tokyo?utm_source=openai)).\n\nIn summary:\n\n- Tokyo Metropolis total population (as of September 1, 2025): **14,263,372** ([ainvest.com](https://www.ainvest.com/aime/share/fact-check-tokyo-metropolitan-government-reports-official-population-14094034-residents-september-2025-encompassing-23-special-wards-tokyo-prefecture-54f325/?utm_source=openai)) \n- Verified earlier data from January 1, 2025: **14,002,534** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2025/03/2025031902?utm_source=openai))\n\nThus, the most current and authoritative estimate is approximately **14.26 million residents**.\n\nLet me know if you\u2019d like a breakdown by wards, the greater metro area figures, or trends over time!" + "text": "Here is the most current and reliable data on Tokyo\u2019s population:\n\n\u2022 According to the preliminary results of Japan\u2019s 2025 national census (conducted as of October 1, 2025), the population of **Tokyo Metropolis** (Tokyo-to) was **14,246,219** residents ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)).\n\n\u2022 A more recent estimate by the Tokyo Metropolitan Government, provided in its FY2025 City Profile, reports the total population as **14,273,066** as of 2025 ([tokyoupdates.metro.tokyo.lg.jp](https://www.tokyoupdates.metro.tokyo.lg.jp/en/brochure2026/pdf/tokyo2026.pdf?utm_source=openai)).\n\n\u2022 Another authority, a financial offering circular published in early 2026, indicates that as of March 1, 2026, Tokyo\u2019s population stood at approximately **14,262,000** (i.e., 14.262 million) ([rns-pdf.londonstockexchange.com](https://www.rns-pdf.londonstockexchange.com/rns/4629B_1-2026-4-22.pdf?utm_source=openai)).\n\nTo summarize:\n\n- **Most authoritative census count** (October 1, 2025): 14,246,219 \n- **Tokyo Metropolitan estimate** (2025): 14,273,066 \n- **Estimate as of March 1, 2026**: approximately 14,262,000 \n\nTherefore, the \u201ccurrent\u201d population of Tokyo (Tokyo Metropolis) is approximately **14.26 million**, based on the most recent estimates available up to early 2026.\n\nLet me know if you'd like population breakdowns by wards, urban agglomeration, or comparisons with Greater Tokyo or other regions!" } ], "role": "assistant" @@ -128,6 +128,24 @@ interactions: "verbosity": "medium" }, "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 1 + } + }, "tools": [ { "type": "web_search", @@ -149,15 +167,16 @@ interactions: "top_p": 1.0, "truncation": "disabled", "usage": { - "input_tokens": 17432, + "input_tokens": 17379, "input_tokens_details": { + "cache_write_tokens": 0, "cached_tokens": 0 }, - "output_tokens": 438, + "output_tokens": 359, "output_tokens_details": { "reasoning_tokens": 0 }, - "total_tokens": 17870 + "total_tokens": 17738 }, "user": null, "metadata": {} @@ -167,4 +186,4 @@ interactions: - application/json status: 200 OK code: 200 - duration: 7.581892042s + duration: 5.243561237s diff --git a/providertests/testdata/TestOpenAIWebSearch/stream.yaml b/providertests/testdata/TestOpenAIWebSearch/stream.yaml index 4d8f9a8e0..51a2b7072 100644 --- a/providertests/testdata/TestOpenAIWebSearch/stream.yaml +++ b/providertests/testdata/TestOpenAIWebSearch/stream.yaml @@ -6,16 +6,16 @@ interactions: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 311 + content_length: 356 host: "" - body: '{"max_output_tokens":4000,"store":false,"input":[{"content":"You are a helpful assistant","role":"system"},{"content":[{"text":"What is the current population of Tokyo? Cite your source.","type":"input_text"}],"role":"user"}],"model":"gpt-4.1","tool_choice":"auto","tools":[{"type":"web_search"}],"stream":true}' + body: '{"max_output_tokens":4000,"store":false,"include":["web_search_call.action.sources"],"input":[{"content":"You are a helpful assistant","role":"system"},{"content":[{"text":"What is the current population of Tokyo? Cite your source.","type":"input_text"}],"role":"user"}],"model":"gpt-4.1","tool_choice":"auto","tools":[{"type":"web_search"}],"stream":true}' headers: Accept: - application/json Content-Type: - application/json User-Agent: - - Charm-Fantasy/0.33.1 (https://charm.land/fantasy) + - Charm-Fantasy/0.40.0 (https://charm.land/fantasy) url: https://api.openai.com/v1/responses method: POST response: @@ -25,89 +25,107 @@ interactions: content_length: -1 body: |+ event: response.created - data: {"type":"response.created","response":{"id":"resp_0200025d411b0585016a3c13b4018c81a3852ca4fd5be2b6a6","object":"response","created_at":1782322100,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + data: {"type":"response.created","response":{"id":"resp_0a0e997357746a3a016ab514b44f4887d292870c0ee0e498f6","object":"response","created_at":1790252212,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} event: response.in_progress - data: {"type":"response.in_progress","response":{"id":"resp_0200025d411b0585016a3c13b4018c81a3852ca4fd5be2b6a6","object":"response","created_at":1782322100,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + data: {"type":"response.in_progress","response":{"id":"resp_0a0e997357746a3a016ab514b44f4887d292870c0ee0e498f6","object":"response","created_at":1790252212,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} event: response.output_item.added - data: {"type":"response.output_item.added","item":{"id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","type":"web_search_call","status":"in_progress","action":{"type":"search"}},"output_index":0,"sequence_number":2} + data: {"type":"response.output_item.added","item":{"id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","type":"web_search_call","status":"in_progress","action":{"type":"search"}},"output_index":0,"sequence_number":2} event: response.web_search_call.in_progress - data: {"type":"response.web_search_call.in_progress","item_id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","output_index":0,"sequence_number":3} + data: {"type":"response.web_search_call.in_progress","item_id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","output_index":0,"sequence_number":3} event: response.web_search_call.searching - data: {"type":"response.web_search_call.searching","item_id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","output_index":0,"sequence_number":4} + data: {"type":"response.web_search_call.searching","item_id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","output_index":0,"sequence_number":4} event: response.web_search_call.completed - data: {"type":"response.web_search_call.completed","item_id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","output_index":0,"sequence_number":5} + data: {"type":"response.web_search_call.completed","item_id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","output_index":0,"sequence_number":5} event: response.output_item.done - data: {"type":"response.output_item.done","item":{"id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","type":"web_search_call","status":"completed","action":{"type":"search","queries":["current population of Tokyo 2024"],"query":"current population of Tokyo 2024"}},"output_index":0,"sequence_number":6} + data: {"type":"response.output_item.done","item":{"id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","type":"web_search_call","status":"completed","action":{"type":"search","queries":["current population of Tokyo 2024"],"query":"current population of Tokyo 2024"}},"output_index":0,"sequence_number":6} event: response.output_item.added - data: {"type":"response.output_item.added","item":{"id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":1,"sequence_number":7} + data: {"type":"response.output_item.added","item":{"id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":1,"sequence_number":7} event: response.content_part.added - data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":8} + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":8} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"Here is the most recent and reliable","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"DQVHD6AboERE","output_index":1,"sequence_number":9} + data: {"type":"response.output_text.delta","content_index":0,"delta":"Here is the most recent and authoritative figure for the population of Tokyo (","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"S3","output_index":1,"sequence_number":9} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":" official estimate for the population of Tokyo:\n\nAs of **May ","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"0Qc","output_index":1,"sequence_number":10} + data: {"type":"response.output_text.delta","content_index":0,"delta":"Tokyo Metropolis):\n\nAccording to the preliminary results of Japan’s 202","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"FLn0ohrWB","output_index":1,"sequence_number":10} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"1, 2026**, the **","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"6cNsksSFn4eHSHI","output_index":1,"sequence_number":11} + data: {"type":"response.output_text.delta","content_index":0,"delta":"5 National Census (conducted on October 1, 2025","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"9","output_index":1,"sequence_number":11} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"Tokyo Metropolitan Government** reports that the total population of","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"uUpi7XJV1fGK","output_index":1,"sequence_number":12} + data: {"type":"response.output_text.delta","content_index":0,"delta":"), the population of Tokyo Metropolis was **14,246,","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"rgnQ257GPcTLD","output_index":1,"sequence_number":12} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":" Tokyo (the entire prefecture) is **","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"IMtFcLkykFF8","output_index":1,"sequence_number":13} + data: {"type":"response.output_text.delta","content_index":0,"delta":"219 people** ","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"oC6","output_index":1,"sequence_number":13} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"14,328,115** people. ","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"Wv8ISvft6Ve","output_index":1,"sequence_number":14} + data: {"type":"response.output_text.delta","content_index":0,"delta":"([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)). This represents the official count for the entire prefect","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"NfrQTqeHNk","output_index":1,"sequence_number":14} + + event: response.output_text.annotation.added + data: {"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","end_index":367,"start_index":260,"title":"令和7年国勢調査 人口及び世帯数（速報）|5月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai"},"annotation_index":0,"content_index":0,"item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","output_index":1,"sequence_number":15} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"ural area, including the 23 special wards, the Tama region,","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"a7SGN","output_index":1,"sequence_number":16} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" and the outlying islands.\n\nFor an alternate administrative snapshot, data based on the **","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"6hlolV","output_index":1,"sequence_number":17} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"([toukei.metro.tokyo.lg.jp](https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai))\n","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"T","output_index":1,"sequence_number":15} + data: {"type":"response.output_text.delta","content_index":0,"delta":"Resident Register (Jūmin Kihon Daichō)** from January 1, ","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"SDYwfMZ","output_index":1,"sequence_number":18} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":"2026, indicates a population of **14,077,552**","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"5r","output_index":1,"sequence_number":19} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai)).","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"RUL","output_index":1,"sequence_number":20} event: response.output_text.annotation.added - data: {"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","end_index":333,"start_index":239,"title":"東京都の統計","url":"https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai"},"annotation_index":0,"content_index":0,"item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","output_index":1,"sequence_number":16} + data: {"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","end_index":786,"start_index":679,"title":"住民基本台帳による東京都の世帯と人口 令和8年1月1日現在|3月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai"},"annotation_index":1,"content_index":0,"item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","output_index":1,"sequence_number":21} + + event: response.output_text.delta + data: {"type":"response.output_text.delta","content_index":0,"delta":" However, the more comprehensive and up-to-date national census figure (14,","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"Pe96p","output_index":1,"sequence_number":22} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"\nThis figure is the latest available from the metropolitan government’s","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"ZRp0P4A9Q","output_index":1,"sequence_number":17} + data: {"type":"response.output_text.delta","content_index":0,"delta":"246,219 as of October 1, 2025) is","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"k4Ddyb6xLiGtojy","output_index":1,"sequence_number":23} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":" statistical department. If you were asking about a different","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"99H","output_index":1,"sequence_number":18} + data: {"type":"response.output_text.delta","content_index":0,"delta":" generally considered the definitive “current” population for Tokyo Metropolis.\n\nThus,","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"lQW2htheOB","output_index":1,"sequence_number":24} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":" area defined as \"Tokyo\" — such as just the ","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"dRK2","output_index":1,"sequence_number":19} + data: {"type":"response.output_text.delta","content_index":0,"delta":" as of the latest official national data, Tokyo Metropolis has approximately **14.","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"GytBLSzMhLm3pP","output_index":1,"sequence_number":25} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"23 special wards (区部) or the","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"zzLe","output_index":1,"sequence_number":20} + data: {"type":"response.output_text.delta","content_index":0,"delta":"25 million** residents.\n\nIf you were referring to another geographic concept—such","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"24RvioEeAoNDsEK","output_index":1,"sequence_number":26} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":" broader metropolitan area — let me know, and I","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"u","output_index":1,"sequence_number":21} + data: {"type":"response.output_text.delta","content_index":0,"delta":" as the population of the 23 special wards or the broader metropolitan ag","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"beIQbRe","output_index":1,"sequence_number":27} event: response.output_text.delta - data: {"type":"response.output_text.delta","content_index":0,"delta":"’d be happy to provide that breakdown as well.","item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"obfuscation":"Hk","output_index":1,"sequence_number":22} + data: {"type":"response.output_text.delta","content_index":0,"delta":"glomeration—let me know, and I can provide those figures too.","item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"obfuscation":"uuD","output_index":1,"sequence_number":28} event: response.output_text.done - data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","logprobs":[],"output_index":1,"sequence_number":23,"text":"Here is the most recent and reliable official estimate for the population of Tokyo:\n\nAs of **May 1, 2026**, the **Tokyo Metropolitan Government** reports that the total population of Tokyo (the entire prefecture) is **14,328,115** people. ([toukei.metro.tokyo.lg.jp](https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai))\n\nThis figure is the latest available from the metropolitan government’s statistical department. If you were asking about a different area defined as \"Tokyo\" — such as just the 23 special wards (区部) or the broader metropolitan area — let me know, and I’d be happy to provide that breakdown as well."} + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","logprobs":[],"output_index":1,"sequence_number":29,"text":"Here is the most recent and authoritative figure for the population of Tokyo (Tokyo Metropolis):\n\nAccording to the preliminary results of Japan’s 2025 National Census (conducted on October 1, 2025), the population of Tokyo Metropolis was **14,246,219 people** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)). This represents the official count for the entire prefectural area, including the 23 special wards, the Tama region, and the outlying islands.\n\nFor an alternate administrative snapshot, data based on the **Resident Register (Jūmin Kihon Daichō)** from January 1, 2026, indicates a population of **14,077,552** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai)). However, the more comprehensive and up-to-date national census figure (14,246,219 as of October 1, 2025) is generally considered the definitive “current” population for Tokyo Metropolis.\n\nThus, as of the latest official national data, Tokyo Metropolis has approximately **14.25 million** residents.\n\nIf you were referring to another geographic concept—such as the population of the 23 special wards or the broader metropolitan agglomeration—let me know, and I can provide those figures too."} event: response.content_part.done - data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","output_index":1,"part":{"type":"output_text","annotations":[{"type":"url_citation","end_index":333,"start_index":239,"title":"東京都の統計","url":"https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and reliable official estimate for the population of Tokyo:\n\nAs of **May 1, 2026**, the **Tokyo Metropolitan Government** reports that the total population of Tokyo (the entire prefecture) is **14,328,115** people. ([toukei.metro.tokyo.lg.jp](https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai))\n\nThis figure is the latest available from the metropolitan government’s statistical department. If you were asking about a different area defined as \"Tokyo\" — such as just the 23 special wards (区部) or the broader metropolitan area — let me know, and I’d be happy to provide that breakdown as well."},"sequence_number":24} + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","output_index":1,"part":{"type":"output_text","annotations":[{"type":"url_citation","end_index":367,"start_index":260,"title":"令和7年国勢調査 人口及び世帯数（速報）|5月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai"},{"type":"url_citation","end_index":786,"start_index":679,"title":"住民基本台帳による東京都の世帯と人口 令和8年1月1日現在|3月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and authoritative figure for the population of Tokyo (Tokyo Metropolis):\n\nAccording to the preliminary results of Japan’s 2025 National Census (conducted on October 1, 2025), the population of Tokyo Metropolis was **14,246,219 people** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)). This represents the official count for the entire prefectural area, including the 23 special wards, the Tama region, and the outlying islands.\n\nFor an alternate administrative snapshot, data based on the **Resident Register (Jūmin Kihon Daichō)** from January 1, 2026, indicates a population of **14,077,552** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai)). However, the more comprehensive and up-to-date national census figure (14,246,219 as of October 1, 2025) is generally considered the definitive “current” population for Tokyo Metropolis.\n\nThus, as of the latest official national data, Tokyo Metropolis has approximately **14.25 million** residents.\n\nIf you were referring to another geographic concept—such as the population of the 23 special wards or the broader metropolitan agglomeration—let me know, and I can provide those figures too."},"sequence_number":30} event: response.output_item.done - data: {"type":"response.output_item.done","item":{"id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":333,"start_index":239,"title":"東京都の統計","url":"https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and reliable official estimate for the population of Tokyo:\n\nAs of **May 1, 2026**, the **Tokyo Metropolitan Government** reports that the total population of Tokyo (the entire prefecture) is **14,328,115** people. ([toukei.metro.tokyo.lg.jp](https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai))\n\nThis figure is the latest available from the metropolitan government’s statistical department. If you were asking about a different area defined as \"Tokyo\" — such as just the 23 special wards (区部) or the broader metropolitan area — let me know, and I’d be happy to provide that breakdown as well."}],"role":"assistant"},"output_index":1,"sequence_number":25} + data: {"type":"response.output_item.done","item":{"id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":367,"start_index":260,"title":"令和7年国勢調査 人口及び世帯数（速報）|5月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai"},{"type":"url_citation","end_index":786,"start_index":679,"title":"住民基本台帳による東京都の世帯と人口 令和8年1月1日現在|3月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and authoritative figure for the population of Tokyo (Tokyo Metropolis):\n\nAccording to the preliminary results of Japan’s 2025 National Census (conducted on October 1, 2025), the population of Tokyo Metropolis was **14,246,219 people** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)). This represents the official count for the entire prefectural area, including the 23 special wards, the Tama region, and the outlying islands.\n\nFor an alternate administrative snapshot, data based on the **Resident Register (Jūmin Kihon Daichō)** from January 1, 2026, indicates a population of **14,077,552** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai)). However, the more comprehensive and up-to-date national census figure (14,246,219 as of October 1, 2025) is generally considered the definitive “current” population for Tokyo Metropolis.\n\nThus, as of the latest official national data, Tokyo Metropolis has approximately **14.25 million** residents.\n\nIf you were referring to another geographic concept—such as the population of the 23 special wards or the broader metropolitan agglomeration—let me know, and I can provide those figures too."}],"role":"assistant"},"output_index":1,"sequence_number":31} event: response.completed - data: {"type":"response.completed","response":{"id":"resp_0200025d411b0585016a3c13b4018c81a3852ca4fd5be2b6a6","object":"response","created_at":1782322100,"status":"completed","background":false,"completed_at":1782322103,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[{"id":"ws_0200025d411b0585016a3c13b4825c81a391999cec332e6f02","type":"web_search_call","status":"completed","action":{"type":"search","queries":["current population of Tokyo 2024"],"query":"current population of Tokyo 2024"}},{"id":"msg_0200025d411b0585016a3c13b6174481a392b9817b2c15a5b4","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":333,"start_index":239,"title":"東京都の統計","url":"https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and reliable official estimate for the population of Tokyo:\n\nAs of **May 1, 2026**, the **Tokyo Metropolitan Government** reports that the total population of Tokyo (the entire prefecture) is **14,328,115** people. ([toukei.metro.tokyo.lg.jp](https://www.toukei.metro.tokyo.lg.jp/index.htm?utm_source=openai))\n\nThis figure is the latest available from the metropolitan government’s statistical department. If you were asking about a different area defined as \"Tokyo\" — such as just the 23 special wards (区部) or the broader metropolitan area — let me know, and I’d be happy to provide that breakdown as well."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":17407,"input_tokens_details":{"cached_tokens":0},"output_tokens":164,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":17571},"user":null,"metadata":{}},"sequence_number":26} + data: {"type":"response.completed","response":{"id":"resp_0a0e997357746a3a016ab514b44f4887d292870c0ee0e498f6","object":"response","created_at":1790252212,"status":"completed","access_programs":null,"background":false,"completed_at":1790252215,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":4000,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","moderation":null,"output":[{"id":"ws_0a0e997357746a3a016ab514b4faa087d2a3e5b27b412be937","type":"web_search_call","status":"completed","action":{"type":"search","queries":["current population of Tokyo 2024"],"query":"current population of Tokyo 2024","sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai"},{"type":"url","url":"https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai"}]}},{"id":"msg_0a0e997357746a3a016ab514b61bc887d288af30ab012e0b2b","type":"message","status":"completed","content":[{"type":"output_text","annotations":[{"type":"url_citation","end_index":367,"start_index":260,"title":"令和7年国勢調査 人口及び世帯数（速報）|5月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai"},{"type":"url_citation","end_index":786,"start_index":679,"title":"住民基本台帳による東京都の世帯と人口 令和8年1月1日現在|3月|都庁総合ホームページ","url":"https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai"}],"logprobs":[],"text":"Here is the most recent and authoritative figure for the population of Tokyo (Tokyo Metropolis):\n\nAccording to the preliminary results of Japan’s 2025 National Census (conducted on October 1, 2025), the population of Tokyo Metropolis was **14,246,219 people** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/05/2026052914?utm_source=openai)). This represents the official count for the entire prefectural area, including the 23 special wards, the Tama region, and the outlying islands.\n\nFor an alternate administrative snapshot, data based on the **Resident Register (Jūmin Kihon Daichō)** from January 1, 2026, indicates a population of **14,077,552** ([metro.tokyo.lg.jp](https://www.metro.tokyo.lg.jp/information/press/2026/03/2026032310?utm_source=openai)). However, the more comprehensive and up-to-date national census figure (14,246,219 as of October 1, 2025) is generally considered the definitive “current” population for Tokyo Metropolis.\n\nThus, as of the latest official national data, Tokyo Metropolis has approximately **14.25 million** residents.\n\nIf you were referring to another geographic concept—such as the population of the 23 special wards or the broader metropolitan agglomeration—let me know, and I can provide those figures too."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":1}},"tools":[{"type":"web_search","return_token_budget":"default","search_content_types":["text"],"search_context_size":"medium","user_location":{"type":"approximate","city":null,"country":"US","region":null,"timezone":null}}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":17221,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":297,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":17518},"user":null,"metadata":{}},"sequence_number":32} headers: Content-Type: - - text/event-stream; charset=utf-8 + - text/event-stream status: 200 OK code: 200 - duration: 444.511042ms + duration: 1.100176876s From 61900cfcf7cb2f64d0dff960cbedcf1ecb27fd00 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 24 Sep 2026 13:22:09 +0000 Subject: [PATCH 02/10] feat(providers/openai): put web search queries in call input The streamed web_search result now waits for the terminal event, so the call emitted at output_item.done was the first signal that the search finished, yet it carried no input. Callers could not show what was searched until the whole answer had streamed. The call input is now {"queries": [...]}, using the deprecated query when queries is absent. A response.failed event also supplies its output when flushing pending results, so its sources are kept. --- providers/openai/responses_language_model.go | 26 +++++++- providers/openai/responses_web_search_test.go | 59 ++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 358161b65..3be13155c 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -1211,6 +1211,7 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call) ProviderExecuted: true, ToolCallID: outputItem.ID, ToolName: "web_search", + Input: webSearchCallInput(outputItem.Action), }) content = append(content, fantasy.ToolResultContent{ ProviderExecuted: true, @@ -1509,6 +1510,7 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( Type: fantasy.StreamPartTypeToolCall, ID: done.Item.ID, ToolCallName: "web_search", + ToolCallInput: webSearchCallInput(done.Item.Action), ProviderExecuted: true, }) { return @@ -1692,7 +1694,7 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( case "response.failed": failed := event.AsResponseFailed() - if !flushWebSearchResults(nil) { + if !flushWebSearchResults(failed.Response.Output) { return } if !yield(fantasy.StreamPart{ @@ -1847,6 +1849,28 @@ func responsesTextMetadata(itemID string, phase responses.ResponseOutputMessageP } } +// webSearchCallInput encodes the queries of a finished web_search_call as +// the input of its provider-executed tool call. The streamed result waits for +// the terminal event, so the call is the first place callers can read what +// was searched. The key is always present so callers can tell this input +// apart from one that describes a search still to run. +func webSearchCallInput(action responses.ResponseOutputItemUnionAction) string { + queries := action.Queries + if len(queries) == 0 && action.Query != "" { + queries = []string{action.Query} + } + if queries == nil { + queries = []string{} + } + input, err := json.Marshal(struct { + Queries []string `json:"queries"` + }{Queries: queries}) + if err != nil { + return "" + } + return string(input) +} + // GetReasoningMetadata extracts reasoning metadata from provider options for responses models. func GetReasoningMetadata(providerOptions fantasy.ProviderOptions) *ResponsesReasoningMetadata { if openaiResponsesOptions, ok := providerOptions[Name]; ok { diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 38e25a71f..34f199fbf 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -18,6 +18,8 @@ var webSearchCallActionCases = []struct { name string action map[string]any want *WebSearchAction + // wantInput is the provider-executed tool call input. + wantInput string }{ { name: "QueriesAndSources", @@ -37,6 +39,7 @@ var webSearchCallActionCases = []struct { {Type: "url", URL: "https://developers.openai.com/api/docs/guides/tools-web-search"}, }, }, + wantInput: `{"queries":["coder agents web search","openai responses sources"]}`, }, { name: "DeprecatedQuery", @@ -48,6 +51,7 @@ var webSearchCallActionCases = []struct { Type: "search", Query: "latest AI news", }, + wantInput: `{"queries":["latest AI news"]}`, }, { name: "NoQueries", @@ -63,6 +67,7 @@ var webSearchCallActionCases = []struct { {Type: "url", URL: "https://example.com/weather"}, }, }, + wantInput: `{"queries":[]}`, }, } @@ -157,11 +162,14 @@ func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { requireWebSearchSourcesInclude(t, server.calls[0].body) var ( + toolCalls []fantasy.ToolCallContent toolResults []fantasy.ToolResultContent sources []fantasy.SourceContent ) for _, c := range resp.Content { switch v := c.(type) { + case fantasy.ToolCallContent: + toolCalls = append(toolCalls, v) case fantasy.ToolResultContent: toolResults = append(toolResults, v) case fantasy.SourceContent: @@ -169,6 +177,9 @@ func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { } } + require.Len(t, toolCalls, 1) + require.True(t, toolCalls[0].ProviderExecuted) + require.JSONEq(t, tc.wantInput, toolCalls[0].Input) require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolName) @@ -222,10 +233,12 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { }) require.NoError(t, err) - var toolResults, sources []fantasy.StreamPart + var toolCalls, toolResults, sources []fantasy.StreamPart for part := range stream { require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) switch part.Type { + case fantasy.StreamPartTypeToolCall: + toolCalls = append(toolCalls, part) case fantasy.StreamPartTypeToolResult: toolResults = append(toolResults, part) case fantasy.StreamPartTypeSource: @@ -235,6 +248,9 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { require.Len(t, sms.calls, 1) requireWebSearchSourcesInclude(t, sms.calls[0].body) + require.Len(t, toolCalls, 1) + require.True(t, toolCalls[0].ProviderExecuted) + require.JSONEq(t, tc.wantInput, toolCalls[0].ToolCallInput) require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolCallName) @@ -344,6 +360,47 @@ func TestResponsesStream_WebSearchCallResultWithoutTerminalEvent(t *testing.T) { }) } +func TestResponsesStream_WebSearchCallSourcesFromFailedResponse(t *testing.T) { + t.Parallel() + + sms := newStreamingMockServer() + defer sms.close() + sms.chunks = []string{ + responsesSSEEvent("response.output_item.done", + `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"]}}}`), + responsesSSEEvent("response.failed", + `{"type":"response.failed","response":{"id":"resp_01","status":"failed","error":{"code":"server_error","message":"boom"},"output":[`+ + `{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"}]}}`+ + `]}}`), + } + + model := newResponsesProvider(t, sms.server.URL) + stream, err := model.Stream(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + + var toolResults, errs []fantasy.StreamPart + for part := range stream { + switch part.Type { + case fantasy.StreamPartTypeToolResult: + require.Empty(t, errs, "the paired result must precede the stream error") + toolResults = append(toolResults, part) + case fantasy.StreamPartTypeError: + errs = append(errs, part) + } + } + + require.Len(t, errs, 1) + require.Len(t, toolResults, 1) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ + Type: "search", + Queries: []string{"tokyo population"}, + Sources: []WebSearchSource{{Type: "url", URL: "https://www.metro.tokyo.lg.jp/"}}, + }) +} + func TestResponsesWebSearchSourcesInclude(t *testing.T) { t.Parallel() From 1e3c4ff581c6a51bdf7d02e2d790d5db96cfc8d8 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 24 Sep 2026 15:50:08 +0000 Subject: [PATCH 03/10] docs: tighten web search comments Remove comments that repeat field names, keep the terminal-event rationale on pendingWebSearchResults, and narrow claims about sources and the deprecated query to what recorded and live responses showed. --- providers/openai/responses_language_model.go | 22 +++++++------------ providers/openai/responses_options.go | 11 +++++----- providers/openai/responses_web_search_test.go | 7 +++--- providertests/openai_web_search_test.go | 3 +-- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 3be13155c..18da43fb6 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -396,8 +396,6 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res params.ToolChoice = toolChoice } - // web_search_call items only carry action.sources, the full list of - // URLs the search consulted, when the request asks for them. if hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { include = append(include, IncludeWebSearchCallActionSources) } @@ -1345,7 +1343,8 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( return func(yield func(fantasy.StreamPart) bool) { // pendingWebSearchResults holds web_search results, in call - // order, until a terminal event supplies the final action data. + // order, until a terminal event. response.output_item.done can + // omit action.sources or list fewer than the terminal response. var pendingWebSearchResults []*WebSearchCallMetadata // flushWebSearchResults emits the pending web_search results, // preferring the action data from the terminal response output @@ -1493,13 +1492,9 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } case "web_search_call": - // Provider-executed web search completed. Source - // citations come from url_citation annotations on the - // streamed message text. The URLs the search consulted - // stay in the result metadata, and the result waits - // for the terminal event because this item can omit - // action.sources, or list only some of them, while - // response.completed carries the full list. + // Provider-executed web search completed. + // Source citations come from url_citation annotations + // on the streamed message text, not from the action. if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputEnd, ID: done.Item.ID, @@ -1850,10 +1845,9 @@ func responsesTextMetadata(itemID string, phase responses.ResponseOutputMessageP } // webSearchCallInput encodes the queries of a finished web_search_call as -// the input of its provider-executed tool call. The streamed result waits for -// the terminal event, so the call is the first place callers can read what -// was searched. The key is always present so callers can tell this input -// apart from one that describes a search still to run. +// its tool call input, because the streamed result arrives only with the +// terminal event. The queries key is present even when empty, so callers can +// tell a finished search from one still to run. func webSearchCallInput(action responses.ResponseOutputItemUnionAction) string { queries := action.Queries if len(queries) == 0 && action.Query != "" { diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index fa357433b..b99c4afff 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -189,8 +189,8 @@ const ( // IncludeMessageOutputTextLogprobs includes message output text log probabilities. IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" // IncludeWebSearchCallActionSources includes the URLs a web search - // consulted in web_search_call action.sources. Requests that carry the - // web_search tool add it automatically. + // consulted in web_search_call action.sources. Requests with a web + // search tool add it automatically. IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" ) @@ -454,11 +454,10 @@ type WebSearchSource struct { type WebSearchAction struct { // Type is the kind of action: "search", "open_page", or "find". Type string `json:"type"` - // Queries are the search queries (present when Type is "search", - // though OpenAI does not always report them). + // Queries are the search queries. OpenAI can omit them. Queries []string `json:"queries,omitempty"` - // Query is the single search query that older responses report - // instead of Queries. OpenAI deprecated it in favor of Queries. + // Query is the deprecated single-query field that OpenAI replaced + // with Queries. Query string `json:"query,omitempty"` // Sources are the URLs the search consulted. OpenAI only returns // them when the request includes web_search_call.action.sources. diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 34f199fbf..9a4880382 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -15,10 +15,9 @@ import ( // older responses with the deprecated single query, and searches that // report no query at all. var webSearchCallActionCases = []struct { - name string - action map[string]any - want *WebSearchAction - // wantInput is the provider-executed tool call input. + name string + action map[string]any + want *WebSearchAction wantInput string }{ { diff --git a/providertests/openai_web_search_test.go b/providertests/openai_web_search_test.go index 3dbda5268..b93ac7fae 100644 --- a/providertests/openai_web_search_test.go +++ b/providertests/openai_web_search_test.go @@ -135,8 +135,7 @@ func TestOpenAIWebSearch(t *testing.T) { } // requireWebSearchActionMetadata checks that web_search results carry the -// search queries and the consulted source URLs, which OpenAI only returns -// when the request includes web_search_call.action.sources. +// search queries and the consulted source URLs. func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultContent) { t.Helper() From d7b2fa9cd009bc641a87d8aa4578dc9e6bd7ad30 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 24 Sep 2026 16:45:38 +0000 Subject: [PATCH 04/10] feat(providers): add WithoutWebSearchSources opt-out Bedrock Mantle rejects web_search_call.action.sources, and the provider added it to every Responses request with a web search tool, so web search against such backends failed with no workaround. The option turns off the automatic include for openai and openaicompat; an explicit ResponsesProviderOptions.Include entry is still sent. --- providers/openai/openai.go | 13 +++- providers/openai/responses_language_model.go | 30 ++++---- providers/openai/responses_options.go | 3 +- providers/openai/responses_web_search_test.go | 55 +++++++++++++ providers/openaicompat/openaicompat.go | 9 +++ .../openaicompat/web_search_sources_test.go | 77 +++++++++++++++++++ 6 files changed, 171 insertions(+), 16 deletions(-) create mode 100644 providers/openaicompat/web_search_sources_test.go diff --git a/providers/openai/openai.go b/providers/openai/openai.go index 918932ba8..b79ebb1a7 100644 --- a/providers/openai/openai.go +++ b/providers/openai/openai.go @@ -38,6 +38,7 @@ type options struct { sdkOptions []option.RequestOption objectMode fantasy.ObjectMode languageModelOptions []LanguageModelOption + skipWebSearchSources bool } // Option defines a function that configures OpenAI provider options. @@ -136,6 +137,16 @@ func WithUseResponsesAPI() Option { } } +// WithoutWebSearchSources stops Responses requests with a web search tool +// from automatically including web_search_call.action.sources. Use it for +// Responses-compatible backends that reject that include, such as Bedrock +// Mantle. A caller can still request it through ResponsesProviderOptions.Include. +func WithoutWebSearchSources() Option { + return func(o *options) { + o.skipWebSearchSources = true + } +} + // WithResponsesAPIFunc sets a custom filter for which models use the Responses API. // When set, this function is called instead of the default IsResponsesModel(). func WithResponsesAPIFunc(fn func(modelID string) bool) Option { @@ -207,7 +218,7 @@ func (o *provider) LanguageModel(_ context.Context, modelID string) (fantasy.Lan // The responses language model has its own parameter and usage // handling; of the language model options, only the header hook // is shared with the chat completions model. - return newResponsesLanguageModel(modelID, o.options.name, client, objectMode, o.options.reasoningModelFunc, languageModelHeaderFunc(o.options.languageModelOptions)), nil + return newResponsesLanguageModel(modelID, o.options.name, client, objectMode, o.options.reasoningModelFunc, languageModelHeaderFunc(o.options.languageModelOptions), o.options.skipWebSearchSources), nil } languageModelOptions := append([]LanguageModelOption{}, o.options.languageModelOptions...) diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 18da43fb6..19187eedf 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -26,23 +26,25 @@ import ( const topLogprobsMax = 20 type responsesLanguageModel struct { - provider string - modelID string - client openai.Client - objectMode fantasy.ObjectMode - reasoningModelFunc func(modelID string) bool - headerFunc LanguageModelHeaderFunc + provider string + modelID string + client openai.Client + objectMode fantasy.ObjectMode + reasoningModelFunc func(modelID string) bool + headerFunc LanguageModelHeaderFunc + skipWebSearchSources bool } // newResponsesLanguageModel implements a responses api model. -func newResponsesLanguageModel(modelID string, provider string, client openai.Client, objectMode fantasy.ObjectMode, reasoningModelFunc func(modelID string) bool, headerFunc LanguageModelHeaderFunc) responsesLanguageModel { +func newResponsesLanguageModel(modelID string, provider string, client openai.Client, objectMode fantasy.ObjectMode, reasoningModelFunc func(modelID string) bool, headerFunc LanguageModelHeaderFunc, skipWebSearchSources bool) responsesLanguageModel { return responsesLanguageModel{ - modelID: modelID, - provider: provider, - client: client, - objectMode: objectMode, - reasoningModelFunc: reasoningModelFunc, - headerFunc: headerFunc, + modelID: modelID, + provider: provider, + client: client, + objectMode: objectMode, + reasoningModelFunc: reasoningModelFunc, + headerFunc: headerFunc, + skipWebSearchSources: skipWebSearchSources, } } @@ -396,7 +398,7 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res params.ToolChoice = toolChoice } - if hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { + if !o.skipWebSearchSources && hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { include = append(include, IncludeWebSearchCallActionSources) } diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index b99c4afff..f4421de62 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -190,7 +190,8 @@ const ( IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" // IncludeWebSearchCallActionSources includes the URLs a web search // consulted in web_search_call action.sources. Requests with a web - // search tool add it automatically. + // search tool add it automatically unless the provider was created with + // WithoutWebSearchSources. IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" ) diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 9a4880382..173f7b916 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -444,4 +444,59 @@ func TestResponsesWebSearchSourcesInclude(t *testing.T) { requireWebSearchSourcesInclude(t, server.calls[0].body) require.Contains(t, server.calls[0].body["include"], string(IncludeReasoningEncryptedContent)) }) + + newOptedOutModel := func(t *testing.T, serverURL string) fantasy.LanguageModel { + t.Helper() + provider, err := New( + WithAPIKey("test-api-key"), + WithBaseURL(serverURL), + WithUseResponsesAPI(), + WithoutWebSearchSources(), + ) + require.NoError(t, err) + model, err := provider.LanguageModel(context.Background(), "gpt-4.1") + require.NoError(t, err) + return model + } + + t.Run("NotRequestedWhenProviderOptsOut", func(t *testing.T) { + t.Parallel() + + server := newMockServer() + defer server.close() + server.response = mockResponsesWebSearchResponse() + + model := newOptedOutModel(t, server.server.URL) + _, err := model.Generate(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + }) + require.NoError(t, err) + require.Len(t, server.calls, 1) + + include, _ := server.calls[0].body["include"].([]any) + require.NotContains(t, include, string(IncludeWebSearchCallActionSources)) + }) + + t.Run("CallerIncludeSentWhenProviderOptsOut", func(t *testing.T) { + t.Parallel() + + server := newMockServer() + defer server.close() + server.response = mockResponsesWebSearchResponse() + + model := newOptedOutModel(t, server.server.URL) + _, err := model.Generate(context.Background(), fantasy.Call{ + Prompt: testPrompt, + Tools: []fantasy.Tool{WebSearchTool(nil)}, + ProviderOptions: fantasy.ProviderOptions{ + Name: &ResponsesProviderOptions{ + Include: []IncludeType{IncludeWebSearchCallActionSources}, + }, + }, + }) + require.NoError(t, err) + require.Len(t, server.calls, 1) + requireWebSearchSourcesInclude(t, server.calls[0].body) + }) } diff --git a/providers/openaicompat/openaicompat.go b/providers/openaicompat/openaicompat.go index af8948579..21da8fb11 100644 --- a/providers/openaicompat/openaicompat.go +++ b/providers/openaicompat/openaicompat.go @@ -123,6 +123,15 @@ func WithUseResponsesAPI() Option { } } +// WithoutWebSearchSources stops Responses requests with a web search tool +// from automatically including web_search_call.action.sources, which some +// compatible backends, such as Bedrock Mantle, reject. +func WithoutWebSearchSources() Option { + return func(o *options) { + o.openaiOptions = append(o.openaiOptions, openai.WithoutWebSearchSources()) + } +} + // WithResponsesAPIFunc sets a custom filter for which models use the Responses API. func WithResponsesAPIFunc(fn func(modelID string) bool) Option { return func(o *options) { diff --git a/providers/openaicompat/web_search_sources_test.go b/providers/openaicompat/web_search_sources_test.go new file mode 100644 index 000000000..dda48f1d6 --- /dev/null +++ b/providers/openaicompat/web_search_sources_test.go @@ -0,0 +1,77 @@ +package openaicompat + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "charm.land/fantasy" + "charm.land/fantasy/providers/openai" + "github.com/stretchr/testify/require" +) + +func TestWithoutWebSearchSources(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + opts []Option + wantInclude bool + }{ + {name: "Default", wantInclude: true}, + {name: "OptedOut", opts: []Option{WithoutWebSearchSources()}, wantInclude: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + include []string + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Include []string `json:"include"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mu.Lock() + include = body.Include + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"resp_01","object":"response","status":"completed","model":"gpt-4.1","output":[{"type":"message","id":"msg_01","status":"completed","role":"assistant","content":[{"type":"output_text","text":"ok","annotations":[]}]}]}`)) + })) + defer server.Close() + + provider, err := New(append([]Option{ + WithBaseURL(server.URL), + WithAPIKey("test-api-key"), + WithUseResponsesAPI(), + }, tc.opts...)...) + require.NoError(t, err) + model, err := provider.LanguageModel(context.Background(), "gpt-4.1") + require.NoError(t, err) + + _, err = model.Generate(context.Background(), fantasy.Call{ + Prompt: fantasy.Prompt{{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{fantasy.TextPart{Text: "search"}}, + }}, + Tools: []fantasy.Tool{openai.WebSearchTool(nil)}, + }) + require.NoError(t, err) + + mu.Lock() + defer mu.Unlock() + if tc.wantInclude { + require.Contains(t, include, string(openai.IncludeWebSearchCallActionSources)) + } else { + require.NotContains(t, include, string(openai.IncludeWebSearchCallActionSources)) + } + }) + } +} From 57ac0677fd2e0eb97c72d467fb9b08ba5d6e5b8b Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 25 Sep 2026 14:55:46 +0000 Subject: [PATCH 05/10] feat: tag web search sources with their search A source part now carries the ID of the provider-executed tool call that returned it; citations carry none. Anthropic already emitted found pages as sources, and now tags them. OpenAI now emits a search's found pages as tagged sources and its result when the search item closes. Live OpenAI streams list a search's own pages on output_item.done, while response.completed also lists pages the answer cited among them, so the result no longer waits for the terminal event. The call input carries the action: type with queries, or url and pattern for open_page and find_in_page. The result metadata keeps the item status. --- agent.go | 1 + content.go | 16 +- model.go | 2 + providers/anthropic/anthropic.go | 17 +- providers/anthropic/anthropic_test.go | 6 +- providers/openai/responses_language_model.go | 148 +++++++------- providers/openai/responses_options.go | 12 +- providers/openai/responses_web_search_test.go | 190 +++++++++--------- providertests/openai_web_search_test.go | 24 ++- 9 files changed, 219 insertions(+), 197 deletions(-) diff --git a/agent.go b/agent.go index 3c6eb8904..6293c949b 100644 --- a/agent.go +++ b/agent.go @@ -1605,6 +1605,7 @@ func (a *agent) processStepStream(ctx context.Context, stream StreamResponse, op ID: part.ID, URL: part.URL, Title: part.Title, + ToolCallID: part.SourceToolCallID, ProviderMetadata: part.ProviderMetadata, } stepContent = append(stepContent, sourceContent) diff --git a/content.go b/content.go index 1318d08c8..e7cccd480 100644 --- a/content.go +++ b/content.go @@ -412,12 +412,16 @@ const ( // SourceContent represents a source that has been used as input to generate the response. type SourceContent struct { - SourceType SourceType `json:"source_type"` // "url" or "document" - ID string `json:"id"` - URL string `json:"url"` // for URL sources - Title string `json:"title"` - MediaType string `json:"media_type"` // for document sources (IANA media type) - Filename string `json:"filename"` // for document sources + SourceType SourceType `json:"source_type"` // "url" or "document" + ID string `json:"id"` + URL string `json:"url"` // for URL sources + Title string `json:"title"` + MediaType string `json:"media_type"` // for document sources (IANA media type) + Filename string `json:"filename"` // for document sources + // ToolCallID is the provider-executed tool call, such as a web search, + // that returned this source. It is empty for sources the response text + // cites. + ToolCallID string `json:"tool_call_id,omitempty"` ProviderMetadata ProviderMetadata `json:"provider_metadata"` } diff --git a/model.go b/model.go index 3e3d6b91e..b248f465c 100644 --- a/model.go +++ b/model.go @@ -187,6 +187,8 @@ type StreamPart struct { SourceType SourceType `json:"source_type"` URL string `json:"url"` Title string `json:"title"` + // SourceToolCallID is SourceContent.ToolCallID for source parts. + SourceToolCallID string `json:"source_tool_call_id,omitempty"` ProviderMetadata ProviderMetadata `json:"provider_metadata"` } diff --git a/providers/anthropic/anthropic.go b/providers/anthropic/anthropic.go index e0158174c..38c70cfe0 100644 --- a/providers/anthropic/anthropic.go +++ b/providers/anthropic/anthropic.go @@ -1683,8 +1683,9 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas if !ok { continue } - // Extract search results as sources/citations, preserving - // encrypted_content for multi-turn round-tripping. + // Emit found pages as sources tagged with the search's ID, + // and keep them in metadata with encrypted_content for + // multi-turn round-tripping. toolResult := fantasy.ToolResultContent{ ToolCallID: webSearchResult.ToolUseID, ToolName: "web_search", @@ -1698,6 +1699,7 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas ID: item.URL, URL: item.URL, Title: item.Title, + ToolCallID: webSearchResult.ToolUseID, }) metadataResults = append(metadataResults, WebSearchResultItem{ URL: item.URL, @@ -1887,11 +1889,12 @@ func (a languageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.S if items := contentBlock.Content.OfWebSearchResultBlockArray; len(items) > 0 { for _, item := range items { if !yield(fantasy.StreamPart{ - Type: fantasy.StreamPartTypeSource, - ID: item.URL, - SourceType: fantasy.SourceTypeURL, - URL: item.URL, - Title: item.Title, + Type: fantasy.StreamPartTypeSource, + ID: item.URL, + SourceType: fantasy.SourceTypeURL, + URL: item.URL, + Title: item.Title, + SourceToolCallID: contentBlock.ToolUseID, }) { return } diff --git a/providers/anthropic/anthropic_test.go b/providers/anthropic/anthropic_test.go index 7a6eab17b..411e33768 100644 --- a/providers/anthropic/anthropic_test.go +++ b/providers/anthropic/anthropic_test.go @@ -2166,13 +2166,16 @@ func TestGenerate_WebSearchResponse(t *testing.T) { require.Equal(t, "web_search", toolCalls[0].ToolName) require.Equal(t, "srvtoolu_01", toolCalls[0].ToolCallID) - // SourceContent entries for each search result. + // SourceContent entries for each search result, tagged with the + // search that found them. require.Len(t, sources, 2) require.Equal(t, "https://example.com/ai-news", sources[0].URL) require.Equal(t, "Latest AI News", sources[0].Title) require.Equal(t, fantasy.SourceTypeURL, sources[0].SourceType) + require.Equal(t, "srvtoolu_01", sources[0].ToolCallID) require.Equal(t, "https://example.com/ml-update", sources[1].URL) require.Equal(t, "ML Update", sources[1].Title) + require.Equal(t, "srvtoolu_01", sources[1].ToolCallID) // ToolResultContent with provider metadata preserving encrypted_content. require.Len(t, toolResults, 1) @@ -2776,6 +2779,7 @@ func TestStream_WebSearchResponse(t *testing.T) { require.Equal(t, "https://example.com/ai-news", sourceParts[0].URL) require.Equal(t, "Latest AI News", sourceParts[0].Title) require.Equal(t, fantasy.SourceTypeURL, sourceParts[0].SourceType) + require.Equal(t, "srvtoolu_01", sourceParts[0].SourceToolCallID) // Text block emits a text delta. require.NotEmpty(t, textDeltas, "should have text deltas") diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 19187eedf..5aca6d44c 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -1202,23 +1202,27 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call) // a ToolCallContent and ToolResultContent as a pair, // matching the vercel/ai pattern for provider tools. // - // Note: source citations come from url_citation annotations - // on the message text (handled in the "message" case above). - // The URLs the search consulted stay in the result metadata - // so callers can tell them apart from citations. - wsMeta := webSearchCallToMetadata(outputItem.ID, outputItem.Action) + // Found pages become sources tagged with the search's ID. + // Citations come untagged from url_citation annotations on + // the message text (handled in the "message" case above). + // A non-streamed response only has the final output, where + // OpenAI also lists pages the answer cited among the + // search's sources, so those appear as found pages here. content = append(content, fantasy.ToolCallContent{ ProviderExecuted: true, ToolCallID: outputItem.ID, ToolName: "web_search", Input: webSearchCallInput(outputItem.Action), }) + for _, source := range webSearchCallSources(outputItem.ID, outputItem.Action) { + content = append(content, source) + } content = append(content, fantasy.ToolResultContent{ ProviderExecuted: true, ToolCallID: outputItem.ID, ToolName: "web_search", ProviderMetadata: fantasy.ProviderMetadata{ - Name: wsMeta, + Name: webSearchCallToMetadata(outputItem.ID, outputItem.Status, outputItem.Action), }, }) @@ -1344,40 +1348,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( sourceStoreEnabled := params.Store.Value return func(yield func(fantasy.StreamPart) bool) { - // pendingWebSearchResults holds web_search results, in call - // order, until a terminal event. response.output_item.done can - // omit action.sources or list fewer than the terminal response. - var pendingWebSearchResults []*WebSearchCallMetadata - // flushWebSearchResults emits the pending web_search results, - // preferring the action data from the terminal response output - // over the data captured when each item finished. - flushWebSearchResults := func(output []responses.ResponseOutputItemUnion) bool { - pending := pendingWebSearchResults - pendingWebSearchResults = nil - for _, meta := range pending { - for _, item := range output { - if item.Type == "web_search_call" && item.ID == meta.ItemID { - meta = webSearchCallToMetadata(item.ID, item.Action) - break - } - } - // Emit a ToolResult so the agent framework includes - // it in round-trip messages. - if !yield(fantasy.StreamPart{ - Type: fantasy.StreamPartTypeToolResult, - ID: meta.ItemID, - ToolCallName: "web_search", - ProviderExecuted: true, - ProviderMetadata: fantasy.ProviderMetadata{ - Name: meta, - }, - }) { - return false - } - } - return true - } - if len(warnings) > 0 { if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeWarnings, @@ -1494,9 +1464,10 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } case "web_search_call": - // Provider-executed web search completed. - // Source citations come from url_citation annotations - // on the streamed message text, not from the action. + // Provider-executed web search completed. Its found + // pages are sources tagged with the search's ID; + // citations come untagged from url_citation + // annotations on the message text. if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputEnd, ID: done.Item.ID, @@ -1512,7 +1483,30 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( }) { return } - pendingWebSearchResults = append(pendingWebSearchResults, webSearchCallToMetadata(done.Item.ID, done.Item.Action)) + for _, source := range webSearchCallSources(done.Item.ID, done.Item.Action) { + if !yield(fantasy.StreamPart{ + Type: fantasy.StreamPartTypeSource, + ID: source.ID, + SourceType: source.SourceType, + URL: source.URL, + SourceToolCallID: source.ToolCallID, + }) { + return + } + } + // Emit a ToolResult so the agent framework + // includes it in round-trip messages. + if !yield(fantasy.StreamPart{ + Type: fantasy.StreamPartTypeToolResult, + ID: done.Item.ID, + ToolCallName: "web_search", + ProviderExecuted: true, + ProviderMetadata: fantasy.ProviderMetadata{ + Name: webSearchCallToMetadata(done.Item.ID, done.Item.Status, done.Item.Action), + }, + }) { + return + } case "message": if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeTextEnd, @@ -1675,9 +1669,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( responseID = completed.Response.ID finishReason = mapResponsesFinishReason(completed.Response.IncompleteDetails.Reason, hasFunctionCall) usage = responsesUsage(completed.Response) - if !flushWebSearchResults(completed.Response.Output) { - return - } case "response.incomplete": sawTerminalEvent = true @@ -1685,15 +1676,9 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( responseID = incomplete.Response.ID finishReason = mapResponsesFinishReason(incomplete.Response.IncompleteDetails.Reason, hasFunctionCall) usage = responsesUsage(incomplete.Response) - if !flushWebSearchResults(incomplete.Response.Output) { - return - } case "response.failed": failed := event.AsResponseFailed() - if !flushWebSearchResults(failed.Response.Output) { - return - } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeError, Error: responsesFailedStreamError(failed.Response.Error.Message, string(failed.Response.Error.Code)), @@ -1704,9 +1689,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( case "error": errorEvent := event.AsError() - if !flushWebSearchResults(nil) { - return - } if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeError, Error: responsesErrorStreamError(errorEvent.Message, errorEvent.Code), @@ -1717,12 +1699,6 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } } - // Every emitted web_search tool call keeps its paired result, - // even when the stream ends without a terminal event. - if !flushWebSearchResults(nil) { - return - } - err := stream.Err() if err != nil && !errors.Is(err, io.EOF) { yield(fantasy.StreamPart{ @@ -1821,13 +1797,15 @@ func toWebSearchToolParam(pt fantasy.ProviderDefinedTool) responses.ToolUnionPar // webSearchCallToMetadata converts an OpenAI web search call output // into our structured metadata for round-tripping. -func webSearchCallToMetadata(itemID string, action responses.ResponseOutputItemUnionAction) *WebSearchCallMetadata { - meta := &WebSearchCallMetadata{ItemID: itemID} +func webSearchCallToMetadata(itemID, status string, action responses.ResponseOutputItemUnionAction) *WebSearchCallMetadata { + meta := &WebSearchCallMetadata{ItemID: itemID, Status: status} if action.Type != "" { a := &WebSearchAction{ Type: action.Type, Queries: action.Queries, Query: action.Query, + URL: action.URL, + Pattern: action.Pattern, } for _, src := range action.Sources { a.Sources = append(a.Sources, WebSearchSource{ @@ -1846,25 +1824,43 @@ func responsesTextMetadata(itemID string, phase responses.ResponseOutputMessageP } } -// webSearchCallInput encodes the queries of a finished web_search_call as -// its tool call input, because the streamed result arrives only with the -// terminal event. The queries key is present even when empty, so callers can -// tell a finished search from one still to run. +// webSearchCallSources returns the pages a web_search_call found as URL +// sources tagged with the call's ID. +func webSearchCallSources(itemID string, action responses.ResponseOutputItemUnionAction) []fantasy.SourceContent { + sources := make([]fantasy.SourceContent, 0, len(action.Sources)) + for _, src := range action.Sources { + if src.URL == "" { + continue + } + sources = append(sources, fantasy.SourceContent{ + SourceType: fantasy.SourceTypeURL, + ID: src.URL, + URL: src.URL, + ToolCallID: itemID, + }) + } + return sources +} + +// webSearchCallInput encodes the web_search_call action as its tool call +// input: the action type with the queries of a search (falling back to the +// deprecated query), the URL of open_page, or the URL and pattern of +// find_in_page. func webSearchCallInput(action responses.ResponseOutputItemUnionAction) string { queries := action.Queries if len(queries) == 0 && action.Query != "" { queries = []string{action.Query} } - if queries == nil { - queries = []string{} - } - input, err := json.Marshal(struct { - Queries []string `json:"queries"` - }{Queries: queries}) + encoded, err := json.Marshal(struct { + Type string `json:"type,omitempty"` + Queries []string `json:"queries,omitempty"` + URL string `json:"url,omitempty"` + Pattern string `json:"pattern,omitempty"` + }{Type: action.Type, Queries: queries, URL: action.URL, Pattern: action.Pattern}) if err != nil { - return "" + return "{}" } - return string(input) + return string(encoded) } // GetReasoningMetadata extracts reasoning metadata from provider options for responses models. diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index f4421de62..486619fbe 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -453,15 +453,19 @@ type WebSearchSource struct { // WebSearchAction represents the action taken during a web search call. type WebSearchAction struct { - // Type is the kind of action: "search", "open_page", or "find". + // Type is the kind of action: "search", "open_page", or "find_in_page". Type string `json:"type"` // Queries are the search queries. OpenAI can omit them. Queries []string `json:"queries,omitempty"` // Query is the deprecated single-query field that OpenAI replaced // with Queries. Query string `json:"query,omitempty"` - // Sources are the URLs the search consulted. OpenAI only returns - // them when the request includes web_search_call.action.sources. + // URL is the page an open_page or find_in_page action used. + URL string `json:"url,omitempty"` + // Pattern is the text a find_in_page action looked for. + Pattern string `json:"pattern,omitempty"` + // Sources are the pages the search found. OpenAI only returns them + // when the request includes web_search_call.action.sources. Sources []WebSearchSource `json:"sources,omitempty"` } @@ -472,6 +476,8 @@ type WebSearchAction struct { type WebSearchCallMetadata struct { // ItemID is the server-side ID of the web_search_call output item. ItemID string `json:"item_id"` + // Status is the item's status, such as "completed" or "failed". + Status string `json:"status,omitempty"` // Action contains the structured action data from the search. Action *WebSearchAction `json:"action,omitempty"` } diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 173f7b916..5c1bb033d 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -11,14 +11,15 @@ import ( ) // webSearchCallActionCases covers the web_search_call action shapes -// OpenAI returns: current responses with queries and consulted sources, -// older responses with the deprecated single query, and searches that -// report no query at all. +// OpenAI returns: searches with queries and found pages, older responses +// with the deprecated single query, searches that report no query, and the +// open_page and find_in_page actions of reasoning models. var webSearchCallActionCases = []struct { name string action map[string]any want *WebSearchAction wantInput string + wantFound []string }{ { name: "QueriesAndSources", @@ -38,7 +39,8 @@ var webSearchCallActionCases = []struct { {Type: "url", URL: "https://developers.openai.com/api/docs/guides/tools-web-search"}, }, }, - wantInput: `{"queries":["coder agents web search","openai responses sources"]}`, + wantInput: `{"type":"search","queries":["coder agents web search","openai responses sources"]}`, + wantFound: []string{"https://coder.com/docs/ai-coder", "https://developers.openai.com/api/docs/guides/tools-web-search"}, }, { name: "DeprecatedQuery", @@ -50,7 +52,7 @@ var webSearchCallActionCases = []struct { Type: "search", Query: "latest AI news", }, - wantInput: `{"queries":["latest AI news"]}`, + wantInput: `{"type":"search","queries":["latest AI news"]}`, }, { name: "NoQueries", @@ -66,16 +68,44 @@ var webSearchCallActionCases = []struct { {Type: "url", URL: "https://example.com/weather"}, }, }, - wantInput: `{"queries":[]}`, + wantInput: `{"type":"search"}`, + wantFound: []string{"https://example.com/weather"}, + }, + { + name: "OpenPage", + action: map[string]any{ + "type": "open_page", + "url": "https://go.dev/dl/", + }, + want: &WebSearchAction{ + Type: "open_page", + URL: "https://go.dev/dl/", + }, + wantInput: `{"type":"open_page","url":"https://go.dev/dl/"}`, + }, + { + name: "FindInPage", + action: map[string]any{ + "type": "find_in_page", + "url": "https://go.dev/doc/devel/release", + "pattern": "go1.27", + }, + want: &WebSearchAction{ + Type: "find_in_page", + URL: "https://go.dev/doc/devel/release", + Pattern: "go1.27", + }, + wantInput: `{"type":"find_in_page","url":"https://go.dev/doc/devel/release","pattern":"go1.27"}`, }, } -func requireWebSearchCallMetadata(t *testing.T, metadata fantasy.ProviderMetadata, want *WebSearchAction) { +func requireWebSearchCallMetadata(t *testing.T, metadata fantasy.ProviderMetadata, wantStatus string, want *WebSearchAction) { t.Helper() wsMeta, ok := metadata[Name].(*WebSearchCallMetadata) require.True(t, ok, "metadata should be *WebSearchCallMetadata, got %T", metadata[Name]) require.Equal(t, "ws_01", wsMeta.ItemID) + require.Equal(t, wantStatus, wsMeta.Status) require.Equal(t, want, wsMeta.Action) // Callers persist the metadata as JSON and read it back on later @@ -182,11 +212,21 @@ func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolName) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.want) - - // Consulted sources must not become citation sources. - require.Len(t, sources, 1) - require.Equal(t, "https://example.com/cited", sources[0].URL) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "completed", tc.want) + + var found, cited []string + for _, source := range sources { + switch source.ToolCallID { + case "ws_01": + found = append(found, source.URL) + case "": + cited = append(cited, source.URL) + default: + t.Fatalf("source %q tagged with unknown tool call %q", source.URL, source.ToolCallID) + } + } + require.Equal(t, tc.wantFound, found) + require.Equal(t, []string{"https://example.com/cited"}, cited) }) } } @@ -253,37 +293,50 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolCallName) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.want) - - // Consulted sources must not become citation sources. - require.Len(t, sources, 1) - require.Equal(t, "https://example.com/cited", sources[0].URL) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "completed", tc.want) + + var found, cited []string + for _, source := range sources { + switch source.SourceToolCallID { + case "ws_01": + found = append(found, source.URL) + case "": + cited = append(cited, source.URL) + default: + t.Fatalf("source %q tagged with unknown tool call %q", source.URL, source.SourceToolCallID) + } + } + require.Equal(t, tc.wantFound, found) + require.Equal(t, []string{"https://example.com/cited"}, cited) }) } } -// TestResponsesStream_WebSearchCallSourcesFromCompletedResponse covers the -// recorded OpenAI behavior where response.output_item.done omits -// action.sources and only response.completed lists them. -func TestResponsesStream_WebSearchCallSourcesFromCompletedResponse(t *testing.T) { +// TestResponsesStream_WebSearchCallFinishesWithItem covers live OpenAI +// streams: a search's found pages are on its output_item.done, and the final +// response summary also lists pages the answer cited among them. The search +// finishes, with its own pages, before the answer streams. +func TestResponsesStream_WebSearchCallFinishesWithItem(t *testing.T) { t.Parallel() sms := newStreamingMockServer() defer sms.close() sms.chunks = []string{ responsesSSEEvent("response.output_item.added", - `{"type":"response.output_item.added","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"in_progress","action":{"type":"search"}}}`), + `{"type":"response.output_item.added","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"in_progress"}}`), responsesSSEEvent("response.output_item.done", - `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"query":"tokyo population"}}}`), + `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"}]}}}`), responsesSSEEvent("response.output_item.added", `{"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"in_progress","content":[]}}`), responsesSSEEvent("response.output_text.delta", `{"type":"response.output_text.delta","item_id":"msg_01","output_index":1,"content_index":0,"delta":"About 14 million."}`), + responsesSSEEvent("response.output_text.annotation.added", + `{"type":"response.output_text.annotation.added","annotation":{"type":"url_citation","url":"https://example.com/tokyo","title":"Tokyo","start_index":0,"end_index":5},"annotation_index":0,"content_index":0,"item_id":"msg_01","output_index":1}`), responsesSSEEvent("response.output_item.done", `{"type":"response.output_item.done","output_index":1,"item":{"type":"message","id":"msg_01","role":"assistant","status":"completed","content":[{"type":"output_text","text":"About 14 million.","annotations":[]}]}}`), responsesSSEEvent("response.completed", `{"type":"response.completed","response":{"id":"resp_01","status":"completed","output":[`+ - `{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"query":"tokyo population","sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"},{"type":"url","url":"https://example.com/tokyo"}]}},`+ + `{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"},{"type":"url","url":"https://example.com/tokyo"}]}},`+ `{"type":"message","id":"msg_01","role":"assistant","status":"completed","content":[{"type":"output_text","text":"About 14 million.","annotations":[]}]}`+ `],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`), } @@ -296,81 +349,35 @@ func TestResponsesStream_WebSearchCallSourcesFromCompletedResponse(t *testing.T) require.NoError(t, err) var types []fantasy.StreamPartType - var toolResults []fantasy.StreamPart + var found, cited []string for part := range stream { require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) types = append(types, part.Type) - if part.Type == fantasy.StreamPartTypeToolResult { - toolResults = append(toolResults, part) - } - } - - // The call keeps its position ahead of the answer text; the - // result follows once the terminal event arrives. - require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolCall), slices.Index(types, fantasy.StreamPartTypeTextDelta)) - require.Less(t, slices.Index(types, fantasy.StreamPartTypeTextDelta), slices.Index(types, fantasy.StreamPartTypeToolResult)) - require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolResult), slices.Index(types, fantasy.StreamPartTypeFinish)) - - require.Len(t, toolResults, 1) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ - Type: "search", - Queries: []string{"tokyo population"}, - Query: "tokyo population", - Sources: []WebSearchSource{ - {Type: "url", URL: "https://www.metro.tokyo.lg.jp/"}, - {Type: "url", URL: "https://example.com/tokyo"}, - }, - }) -} - -func TestResponsesStream_WebSearchCallResultWithoutTerminalEvent(t *testing.T) { - t.Parallel() - - sms := newStreamingMockServer() - defer sms.close() - sms.chunks = []string{ - responsesSSEEvent("response.output_item.done", - `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"]}}}`), - } - - model := newResponsesProvider(t, sms.server.URL) - stream, err := model.Stream(context.Background(), fantasy.Call{ - Prompt: testPrompt, - Tools: []fantasy.Tool{WebSearchTool(nil)}, - }) - require.NoError(t, err) - - var toolResults, errs []fantasy.StreamPart - for part := range stream { - switch part.Type { - case fantasy.StreamPartTypeToolResult: - require.Empty(t, errs, "the paired result must precede the stream error") - toolResults = append(toolResults, part) - case fantasy.StreamPartTypeError: - errs = append(errs, part) + if part.Type == fantasy.StreamPartTypeSource { + if part.SourceToolCallID == "ws_01" { + found = append(found, part.URL) + } else { + cited = append(cited, part.URL) + } } } - require.Len(t, errs, 1) - require.Len(t, toolResults, 1) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ - Type: "search", - Queries: []string{"tokyo population"}, - }) + require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolCall), slices.Index(types, fantasy.StreamPartTypeSource)) + require.Less(t, slices.Index(types, fantasy.StreamPartTypeToolResult), slices.Index(types, fantasy.StreamPartTypeTextDelta)) + require.Equal(t, []string{"https://www.metro.tokyo.lg.jp/"}, found) + require.Equal(t, []string{"https://example.com/tokyo"}, cited) } -func TestResponsesStream_WebSearchCallSourcesFromFailedResponse(t *testing.T) { +func TestResponsesStream_WebSearchCallFailedStatus(t *testing.T) { t.Parallel() sms := newStreamingMockServer() defer sms.close() sms.chunks = []string{ responsesSSEEvent("response.output_item.done", - `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"]}}}`), - responsesSSEEvent("response.failed", - `{"type":"response.failed","response":{"id":"resp_01","status":"failed","error":{"code":"server_error","message":"boom"},"output":[`+ - `{"type":"web_search_call","id":"ws_01","status":"completed","action":{"type":"search","queries":["tokyo population"],"sources":[{"type":"url","url":"https://www.metro.tokyo.lg.jp/"}]}}`+ - `]}}`), + `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"failed","action":{"type":"search","queries":["tokyo population"]}}}`), + responsesSSEEvent("response.completed", + `{"type":"response.completed","response":{"id":"resp_01","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`), } model := newResponsesProvider(t, sms.server.URL) @@ -380,23 +387,18 @@ func TestResponsesStream_WebSearchCallSourcesFromFailedResponse(t *testing.T) { }) require.NoError(t, err) - var toolResults, errs []fantasy.StreamPart + var toolResults []fantasy.StreamPart for part := range stream { - switch part.Type { - case fantasy.StreamPartTypeToolResult: - require.Empty(t, errs, "the paired result must precede the stream error") + require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) + if part.Type == fantasy.StreamPartTypeToolResult { toolResults = append(toolResults, part) - case fantasy.StreamPartTypeError: - errs = append(errs, part) } } - require.Len(t, errs, 1) require.Len(t, toolResults, 1) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, &WebSearchAction{ + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "failed", &WebSearchAction{ Type: "search", Queries: []string{"tokyo population"}, - Sources: []WebSearchSource{{Type: "url", URL: "https://www.metro.tokyo.lg.jp/"}}, }) } diff --git a/providertests/openai_web_search_test.go b/providertests/openai_web_search_test.go index b93ac7fae..2a0535191 100644 --- a/providertests/openai_web_search_test.go +++ b/providertests/openai_web_search_test.go @@ -4,6 +4,7 @@ import ( "cmp" "net/http" "os" + "slices" "testing" "charm.land/fantasy" @@ -78,10 +79,7 @@ func TestOpenAIWebSearch(t *testing.T) { require.NotEmpty(t, providerToolCalls, "should have provider-executed tool calls") require.Equal(t, "web_search", providerToolCalls[0].ToolName) - requireWebSearchActionMetadata(t, providerToolResults) - // Sources come from url_citation annotations; the model - // may or may not include inline citations so we don't - // require them, but if present they should have URLs. + requireWebSearchActionMetadata(t, providerToolResults, sources) for _, src := range sources { require.NotEmpty(t, src.URL, "source should have a URL") } @@ -113,6 +111,7 @@ func TestOpenAIWebSearch(t *testing.T) { // Verify provider-executed tool calls and results in steps. var providerToolCalls []fantasy.ToolCallContent var providerToolResults []fantasy.ToolResultContent + var sources []fantasy.SourceContent for _, step := range result.Steps { for _, c := range step.Content { switch v := c.(type) { @@ -124,19 +123,23 @@ func TestOpenAIWebSearch(t *testing.T) { if v.ProviderExecuted { providerToolResults = append(providerToolResults, v) } + case fantasy.SourceContent: + sources = append(sources, v) } } } require.NotEmpty(t, providerToolCalls, "should have provider-executed tool calls") require.Equal(t, "web_search", providerToolCalls[0].ToolName) require.NotEmpty(t, providerToolResults, "should have provider-executed tool results") - requireWebSearchActionMetadata(t, providerToolResults) + requireWebSearchActionMetadata(t, providerToolResults, sources) }) } // requireWebSearchActionMetadata checks that web_search results carry the -// search queries and the consulted source URLs. -func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultContent) { +// search queries, and that every page a search found is a source tagged +// with that search's ID. The recorded gpt-4.1 stream reports no found pages +// on its search item, so found pages are not required. +func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultContent, sources []fantasy.SourceContent) { t.Helper() require.NotEmpty(t, results, "should have provider-executed tool results") @@ -146,9 +149,10 @@ func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultCo require.NotNil(t, meta.Action) require.Equal(t, "search", meta.Action.Type) require.NotEmpty(t, meta.Action.Queries, "search action should report its queries") - require.NotEmpty(t, meta.Action.Sources, "search action should report its consulted sources") - for _, source := range meta.Action.Sources { - require.NotEmpty(t, source.URL, "consulted source should have a URL") + for _, found := range meta.Action.Sources { + require.True(t, slices.ContainsFunc(sources, func(source fantasy.SourceContent) bool { + return source.URL == found.URL && source.ToolCallID == meta.ItemID + }), "found page %q should be a source tagged with %q", found.URL, meta.ItemID) } } } From cb04ffd59797bc80c87bf45306af7eafaea26110 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 25 Sep 2026 15:44:23 +0000 Subject: [PATCH 06/10] docs(providers/openai): scope the web search stream claim by model gpt-4.1-mini and gpt-5-mini search items carried their found pages on output_item.done. gpt-4.1 items carried none, and its final summary listed only cited pages, so gpt-4.1 searches report no found pages. --- providers/openai/responses_options.go | 4 ++-- providers/openai/responses_web_search_test.go | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index 486619fbe..52c2bcccd 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -188,8 +188,8 @@ const ( IncludeFileSearchCallResults IncludeType = "file_search_call.results" // IncludeMessageOutputTextLogprobs includes message output text log probabilities. IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" - // IncludeWebSearchCallActionSources includes the URLs a web search - // consulted in web_search_call action.sources. Requests with a web + // IncludeWebSearchCallActionSources includes the pages a web search + // found in web_search_call action.sources. Requests with a web // search tool add it automatically unless the provider was created with // WithoutWebSearchSources. IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 5c1bb033d..5c0f1bdb7 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -312,10 +312,12 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { } } -// TestResponsesStream_WebSearchCallFinishesWithItem covers live OpenAI -// streams: a search's found pages are on its output_item.done, and the final -// response summary also lists pages the answer cited among them. The search -// finishes, with its own pages, before the answer streams. +// TestResponsesStream_WebSearchCallFinishesWithItem covers live gpt-4.1-mini +// and gpt-5-mini streams: a search's found pages are on its output_item.done, +// and the final response summary also lists pages the answer cited among +// them. gpt-4.1 search items carried no pages, and its summary listed only +// cited pages. The search finishes, with its own pages, before the answer +// streams. func TestResponsesStream_WebSearchCallFinishesWithItem(t *testing.T) { t.Parallel() From a23b9d509a452942e076279354b92e6f540430a2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Sat, 3 Oct 2026 16:01:10 +0000 Subject: [PATCH 07/10] refactor(providers): remove WithoutWebSearchSources The opt-out was added because vercel/ai#18507 reported that Bedrock Mantle rejects web_search_call.action.sources. Mantle now accepts the include in us-east-1 and us-west-2. The one other host found to reject it, vLLM, cannot run a web_search tool, so turning web search off loses nothing there. Nothing calls the option. --- providers/openai/openai.go | 13 +--- providers/openai/responses_language_model.go | 30 ++++---- providers/openai/responses_options.go | 3 +- providers/openai/responses_web_search_test.go | 55 ------------- providers/openaicompat/openaicompat.go | 9 --- .../openaicompat/web_search_sources_test.go | 77 ------------------- 6 files changed, 16 insertions(+), 171 deletions(-) delete mode 100644 providers/openaicompat/web_search_sources_test.go diff --git a/providers/openai/openai.go b/providers/openai/openai.go index b79ebb1a7..918932ba8 100644 --- a/providers/openai/openai.go +++ b/providers/openai/openai.go @@ -38,7 +38,6 @@ type options struct { sdkOptions []option.RequestOption objectMode fantasy.ObjectMode languageModelOptions []LanguageModelOption - skipWebSearchSources bool } // Option defines a function that configures OpenAI provider options. @@ -137,16 +136,6 @@ func WithUseResponsesAPI() Option { } } -// WithoutWebSearchSources stops Responses requests with a web search tool -// from automatically including web_search_call.action.sources. Use it for -// Responses-compatible backends that reject that include, such as Bedrock -// Mantle. A caller can still request it through ResponsesProviderOptions.Include. -func WithoutWebSearchSources() Option { - return func(o *options) { - o.skipWebSearchSources = true - } -} - // WithResponsesAPIFunc sets a custom filter for which models use the Responses API. // When set, this function is called instead of the default IsResponsesModel(). func WithResponsesAPIFunc(fn func(modelID string) bool) Option { @@ -218,7 +207,7 @@ func (o *provider) LanguageModel(_ context.Context, modelID string) (fantasy.Lan // The responses language model has its own parameter and usage // handling; of the language model options, only the header hook // is shared with the chat completions model. - return newResponsesLanguageModel(modelID, o.options.name, client, objectMode, o.options.reasoningModelFunc, languageModelHeaderFunc(o.options.languageModelOptions), o.options.skipWebSearchSources), nil + return newResponsesLanguageModel(modelID, o.options.name, client, objectMode, o.options.reasoningModelFunc, languageModelHeaderFunc(o.options.languageModelOptions)), nil } languageModelOptions := append([]LanguageModelOption{}, o.options.languageModelOptions...) diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 5aca6d44c..8d70818b1 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -26,25 +26,23 @@ import ( const topLogprobsMax = 20 type responsesLanguageModel struct { - provider string - modelID string - client openai.Client - objectMode fantasy.ObjectMode - reasoningModelFunc func(modelID string) bool - headerFunc LanguageModelHeaderFunc - skipWebSearchSources bool + provider string + modelID string + client openai.Client + objectMode fantasy.ObjectMode + reasoningModelFunc func(modelID string) bool + headerFunc LanguageModelHeaderFunc } // newResponsesLanguageModel implements a responses api model. -func newResponsesLanguageModel(modelID string, provider string, client openai.Client, objectMode fantasy.ObjectMode, reasoningModelFunc func(modelID string) bool, headerFunc LanguageModelHeaderFunc, skipWebSearchSources bool) responsesLanguageModel { +func newResponsesLanguageModel(modelID string, provider string, client openai.Client, objectMode fantasy.ObjectMode, reasoningModelFunc func(modelID string) bool, headerFunc LanguageModelHeaderFunc) responsesLanguageModel { return responsesLanguageModel{ - modelID: modelID, - provider: provider, - client: client, - objectMode: objectMode, - reasoningModelFunc: reasoningModelFunc, - headerFunc: headerFunc, - skipWebSearchSources: skipWebSearchSources, + modelID: modelID, + provider: provider, + client: client, + objectMode: objectMode, + reasoningModelFunc: reasoningModelFunc, + headerFunc: headerFunc, } } @@ -398,7 +396,7 @@ func (o responsesLanguageModel) prepareParams(call fantasy.Call) (*responses.Res params.ToolChoice = toolChoice } - if !o.skipWebSearchSources && hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { + if hasResponsesWebSearchTool(tools) && !slices.Contains(include, IncludeWebSearchCallActionSources) { include = append(include, IncludeWebSearchCallActionSources) } diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index 52c2bcccd..20ac163f4 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -190,8 +190,7 @@ const ( IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" // IncludeWebSearchCallActionSources includes the pages a web search // found in web_search_call action.sources. Requests with a web - // search tool add it automatically unless the provider was created with - // WithoutWebSearchSources. + // search tool add it automatically. IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" ) diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 5c0f1bdb7..d9cff2c13 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -448,59 +448,4 @@ func TestResponsesWebSearchSourcesInclude(t *testing.T) { requireWebSearchSourcesInclude(t, server.calls[0].body) require.Contains(t, server.calls[0].body["include"], string(IncludeReasoningEncryptedContent)) }) - - newOptedOutModel := func(t *testing.T, serverURL string) fantasy.LanguageModel { - t.Helper() - provider, err := New( - WithAPIKey("test-api-key"), - WithBaseURL(serverURL), - WithUseResponsesAPI(), - WithoutWebSearchSources(), - ) - require.NoError(t, err) - model, err := provider.LanguageModel(context.Background(), "gpt-4.1") - require.NoError(t, err) - return model - } - - t.Run("NotRequestedWhenProviderOptsOut", func(t *testing.T) { - t.Parallel() - - server := newMockServer() - defer server.close() - server.response = mockResponsesWebSearchResponse() - - model := newOptedOutModel(t, server.server.URL) - _, err := model.Generate(context.Background(), fantasy.Call{ - Prompt: testPrompt, - Tools: []fantasy.Tool{WebSearchTool(nil)}, - }) - require.NoError(t, err) - require.Len(t, server.calls, 1) - - include, _ := server.calls[0].body["include"].([]any) - require.NotContains(t, include, string(IncludeWebSearchCallActionSources)) - }) - - t.Run("CallerIncludeSentWhenProviderOptsOut", func(t *testing.T) { - t.Parallel() - - server := newMockServer() - defer server.close() - server.response = mockResponsesWebSearchResponse() - - model := newOptedOutModel(t, server.server.URL) - _, err := model.Generate(context.Background(), fantasy.Call{ - Prompt: testPrompt, - Tools: []fantasy.Tool{WebSearchTool(nil)}, - ProviderOptions: fantasy.ProviderOptions{ - Name: &ResponsesProviderOptions{ - Include: []IncludeType{IncludeWebSearchCallActionSources}, - }, - }, - }) - require.NoError(t, err) - require.Len(t, server.calls, 1) - requireWebSearchSourcesInclude(t, server.calls[0].body) - }) } diff --git a/providers/openaicompat/openaicompat.go b/providers/openaicompat/openaicompat.go index 21da8fb11..af8948579 100644 --- a/providers/openaicompat/openaicompat.go +++ b/providers/openaicompat/openaicompat.go @@ -123,15 +123,6 @@ func WithUseResponsesAPI() Option { } } -// WithoutWebSearchSources stops Responses requests with a web search tool -// from automatically including web_search_call.action.sources, which some -// compatible backends, such as Bedrock Mantle, reject. -func WithoutWebSearchSources() Option { - return func(o *options) { - o.openaiOptions = append(o.openaiOptions, openai.WithoutWebSearchSources()) - } -} - // WithResponsesAPIFunc sets a custom filter for which models use the Responses API. func WithResponsesAPIFunc(fn func(modelID string) bool) Option { return func(o *options) { diff --git a/providers/openaicompat/web_search_sources_test.go b/providers/openaicompat/web_search_sources_test.go deleted file mode 100644 index dda48f1d6..000000000 --- a/providers/openaicompat/web_search_sources_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package openaicompat - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "charm.land/fantasy" - "charm.land/fantasy/providers/openai" - "github.com/stretchr/testify/require" -) - -func TestWithoutWebSearchSources(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - opts []Option - wantInclude bool - }{ - {name: "Default", wantInclude: true}, - {name: "OptedOut", opts: []Option{WithoutWebSearchSources()}, wantInclude: false}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - var ( - mu sync.Mutex - include []string - ) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body struct { - Include []string `json:"include"` - } - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - mu.Lock() - include = body.Include - mu.Unlock() - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"id":"resp_01","object":"response","status":"completed","model":"gpt-4.1","output":[{"type":"message","id":"msg_01","status":"completed","role":"assistant","content":[{"type":"output_text","text":"ok","annotations":[]}]}]}`)) - })) - defer server.Close() - - provider, err := New(append([]Option{ - WithBaseURL(server.URL), - WithAPIKey("test-api-key"), - WithUseResponsesAPI(), - }, tc.opts...)...) - require.NoError(t, err) - model, err := provider.LanguageModel(context.Background(), "gpt-4.1") - require.NoError(t, err) - - _, err = model.Generate(context.Background(), fantasy.Call{ - Prompt: fantasy.Prompt{{ - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{fantasy.TextPart{Text: "search"}}, - }}, - Tools: []fantasy.Tool{openai.WebSearchTool(nil)}, - }) - require.NoError(t, err) - - mu.Lock() - defer mu.Unlock() - if tc.wantInclude { - require.Contains(t, include, string(openai.IncludeWebSearchCallActionSources)) - } else { - require.NotContains(t, include, string(openai.IncludeWebSearchCallActionSources)) - } - }) - } -} From b8d250af1abf8f771783b973513322d6228833f2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 13 Oct 2026 11:52:19 +0000 Subject: [PATCH 08/10] test(providers/openai): fold the failed web search into the action table The stream-only failed-status test repeated the action table's setup, and as a table case it also covers Generate. --- providers/openai/responses_web_search_test.go | 72 ++++++++----------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index d9cff2c13..66562f9b7 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -13,14 +13,18 @@ import ( // webSearchCallActionCases covers the web_search_call action shapes // OpenAI returns: searches with queries and found pages, older responses // with the deprecated single query, searches that report no query, and the -// open_page and find_in_page actions of reasoning models. -var webSearchCallActionCases = []struct { +// open_page and find_in_page actions of reasoning models. Status defaults +// to "completed". +type webSearchCallActionCase struct { name string + status string action map[string]any want *WebSearchAction wantInput string wantFound []string -}{ +} + +var webSearchCallActionCases = []webSearchCallActionCase{ { name: "QueriesAndSources", action: map[string]any{ @@ -97,6 +101,26 @@ var webSearchCallActionCases = []struct { }, wantInput: `{"type":"find_in_page","url":"https://go.dev/doc/devel/release","pattern":"go1.27"}`, }, + { + name: "Failed", + status: "failed", + action: map[string]any{ + "type": "search", + "queries": []any{"tokyo population"}, + }, + want: &WebSearchAction{ + Type: "search", + Queries: []string{"tokyo population"}, + }, + wantInput: `{"type":"search","queries":["tokyo population"]}`, + }, +} + +func (tc webSearchCallActionCase) itemStatus() string { + if tc.status == "" { + return "completed" + } + return tc.status } func requireWebSearchCallMetadata(t *testing.T, metadata fantasy.ProviderMetadata, wantStatus string, want *WebSearchAction) { @@ -153,7 +177,7 @@ func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { map[string]any{ "type": "web_search_call", "id": "ws_01", - "status": "completed", + "status": tc.itemStatus(), "action": tc.action, }, map[string]any{ @@ -212,7 +236,7 @@ func TestResponsesGenerate_WebSearchCallAction(t *testing.T) { require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolName) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "completed", tc.want) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.itemStatus(), tc.want) var found, cited []string for _, source := range sources { @@ -241,7 +265,7 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { item, err := json.Marshal(map[string]any{ "type": "web_search_call", "id": "ws_01", - "status": "completed", + "status": tc.itemStatus(), "action": tc.action, }) require.NoError(t, err) @@ -293,7 +317,7 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { require.Len(t, toolResults, 1) require.True(t, toolResults[0].ProviderExecuted) require.Equal(t, "web_search", toolResults[0].ToolCallName) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "completed", tc.want) + requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, tc.itemStatus(), tc.want) var found, cited []string for _, source := range sources { @@ -370,40 +394,6 @@ func TestResponsesStream_WebSearchCallFinishesWithItem(t *testing.T) { require.Equal(t, []string{"https://example.com/tokyo"}, cited) } -func TestResponsesStream_WebSearchCallFailedStatus(t *testing.T) { - t.Parallel() - - sms := newStreamingMockServer() - defer sms.close() - sms.chunks = []string{ - responsesSSEEvent("response.output_item.done", - `{"type":"response.output_item.done","output_index":0,"item":{"type":"web_search_call","id":"ws_01","status":"failed","action":{"type":"search","queries":["tokyo population"]}}}`), - responsesSSEEvent("response.completed", - `{"type":"response.completed","response":{"id":"resp_01","status":"completed","output":[],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`), - } - - model := newResponsesProvider(t, sms.server.URL) - stream, err := model.Stream(context.Background(), fantasy.Call{ - Prompt: testPrompt, - Tools: []fantasy.Tool{WebSearchTool(nil)}, - }) - require.NoError(t, err) - - var toolResults []fantasy.StreamPart - for part := range stream { - require.NotEqual(t, fantasy.StreamPartTypeError, part.Type, "unexpected stream error: %v", part.Error) - if part.Type == fantasy.StreamPartTypeToolResult { - toolResults = append(toolResults, part) - } - } - - require.Len(t, toolResults, 1) - requireWebSearchCallMetadata(t, toolResults[0].ProviderMetadata, "failed", &WebSearchAction{ - Type: "search", - Queries: []string{"tokyo population"}, - }) -} - func TestResponsesWebSearchSourcesInclude(t *testing.T) { t.Parallel() From f9b82dd31326f100d9045b6c671becea53962fa3 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Mon, 26 Oct 2026 15:24:35 +0000 Subject: [PATCH 09/10] fix: keep SourceContent.ToolCallID through JSON SourceContent's custom MarshalJSON and UnmarshalJSON build their own field lists, so the tool call ID that tags a found page was dropped on a round trip and the page read back as a citation. --- content_json.go | 4 ++++ json_test.go | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/content_json.go b/content_json.go index 8295eea1e..176ca44ac 100644 --- a/content_json.go +++ b/content_json.go @@ -187,6 +187,7 @@ func (s SourceContent) MarshalJSON() ([]byte, error) { Title string `json:"title,omitempty"` MediaType string `json:"media_type,omitempty"` Filename string `json:"filename,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` ProviderMetadata ProviderMetadata `json:"provider_metadata,omitempty"` }{ SourceType: s.SourceType, @@ -195,6 +196,7 @@ func (s SourceContent) MarshalJSON() ([]byte, error) { Title: s.Title, MediaType: s.MediaType, Filename: s.Filename, + ToolCallID: s.ToolCallID, ProviderMetadata: s.ProviderMetadata, }) if err != nil { @@ -221,6 +223,7 @@ func (s *SourceContent) UnmarshalJSON(data []byte) error { Title string `json:"title,omitempty"` MediaType string `json:"media_type,omitempty"` Filename string `json:"filename,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` ProviderMetadata map[string]json.RawMessage `json:"provider_metadata,omitempty"` } @@ -234,6 +237,7 @@ func (s *SourceContent) UnmarshalJSON(data []byte) error { s.Title = aux.Title s.MediaType = aux.MediaType s.Filename = aux.Filename + s.ToolCallID = aux.ToolCallID if len(aux.ProviderMetadata) > 0 { metadata, err := UnmarshalProviderMetadata(aux.ProviderMetadata) diff --git a/json_test.go b/json_test.go index f461300b8..aa0e952c3 100644 --- a/json_test.go +++ b/json_test.go @@ -610,6 +610,25 @@ func (m *mockProviderData) UnmarshalJSON(data []byte) error { return nil } +func TestSourceContentJSONRoundTrip(t *testing.T) { + for _, source := range []SourceContent{ + {SourceType: SourceTypeURL, ID: "found", URL: "https://example.com", ToolCallID: "ws_1"}, + {SourceType: SourceTypeURL, ID: "cited", URL: "https://example.com", Title: "Example"}, + } { + data, err := json.Marshal(source) + if err != nil { + t.Fatalf("failed to marshal source: %v", err) + } + decoded, err := UnmarshalContent(data) + if err != nil { + t.Fatalf("failed to unmarshal source: %v", err) + } + if !reflect.DeepEqual(decoded, source) { + t.Errorf("round trip mismatch: got %#v, want %#v", decoded, source) + } + } +} + func TestPromptSerialization(t *testing.T) { t.Run("serialize prompt (message slice)", func(t *testing.T) { prompt := Prompt{ From 332835cd4956b03aee39ae7c10afbece9eb1175a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 27 Oct 2026 10:01:34 +0000 Subject: [PATCH 10/10] docs: trim web search comments Keep each explanation in one place and match the surrounding comment style. Restore upstream comments that still hold. --- content.go | 17 +++++------- model.go | 9 +++---- providers/anthropic/anthropic.go | 5 ++-- providers/anthropic/anthropic_test.go | 3 +-- providers/openai/responses_language_model.go | 27 ++++++++----------- providers/openai/responses_options.go | 19 ++++++------- providers/openai/responses_web_search_test.go | 19 +++++-------- providertests/openai_web_search_test.go | 10 ++++--- 8 files changed, 45 insertions(+), 64 deletions(-) diff --git a/content.go b/content.go index e7cccd480..45cab6ee1 100644 --- a/content.go +++ b/content.go @@ -412,16 +412,13 @@ const ( // SourceContent represents a source that has been used as input to generate the response. type SourceContent struct { - SourceType SourceType `json:"source_type"` // "url" or "document" - ID string `json:"id"` - URL string `json:"url"` // for URL sources - Title string `json:"title"` - MediaType string `json:"media_type"` // for document sources (IANA media type) - Filename string `json:"filename"` // for document sources - // ToolCallID is the provider-executed tool call, such as a web search, - // that returned this source. It is empty for sources the response text - // cites. - ToolCallID string `json:"tool_call_id,omitempty"` + SourceType SourceType `json:"source_type"` // "url" or "document" + ID string `json:"id"` + URL string `json:"url"` // for URL sources + Title string `json:"title"` + MediaType string `json:"media_type"` // for document sources (IANA media type) + Filename string `json:"filename"` // for document sources + ToolCallID string `json:"tool_call_id,omitempty"` // for sources a provider-executed tool call returned ProviderMetadata ProviderMetadata `json:"provider_metadata"` } diff --git a/model.go b/model.go index b248f465c..5a0561fa2 100644 --- a/model.go +++ b/model.go @@ -184,11 +184,10 @@ type StreamPart struct { Warnings []CallWarning `json:"warnings"` // Source-related fields - SourceType SourceType `json:"source_type"` - URL string `json:"url"` - Title string `json:"title"` - // SourceToolCallID is SourceContent.ToolCallID for source parts. - SourceToolCallID string `json:"source_tool_call_id,omitempty"` + SourceType SourceType `json:"source_type"` + URL string `json:"url"` + Title string `json:"title"` + SourceToolCallID string `json:"source_tool_call_id,omitempty"` ProviderMetadata ProviderMetadata `json:"provider_metadata"` } diff --git a/providers/anthropic/anthropic.go b/providers/anthropic/anthropic.go index 38c70cfe0..6d5d591c2 100644 --- a/providers/anthropic/anthropic.go +++ b/providers/anthropic/anthropic.go @@ -1683,9 +1683,8 @@ func (a languageModel) Generate(ctx context.Context, call fantasy.Call) (*fantas if !ok { continue } - // Emit found pages as sources tagged with the search's ID, - // and keep them in metadata with encrypted_content for - // multi-turn round-tripping. + // Extract search results as sources/citations, preserving + // encrypted_content for multi-turn round-tripping. toolResult := fantasy.ToolResultContent{ ToolCallID: webSearchResult.ToolUseID, ToolName: "web_search", diff --git a/providers/anthropic/anthropic_test.go b/providers/anthropic/anthropic_test.go index 411e33768..84f0f9e34 100644 --- a/providers/anthropic/anthropic_test.go +++ b/providers/anthropic/anthropic_test.go @@ -2166,8 +2166,7 @@ func TestGenerate_WebSearchResponse(t *testing.T) { require.Equal(t, "web_search", toolCalls[0].ToolName) require.Equal(t, "srvtoolu_01", toolCalls[0].ToolCallID) - // SourceContent entries for each search result, tagged with the - // search that found them. + // SourceContent entries for each search result. require.Len(t, sources, 2) require.Equal(t, "https://example.com/ai-news", sources[0].URL) require.Equal(t, "Latest AI News", sources[0].Title) diff --git a/providers/openai/responses_language_model.go b/providers/openai/responses_language_model.go index 8d70818b1..4a92085b5 100644 --- a/providers/openai/responses_language_model.go +++ b/providers/openai/responses_language_model.go @@ -1200,12 +1200,10 @@ func (o responsesLanguageModel) Generate(ctx context.Context, call fantasy.Call) // a ToolCallContent and ToolResultContent as a pair, // matching the vercel/ai pattern for provider tools. // - // Found pages become sources tagged with the search's ID. - // Citations come untagged from url_citation annotations on - // the message text (handled in the "message" case above). - // A non-streamed response only has the final output, where - // OpenAI also lists pages the answer cited among the - // search's sources, so those appear as found pages here. + // Note: source citations come from url_citation annotations + // on the message text (handled in the "message" case above), + // not from the web_search_call action. Unlike the streamed + // item, the final output's action.sources also lists them. content = append(content, fantasy.ToolCallContent{ ProviderExecuted: true, ToolCallID: outputItem.ID, @@ -1462,10 +1460,9 @@ func (o responsesLanguageModel) Stream(ctx context.Context, call fantasy.Call) ( } case "web_search_call": - // Provider-executed web search completed. Its found - // pages are sources tagged with the search's ID; - // citations come untagged from url_citation - // annotations on the message text. + // Provider-executed web search completed. + // Source citations come from url_citation annotations + // on the streamed message text, not from the action. if !yield(fantasy.StreamPart{ Type: fantasy.StreamPartTypeToolInputEnd, ID: done.Item.ID, @@ -1822,8 +1819,8 @@ func responsesTextMetadata(itemID string, phase responses.ResponseOutputMessageP } } -// webSearchCallSources returns the pages a web_search_call found as URL -// sources tagged with the call's ID. +// webSearchCallSources converts the pages a web search call found into +// sources. func webSearchCallSources(itemID string, action responses.ResponseOutputItemUnionAction) []fantasy.SourceContent { sources := make([]fantasy.SourceContent, 0, len(action.Sources)) for _, src := range action.Sources { @@ -1840,10 +1837,8 @@ func webSearchCallSources(itemID string, action responses.ResponseOutputItemUnio return sources } -// webSearchCallInput encodes the web_search_call action as its tool call -// input: the action type with the queries of a search (falling back to the -// deprecated query), the URL of open_page, or the URL and pattern of -// find_in_page. +// webSearchCallInput encodes a web search call's action as its tool call +// input. func webSearchCallInput(action responses.ResponseOutputItemUnionAction) string { queries := action.Queries if len(queries) == 0 && action.Query != "" { diff --git a/providers/openai/responses_options.go b/providers/openai/responses_options.go index 20ac163f4..26bdb79dd 100644 --- a/providers/openai/responses_options.go +++ b/providers/openai/responses_options.go @@ -188,9 +188,8 @@ const ( IncludeFileSearchCallResults IncludeType = "file_search_call.results" // IncludeMessageOutputTextLogprobs includes message output text log probabilities. IncludeMessageOutputTextLogprobs IncludeType = "message.output_text.logprobs" - // IncludeWebSearchCallActionSources includes the pages a web search - // found in web_search_call action.sources. Requests with a web - // search tool add it automatically. + // IncludeWebSearchCallActionSources includes web search call sources. + // It is added whenever a web search tool is set. IncludeWebSearchCallActionSources IncludeType = "web_search_call.action.sources" ) @@ -454,17 +453,15 @@ type WebSearchSource struct { type WebSearchAction struct { // Type is the kind of action: "search", "open_page", or "find_in_page". Type string `json:"type"` - // Queries are the search queries. OpenAI can omit them. + // Queries are the search queries (present when Type is "search"). Queries []string `json:"queries,omitempty"` - // Query is the deprecated single-query field that OpenAI replaced - // with Queries. + // Query is the search query (deprecated by OpenAI in favor of Queries). Query string `json:"query,omitempty"` - // URL is the page an open_page or find_in_page action used. + // URL is the page URL (present when Type is "open_page" or "find_in_page"). URL string `json:"url,omitempty"` - // Pattern is the text a find_in_page action looked for. + // Pattern is the text to find (present when Type is "find_in_page"). Pattern string `json:"pattern,omitempty"` - // Sources are the pages the search found. OpenAI only returns them - // when the request includes web_search_call.action.sources. + // Sources are the results returned by the search. Sources []WebSearchSource `json:"sources,omitempty"` } @@ -475,7 +472,7 @@ type WebSearchAction struct { type WebSearchCallMetadata struct { // ItemID is the server-side ID of the web_search_call output item. ItemID string `json:"item_id"` - // Status is the item's status, such as "completed" or "failed". + // Status is the status of the web_search_call output item. Status string `json:"status,omitempty"` // Action contains the structured action data from the search. Action *WebSearchAction `json:"action,omitempty"` diff --git a/providers/openai/responses_web_search_test.go b/providers/openai/responses_web_search_test.go index 66562f9b7..1fc7d59a1 100644 --- a/providers/openai/responses_web_search_test.go +++ b/providers/openai/responses_web_search_test.go @@ -10,11 +10,8 @@ import ( "github.com/stretchr/testify/require" ) -// webSearchCallActionCases covers the web_search_call action shapes -// OpenAI returns: searches with queries and found pages, older responses -// with the deprecated single query, searches that report no query, and the -// open_page and find_in_page actions of reasoning models. Status defaults -// to "completed". +// webSearchCallActionCases are the web_search_call action shapes OpenAI +// returns. type webSearchCallActionCase struct { name string status string @@ -132,8 +129,7 @@ func requireWebSearchCallMetadata(t *testing.T, metadata fantasy.ProviderMetadat require.Equal(t, wantStatus, wsMeta.Status) require.Equal(t, want, wsMeta.Action) - // Callers persist the metadata as JSON and read it back on later - // turns, so the action fields must survive the round trip. + // The metadata must survive a JSON round trip. encoded, err := json.Marshal(metadata) require.NoError(t, err) var raw map[string]json.RawMessage @@ -336,12 +332,9 @@ func TestResponsesStream_WebSearchCallAction(t *testing.T) { } } -// TestResponsesStream_WebSearchCallFinishesWithItem covers live gpt-4.1-mini -// and gpt-5-mini streams: a search's found pages are on its output_item.done, -// and the final response summary also lists pages the answer cited among -// them. gpt-4.1 search items carried no pages, and its summary listed only -// cited pages. The search finishes, with its own pages, before the answer -// streams. +// TestResponsesStream_WebSearchCallFinishesWithItem checks that a search +// reports the pages of its output_item.done, not the cited pages that +// response.completed adds to its sources. func TestResponsesStream_WebSearchCallFinishesWithItem(t *testing.T) { t.Parallel() diff --git a/providertests/openai_web_search_test.go b/providertests/openai_web_search_test.go index 2a0535191..7c76afbe3 100644 --- a/providertests/openai_web_search_test.go +++ b/providertests/openai_web_search_test.go @@ -80,6 +80,9 @@ func TestOpenAIWebSearch(t *testing.T) { require.NotEmpty(t, providerToolCalls, "should have provider-executed tool calls") require.Equal(t, "web_search", providerToolCalls[0].ToolName) requireWebSearchActionMetadata(t, providerToolResults, sources) + // Sources come from url_citation annotations and found pages; + // the model may or may not include inline citations so we don't + // require them, but if present they should have URLs. for _, src := range sources { require.NotEmpty(t, src.URL, "source should have a URL") } @@ -135,10 +138,9 @@ func TestOpenAIWebSearch(t *testing.T) { }) } -// requireWebSearchActionMetadata checks that web_search results carry the -// search queries, and that every page a search found is a source tagged -// with that search's ID. The recorded gpt-4.1 stream reports no found pages -// on its search item, so found pages are not required. +// requireWebSearchActionMetadata checks that web_search results carry their +// queries and that found pages are tagged with their search. Found pages are +// optional because gpt-4.1 streams report none. func requireWebSearchActionMetadata(t *testing.T, results []fantasy.ToolResultContent, sources []fantasy.SourceContent) { t.Helper()