diff --git a/packages/api/src/harness/inference.test.ts b/packages/api/src/harness/inference.test.ts index cef9e6c1..34413f02 100644 --- a/packages/api/src/harness/inference.test.ts +++ b/packages/api/src/harness/inference.test.ts @@ -329,7 +329,7 @@ describe('runInference', () => { test('a log grown long is summarized first, and the model reads the summary in place of what it covers', async () => { const {runInference} = await import('./inference'); - const {COMPACT_AT_TOKENS} = await import('./context'); + const {COMPACT_AT_TOKENS} = await import('@proxy/utils/harness-context'); const long = [ entry('person', 'One'), entry('assistant', '', {toolCalls: [{id: 'c1', name: 'read', arguments: '{}'}], inputTokens: 10}), diff --git a/packages/api/src/harness/inference.ts b/packages/api/src/harness/inference.ts index d3056b2a..4fac357e 100644 --- a/packages/api/src/harness/inference.ts +++ b/packages/api/src/harness/inference.ts @@ -7,7 +7,7 @@ import {askForTurn, type ModelTurn} from '../llm/open-router'; import {agentInstructions} from '../mcp/tools'; import {log, serializeError} from '../observability/log'; import {env} from '../utils/env'; -import {compactionCut, toMessages, toTranscript} from './context'; +import {compactionCut, toMessages, toTranscript} from '@proxy/utils/harness-context'; import {afterAnswer, planCalls} from './conversation'; import type {HarnessStore} from './store'; import {toolsFor} from './tool-surface'; diff --git a/packages/api/src/llm/open-router.ts b/packages/api/src/llm/open-router.ts index 6d123a34..a70e94c4 100644 --- a/packages/api/src/llm/open-router.ts +++ b/packages/api/src/llm/open-router.ts @@ -3,6 +3,7 @@ import {z} from 'zod'; import {ApiErr, type ApiError, type ChatModel, Do, type FetchError, httpRequest, parseSchema} from '@proxy/utils'; import type {ToolCall} from '@proxy/utils/harness'; +import type {ModelMessage} from '@proxy/utils/harness-context'; /** * The models this server asks over OpenRouter's chat completions, with the key in @@ -36,14 +37,8 @@ const MAX_SAID = 500; /** A tool as the model is given it: its name, what it does, and the JSON Schema of its arguments. */ export type ModelTool = {name: string; description: string; parameters: object}; -type ModelToolCall = {id: string; type: 'function'; function: {name: string; arguments: string}}; - -/** The conversation as the model reads it, in the chat completions form. */ -export type ModelMessage = - | {role: 'system'; content: string} - | {role: 'user'; content: string} - | {role: 'assistant'; content: string | null; tool_calls?: ModelToolCall[]} - | {role: 'tool'; tool_call_id: string; content: string}; +/** The conversation as the model reads it, in the chat completions form (@proxy/utils/harness-context, which builds it from a log). */ +export type {ModelMessage}; // Which hosts of the model OpenRouter may send the request to. type ProviderPolicy = {order?: string[]; allow_fallbacks?: boolean; data_collection: 'deny'; zdr?: boolean}; diff --git a/packages/db/src/harness.ts b/packages/db/src/harness.ts index 239c2c8f..0a5d1a1f 100644 --- a/packages/db/src/harness.ts +++ b/packages/db/src/harness.ts @@ -211,7 +211,8 @@ function toPersona(persona: { return {...persona, look: parseLook(persona.look)}; } -const entrySelect = { +/** The columns of an entry, as `EntryRow` has them; the admin reads with it too. */ +export const entrySelect = { id: true, seq: true, threadId: true, @@ -251,11 +252,12 @@ const jobEventSelect = { at: true, } as const; -type EntryRecord = Prisma.AgentEntryGetPayload<{select: typeof entrySelect}>; +export type EntryRecord = Prisma.AgentEntryGetPayload<{select: typeof entrySelect}>; type Tx = Prisma.TransactionClient; -function toEntry(entry: EntryRecord): EntryRow { +/** An entry as read, its calls parsed. */ +export function toEntry(entry: EntryRecord): EntryRow { return { ...entry, toolCalls: entry.toolCalls === null ? null : (JSON.parse(entry.toolCalls) as ToolCall[]), diff --git a/packages/utils/package.json b/packages/utils/package.json index 71a5121a..b5b024ad 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -9,6 +9,7 @@ "./days": "./src/days.ts", "./email-providers": "./src/email-providers.ts", "./harness": "./src/harness.ts", + "./harness-context": "./src/harness-context.ts", "./secret-crypto": "./src/secret-crypto.ts", "./tasks": "./src/tasks.ts", "./units": "./src/units.ts", diff --git a/packages/api/src/harness/context.test.ts b/packages/utils/src/harness-context.test.ts similarity index 65% rename from packages/api/src/harness/context.test.ts rename to packages/utils/src/harness-context.test.ts index b59eab12..7d142296 100644 --- a/packages/api/src/harness/context.test.ts +++ b/packages/utils/src/harness-context.test.ts @@ -1,35 +1,37 @@ import {describe, expect, test} from 'bun:test'; -import type {EntryRow} from '@proxy/db/harness'; +import { + COMPACT_AT_TOKENS, + compactionCut, + type ContextEntry, + toMessages, + toTranscript, +} from './harness-context'; -import {COMPACT_AT_TOKENS, compactionCut, toMessages, toTranscript} from './context'; +type Kind = ContextEntry['kind']; -const now = new Date('2026-10-21T10:00:00Z'); - -type Kind = EntryRow['kind']; - -function entry(seq: number, kind: Kind, text: string, extra: Partial> = {}): EntryRow { +function entry( + seq: number, + kind: Kind, + text: string, + extra: Partial> = {}, +): ContextEntry { return { - id: `e${seq}`, seq, - threadId: 'thread-1', kind, - pending: false, text, toolCalls: extra.toolCalls ?? null, callId: extra.callId ?? null, isError: false, inputTokens: extra.inputTokens ?? null, - outputTokens: null, coversSeq: extra.coversSeq ?? null, - system: null, - jobId: null, - createdAt: now, }; } -const read = (seq: number, id: string, inputTokens = 10) => entry(seq, 'assistant', '', {toolCalls: [{id, name: 'read', arguments: '{}'}], inputTokens}); -const result = (seq: number, id: string, text = 'rows') => entry(seq, 'tool_result', text, {callId: id}); +const read = (seq: number, id: string, inputTokens = 10) => + entry(seq, 'assistant', '', {toolCalls: [{id, name: 'read', arguments: '{}'}], inputTokens}); +const result = (seq: number, id: string, text = 'rows') => + entry(seq, 'tool_result', text, {callId: id}); // Three turns of the person, each answered after a read. const long = [ @@ -50,7 +52,11 @@ describe('toMessages', () => { test('gives the log in the chat form, as it is, where it has no summary', () => { expect(toMessages(long.slice(0, 4), [], {clearOld: true})).toEqual([ {role: 'user', content: 'One'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c1', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c1', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c1', content: 'rows one'}, {role: 'assistant', content: 'Done one'}, ]); @@ -61,33 +67,72 @@ describe('toMessages', () => { const messages = toMessages(log, [], {clearOld: true}); - expect(messages[0]).toEqual({role: 'user', content: expect.stringContaining('The person asked for one.')}); + expect(messages[0]).toEqual({ + role: 'user', + content: expect.stringContaining('The person asked for one.'), + }); expect(String(messages[0]?.content)).toContain('The person did not write this'); expect(messages.slice(1)).toEqual([ {role: 'user', content: 'Two'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c2', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c2', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c2', content: expect.stringContaining('cleared')}, {role: 'assistant', content: 'Done two'}, {role: 'user', content: 'Three'}, - {role: 'assistant', content: null, tool_calls: [{id: 'c3', type: 'function', function: {name: 'read', arguments: '{}'}}]}, + { + role: 'assistant', + content: null, + tool_calls: [{id: 'c3', type: 'function', function: {name: 'read', arguments: '{}'}}], + }, {role: 'tool', tool_call_id: 'c3', content: 'rows three'}, ]); - expect(toMessages(log, [], {clearOld: false})[3]).toEqual({role: 'tool', tool_call_id: 'c2', content: 'rows two'}); + expect(toMessages(log, [], {clearOld: false})[3]).toEqual({ + role: 'tool', + tool_call_id: 'c2', + content: 'rows two', + }); }); test('reads what a sub-agent said as a message, led in with whose it is, by id, and clears old ones too', () => { const delegated = entry(1, 'person', 'Find my invoices'); - const call = entry(2, 'assistant', 'Looking.', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}'}], inputTokens: 10}); + const call = entry(2, 'assistant', 'Looking.', { + toolCalls: [ + { + id: 'd1', + name: 'create_subagent', + arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}', + }, + ], + inputTokens: 10, + }); const started = entry(3, 'tool_result', 'Started, id sub-1.', {callId: 'd1'}); const report = entry(4, 'event', 'Three invoices from Dana.', {callId: 'd1'}); - const failed = {...entry(5, 'event', 'The sub-agent failed: boom', {callId: 'd1'}), isError: true}; + const failed = { + ...entry(5, 'event', 'The sub-agent failed: boom', {callId: 'd1'}), + isError: true, + }; const subAgents = [{id: 'sub-1', parentCallId: 'd1', title: 'Find the invoices'}]; - const messages = toMessages([delegated, call, started, report, failed], subAgents, {clearOld: true}); + const messages = toMessages([delegated, call, started, report, failed], subAgents, { + clearOld: true, + }); - expect(messages[3]).toEqual({role: 'user', content: expect.stringContaining('A message from the sub-agent “Find the invoices” (id sub-1)')}); + expect(messages[3]).toEqual({ + role: 'user', + content: expect.stringContaining( + 'A message from the sub-agent “Find the invoices” (id sub-1)', + ), + }); expect(String(messages[3]?.content)).toContain('Three invoices from Dana.'); - expect(messages[4]).toEqual({role: 'user', content: expect.stringContaining('What the sub-agent “Find the invoices” (id sub-1) failed with')}); + expect(messages[4]).toEqual({ + role: 'user', + content: expect.stringContaining( + 'What the sub-agent “Find the invoices” (id sub-1) failed with', + ), + }); const later = [ delegated, @@ -104,13 +149,32 @@ describe('toMessages', () => { }); test('leads a message in by the call where no sub-agent was started by it', () => { - const call = entry(2, 'assistant', '', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":'}], inputTokens: 10}); - const messages = toMessages([entry(1, 'person', 'Go'), call, entry(3, 'tool_result', 'Started.', {callId: 'd1'}), entry(4, 'event', 'Done.', {callId: 'd1'})], [], {clearOld: true}); - expect(messages[3]).toEqual({role: 'user', content: expect.stringContaining('A message from your call d1')}); + const call = entry(2, 'assistant', '', { + toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":'}], + inputTokens: 10, + }); + const messages = toMessages( + [ + entry(1, 'person', 'Go'), + call, + entry(3, 'tool_result', 'Started.', {callId: 'd1'}), + entry(4, 'event', 'Done.', {callId: 'd1'}), + ], + [], + {clearOld: true}, + ); + expect(messages[3]).toEqual({ + role: 'user', + content: expect.stringContaining('A message from your call d1'), + }); }); test('with summaries on summaries, only the last counts', () => { - const log = [...long, entry(12, 'summary', 'First.', {coversSeq: 4}), entry(13, 'summary', 'Second.', {coversSeq: 8})]; + const log = [ + ...long, + entry(12, 'summary', 'First.', {coversSeq: 4}), + entry(13, 'summary', 'Second.', {coversSeq: 8}), + ]; const messages = toMessages(log, [], {clearOld: true}); @@ -130,7 +194,9 @@ describe('compactionCut', () => { }); test('is nothing where the last turns are the whole log', () => { - expect(compactionCut([entry(1, 'person', 'One'), read(2, 'c1', COMPACT_AT_TOKENS), result(3, 'c1')])).toBeNull(); + expect( + compactionCut([entry(1, 'person', 'One'), read(2, 'c1', COMPACT_AT_TOKENS), result(3, 'c1')]), + ).toBeNull(); }); test('moves back so that no call is left without its result', () => { @@ -178,7 +244,12 @@ describe('compactionCut', () => { ]; expect(compactionCut(log)).toBe(3); - const summarized = [...log, entry(10, 'summary', 'Two reads so far.', {coversSeq: 3}), read(11, 'c5', COMPACT_AT_TOKENS), result(12, 'c5')]; + const summarized = [ + ...log, + entry(10, 'summary', 'Two reads so far.', {coversSeq: 3}), + read(11, 'c5', COMPACT_AT_TOKENS), + result(12, 'c5'), + ]; expect(compactionCut(summarized)).toBe(5); }); }); @@ -186,7 +257,16 @@ describe('compactionCut', () => { describe('toTranscript', () => { test('writes the log out as text from its last summary on, the calls, their answers and the sub-agents’ messages with whose they are', () => { const delegated = entry(1, 'person', 'Find my invoices'); - const call = entry(2, 'assistant', 'Looking.', {toolCalls: [{id: 'd1', name: 'create_subagent', arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}'}], inputTokens: 10}); + const call = entry(2, 'assistant', 'Looking.', { + toolCalls: [ + { + id: 'd1', + name: 'create_subagent', + arguments: '{"title":"Find the invoices","brief":"Search the mailbox."}', + }, + ], + inputTokens: 10, + }); const started = entry(3, 'tool_result', 'Started, id sub-1.', {callId: 'd1'}); const report = entry(4, 'event', 'Three invoices from Dana.', {callId: 'd1'}); const failed = {...entry(5, 'tool_result', 'No such tool.', {callId: 'x9'}), isError: true}; @@ -202,7 +282,10 @@ describe('toTranscript', () => { ].join('\n\n'), ); - const transcript = toTranscript([...long, entry(12, 'summary', 'The person asked for one.', {coversSeq: 4})], []); + const transcript = toTranscript( + [...long, entry(12, 'summary', 'The person asked for one.', {coversSeq: 4})], + [], + ); expect(transcript.startsWith('What came before in this conversation')).toBe(true); expect(transcript).toContain('The person asked for one.\n\nPerson:\nTwo'); expect(transcript).not.toContain('rows one'); diff --git a/packages/api/src/harness/context.ts b/packages/utils/src/harness-context.ts similarity index 62% rename from packages/api/src/harness/context.ts rename to packages/utils/src/harness-context.ts index c488bf1e..ca7b8808 100644 --- a/packages/api/src/harness/context.ts +++ b/packages/utils/src/harness-context.ts @@ -1,10 +1,35 @@ -import {pendingToolCalls, type ToolCall} from '@proxy/utils/harness'; -import type {EntryRow} from '@proxy/db/harness'; - -import type {ModelMessage} from '../llm/open-router'; +import {pendingToolCalls, type ToolCall} from './harness'; + +/** The conversation as the model reads it, in the chat completions form. */ +export type ModelMessage = + | {role: 'system'; content: string} + | {role: 'user'; content: string} + | { + role: 'assistant'; + content: string | null; + tool_calls?: Array<{ + id: string; + type: 'function'; + function: {name: string; arguments: string}; + }>; + } + | {role: 'tool'; tool_call_id: string; content: string}; + +/** An entry of the log, as much of it as reading the log needs: its kind and text, its calls or the call it answers, its place, how far a summary reaches, and what an answer cost. */ +export type ContextEntry = { + kind: 'person' | 'assistant' | 'tool_result' | 'event' | 'summary'; + text: string; + toolCalls: ToolCall[] | null; + callId: string | null; + isError: boolean; + seq: number; + coversSeq: number | null; + inputTokens: number | null; +}; /** - * The log as the model reads it. An event, what a sub-agent said to the agent that started it, is + * The log as the model reads it, here rather than in the api so that the admin (the internal + * repo) can rebuild what the model was sent from a log it reads. An event, what a sub-agent said to the agent that started it, is * read as a message, since it came on its own, after the call that started the sub-agent was * answered with its id; which sub-agent, by that call, is known from the thread's sub-agents * (`SubAgent`), given alongside the log. A log that has grown long @@ -26,7 +51,8 @@ const KEEP_TURNS = 2; // person than that: a sub-agent's, one brief and the calls made on it. const KEEP_STEPS = 3; -const SUMMARY_LEAD = 'What came before in this conversation, summarized so that it fits. The person did not write this; take it as what happened.'; +const SUMMARY_LEAD = + 'What came before in this conversation, summarized so that it fits. The person did not write this; take it as what happened.'; const CLEARED = '(This was cleared from the conversation to save room. Ask again if you need it.)'; @@ -34,21 +60,25 @@ const CLEARED = '(This was cleared from the conversation to save room. Ask again export type SubAgent = {id: string; parentCallId: string | null; title: string | null}; // The call an entry answers, by the id the model gave it, found in the log. -function callOf(entries: EntryRow[], callId: string | null): ToolCall | undefined { - return entries.flatMap((entry) => entry.toolCalls ?? []).find((candidate) => candidate.id === callId); +function callOf(entries: ContextEntry[], callId: string | null): ToolCall | undefined { + return entries + .flatMap((entry) => entry.toolCalls ?? []) + .find((candidate) => candidate.id === callId); } // How an event is led in: which sub-agent said it, by the call that started it. -function eventLead(subAgents: SubAgent[], event: EntryRow): string { +function eventLead(subAgents: SubAgent[], event: ContextEntry): string { const from = subAgents.find((candidate) => candidate.parentCallId === event.callId); - const who = from ? `the sub-agent “${from.title ?? ''}” (id ${from.id})` : `your call ${event.callId ?? ''}`; + const who = from + ? `the sub-agent “${from.title ?? ''}” (id ${from.id})` + : `your call ${event.callId ?? ''}`; if (event.isError) { return `What ${who} failed with. The person did not write this.`; } return `A message from ${who}. The person did not write this; it is for you to act on or tell them of.`; } -function toMessage(subAgents: SubAgent[], entry: EntryRow): ModelMessage { +function toMessage(subAgents: SubAgent[], entry: ContextEntry): ModelMessage { if (entry.kind === 'person') { return {role: 'user', content: entry.text}; } @@ -65,16 +95,30 @@ function toMessage(subAgents: SubAgent[], entry: EntryRow): ModelMessage { return { role: 'assistant', content: entry.text === '' && calls.length > 0 ? null : entry.text, - ...(calls.length > 0 ? {tool_calls: calls.map((call) => ({id: call.id, type: 'function' as const, function: {name: call.name, arguments: call.arguments}}))} : {}), + ...(calls.length > 0 + ? { + tool_calls: calls.map((call) => ({ + id: call.id, + type: 'function' as const, + function: {name: call.name, arguments: call.arguments}, + })), + } + : {}), }; } // The log from its last summary on: the summary, then the entries it doesn't cover, no other // summary among them. -function sinceSummary(entries: EntryRow[]): {summary: EntryRow | null; tail: EntryRow[]} { +function sinceSummary(entries: ContextEntry[]): { + summary: ContextEntry | null; + tail: ContextEntry[]; +} { const summary = entries.findLast((entry) => entry.kind === 'summary') ?? null; const covered = summary?.coversSeq ?? -1; - return {summary, tail: entries.filter((entry) => entry.kind !== 'summary' && entry.seq > covered)}; + return { + summary, + tail: entries.filter((entry) => entry.kind !== 'summary' && entry.seq > covered), + }; } /** @@ -82,18 +126,29 @@ function sinceSummary(entries: EntryRow[]): {summary: EntryRow | null; tail: Ent * what events brought before the person's current turn is cleared, once there is a summary: that * is how the model reads a log, where a summary is made from the log as it is. */ -export function toMessages(entries: EntryRow[], subAgents: SubAgent[], {clearOld}: {clearOld: boolean}): ModelMessage[] { +export function toMessages( + entries: ContextEntry[], + subAgents: SubAgent[], + {clearOld}: {clearOld: boolean}, +): ModelMessage[] { const {summary, tail} = sinceSummary(entries); if (!summary) { return tail.map((entry) => toMessage(subAgents, entry)); } const current = clearOld ? tail.findLastIndex((entry) => entry.kind === 'person') : -1; - const bulky = (entry: EntryRow) => entry.kind === 'tool_result' || entry.kind === 'event'; - return [toMessage(subAgents, summary), ...tail.map((entry, index) => (bulky(entry) && index < current ? toMessage(subAgents, {...entry, text: CLEARED}) : toMessage(subAgents, entry)))]; + const bulky = (entry: ContextEntry) => entry.kind === 'tool_result' || entry.kind === 'event'; + return [ + toMessage(subAgents, summary), + ...tail.map((entry, index) => + bulky(entry) && index < current + ? toMessage(subAgents, {...entry, text: CLEARED}) + : toMessage(subAgents, entry), + ), + ]; } // An entry as the transcript has it. -function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): string { +function toLine(entries: ContextEntry[], subAgents: SubAgent[], entry: ContextEntry): string { if (entry.kind === 'person') { return `Person:\n${entry.text}`; } @@ -106,7 +161,9 @@ function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): st if (entry.kind === 'summary') { return `${SUMMARY_LEAD}\n${entry.text}`; } - const calls = (entry.toolCalls ?? []).map((call) => `Agent called ${call.name} with ${call.arguments}`); + const calls = (entry.toolCalls ?? []).map( + (call) => `Agent called ${call.name} with ${call.arguments}`, + ); return [...(entry.text === '' ? [] : [`Agent:\n${entry.text}`]), ...calls].join('\n'); } @@ -115,9 +172,11 @@ function toLine(entries: EntryRow[], subAgents: SubAgent[], entry: EntryRow): st * what they answered written out, since a request that carries them as such has to carry the * tools too, which a summary needs none of. */ -export function toTranscript(entries: EntryRow[], subAgents: SubAgent[]): string { +export function toTranscript(entries: ContextEntry[], subAgents: SubAgent[]): string { const {summary, tail} = sinceSummary(entries); - return [...(summary ? [summary] : []), ...tail].map((entry) => toLine(entries, subAgents, entry)).join('\n\n'); + return [...(summary ? [summary] : []), ...tail] + .map((entry) => toLine(entries, subAgents, entry)) + .join('\n\n'); } /** @@ -126,7 +185,7 @@ export function toTranscript(entries: EntryRow[], subAgents: SubAgent[]): string * steps, cut so that no call is left without its result. Null when the log needs none: the last * request to the model was under the limit, or nothing since the last summary would be covered. */ -export function compactionCut(entries: EntryRow[]): number | null { +export function compactionCut(entries: ContextEntry[]): number | null { const {summary, tail} = sinceSummary(entries); const lastAnswer = tail.findLast((entry) => entry.kind === 'assistant'); if ((lastAnswer?.inputTokens ?? 0) < COMPACT_AT_TOKENS) {