diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6a7bb19d..7b74fb11 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1355,7 +1355,13 @@ unverified). Messages of a chat are accepted in order; each starts a normal chat an πŸ‘€ reaction) and the answer is converted (Telegram HTML, Slack mrkdwn, Teams Markdown) and split. Chat commands: `/help`, `/agents`, `/agent ` (switch; in a Slack thread the channel follows), `/new`, `/stop`; Slack uses `/godmode `. Claude Code's own slash commands are not available from chats. Attachments (any number; -≀ 25 MB each, Telegram ≀ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Limits: 20 messages per chat and 120 per bot per minute. Backups carry +≀ 25 MB each, Telegram ≀ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Telegram voice and video +messages (`InboundFile.spoken`) are transcribed first (`transcribeRecording` in `voice/voice.ts`): their words become +the message, also as the owner's answer to a question, and the recording stays attached. The key is the OpenAI or +ElevenLabs voice key, else an OpenAI or ElevenLabs API tool the chat's agent may use (sent to that provider's own +address only); without one, or when the provider fails, the agent gets the recording and a note with a neutral reason +(the provider's own words stay in the log). A spoken answer to a question is echoed ("I heard: …"); one that couldn't +be understood leaves the question open and asks for a typed answer. Limits: 20 messages per chat and 120 per bot per minute. Backups carry connections but restore them turned off, so two machines never answer for one bot. ## Phone app diff --git a/packages/core/src/messaging/bridge.ts b/packages/core/src/messaging/bridge.ts index 356e2aa5..71766c2b 100644 --- a/packages/core/src/messaging/bridge.ts +++ b/packages/core/src/messaging/bridge.ts @@ -16,6 +16,7 @@ import { notify } from "../services/notifications"; import { getSettings } from "../services/settings"; import { bus } from "../events/bus"; import { HttpError, newId, now, parseJson } from "../util"; +import { transcribeRecording } from "../voice/voice"; import { agentIdsOf, chatRow, @@ -362,8 +363,13 @@ function conversationFor(conn: ConnectionRow, msg: InboundMessage, chat: ChatRow return conversation.id; } -async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: string; mime: string; data: string }[]; problems: string[] }> { - const files: { name: string; mime: string; data: string }[] = []; +type Attachment = { name: string; mime: string; data: string }; +/** A voice message: the attachment it arrived as, and its bytes. */ +type Recording = { file: Attachment; bytes: Uint8Array }; + +async function attachmentsOf(msg: InboundMessage): Promise<{ files: Attachment[]; recordings: Recording[]; problems: string[] }> { + const files: Attachment[] = []; + const recordings: Recording[] = []; const problems: string[] = []; let total = 0; let over = 0; @@ -386,13 +392,52 @@ async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: stri continue; } total += bytes.byteLength; - files.push({ name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") }); + const file = { name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") }; + files.push(file); + if (f.spoken) recordings.push({ file, bytes }); } catch (err) { problems.push(`${f.name}: ${err instanceof Error ? err.message : String(err)}`); } } if (over) problems.push(`${over} more didn't fit, a message carries up to ${MAX_MESSAGE_ATTACHMENT_BYTES / 1024 / 1024} MB of files`); - return { files, problems }; + return { files, recordings, problems }; +} + +/** Why a voice message has no words, for the agent and the chat: what the provider answered stays in the log. */ +function unheardReason(err: unknown): string { + const code = err instanceof HttpError ? err.code : undefined; + if (code === "voice_no_key") return "no OpenAI or ElevenLabs key is set up in Godmode (Settings β†’ Integrations)"; + if (code === "voice_invalid_key") return "the speech-to-text key was rejected"; + if (code === "voice_rate_limited") return "the speech-to-text limit is reached"; + if (code === "voice_unreachable") return "the speech-to-text service didn't answer"; + if (code === "voice_empty") return "it is empty"; + if (code === "too_large") return "it is longer than speech-to-text takes (25 MB)"; + if (code === "vault_locked") return "Godmode's vault is locked"; + return "the transcription failed"; +} + +/** What was said in the voice messages, which of them were understood, and why the others weren't. */ +async function heardIn(recordings: Recording[], agent: Agent): Promise<{ said: string; heard: Set; unheard: string[] }> { + const said: string[] = []; + const heard = new Set(); + const unheard: string[] = []; + for (const { file, bytes } of recordings) { + try { + const transcript = await transcribeRecording(new Blob([bytes as Uint8Array], { type: file.mime }), { filename: file.name, agent }); + // Spoken words never count as a slash command. + const text = transcript.replace(/^[\s/]+/, ""); + if (!text) { + unheard.push("nothing audible in it"); + continue; + } + said.push(text); + heard.add(file); + } catch (err) { + log.warn(`could not transcribe ${file.name}`, err instanceof Error ? err.message : err); + unheard.push(unheardReason(err)); + } + } + return { said: said.join("\n\n"), heard, unheard: [...new Set(unheard)] }; } async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: InboundMessage, chatId: string, from: { userId: string; owner: boolean }) { @@ -407,12 +452,18 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In await say(adapter, msg, `${agentLabel(agent)} is turned off right now.`); return; } - const { files, problems } = await attachmentsOf(msg); + const { files, recordings, problems } = await attachmentsOf(msg); if (!msg.text.trim() && !files.length) { await say(adapter, msg, `I couldn't open that file (${problems.join("; ")}).`); return; } - let content = msg.kind === "group" ? `${msg.user.name}: ${msg.text}` : msg.text; + // Listening takes a moment: the chat already shows that something is happening. + const stopListening = recordings.length ? await adapter.working(msg.target, msg.messageId).catch(() => async () => {}) : async () => {}; + const { said, heard, unheard } = await heardIn(recordings, agent).finally(() => stopListening().catch(() => undefined)); + const words = [msg.text.trim(), said].filter(Boolean).join("\n\n"); + let content = msg.kind === "group" ? `${msg.user.name}: ${words}` : words; + if (said) content += "\n\n(Sent as a voice message and transcribed by Godmode; the recording is attached.)"; + if (unheard.length) content += `\n\n(A voice message is attached that Godmode couldn't transcribe: ${unheard.join("; ")}.)`; if (problems.length) content += `\n\n(Some attachments couldn't be downloaded: ${problems.join("; ")}.)`; chat = chatById(chatId); @@ -423,9 +474,16 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In // not prefixed with their name β€” and the files they sent). if (from.owner) await untilAsked(conversationId); if (from.owner && openQuestionOf(conversationId)) { + // An answer nobody could hear is no answer: the question stays open. + if (unheard.length) { + await say(adapter, msg, `I couldn't understand your voice message (${unheard.join("; ")}). Please type your answer.`); + return; + } let answered: ReturnType = null; + // A spoken answer counts like a typed one ("yes", "2"): its recording isn't sent along. + const sent = files.filter((f) => !heard.has(f)); try { - answered = answerByMessage(conversationId, { content: msg.text, attachments: files.length ? files : undefined }, { + answered = answerByMessage(conversationId, { content: words, attachments: sent.length ? sent : undefined }, { actor: `messaging:${from.userId}`, via: conn.provider, }); @@ -435,6 +493,7 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In } if (answered) { patchChat(chatId, { last_message_at: now() }); + if (said) await say(adapter, msg, `I heard: β€œ${said}”`); const stopAnswered = await adapter.working(msg.target, msg.messageId).catch(() => async () => {}); void deliver(conn, adapter, msg, answered.run.id, stopAnswered, true); return; diff --git a/packages/core/src/messaging/telegram.ts b/packages/core/src/messaging/telegram.ts index b1e30c94..4f1fc491 100644 --- a/packages/core/src/messaging/telegram.ts +++ b/packages/core/src/messaging/telegram.ts @@ -85,6 +85,7 @@ interface TgMessage { voice?: TgFileRef; audio?: TgFileRef; video?: TgFileRef; + video_note?: TgFileRef; } interface TgUpdate { @@ -260,16 +261,23 @@ export class TelegramAdapter implements MessagingAdapter { private files(m: TgMessage): InboundFile[] { const out: InboundFile[] = []; - const add = (ref: TgFileRef | undefined, name: string, mime: string) => { + const add = (ref: TgFileRef | undefined, name: string, mime: string, spoken = false) => { if (!ref) return; - out.push({ name: ref.file_name || name, mime: ref.mime_type || mime, size: ref.file_size ?? null, download: () => this.download(ref.file_id) }); + out.push({ + name: ref.file_name || name, + mime: ref.mime_type || mime, + size: ref.file_size ?? null, + ...(spoken ? { spoken } : {}), + download: () => this.download(ref.file_id), + }); }; const photo = m.photo?.[m.photo.length - 1]; add(photo, `photo-${m.message_id}.jpg`, "image/jpeg"); add(m.document, `file-${m.message_id}`, "application/octet-stream"); - add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg"); + add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg", true); add(m.audio, `audio-${m.message_id}.mp3`, "audio/mpeg"); add(m.video, `video-${m.message_id}.mp4`, "video/mp4"); + add(m.video_note, `video-message-${m.message_id}.mp4`, "video/mp4", true); return out; } diff --git a/packages/core/src/messaging/types.ts b/packages/core/src/messaging/types.ts index 1713b620..7a7b4d86 100644 --- a/packages/core/src/messaging/types.ts +++ b/packages/core/src/messaging/types.ts @@ -5,6 +5,8 @@ export interface InboundFile { mime: string; /** Bytes, when the platform says. */ size: number | null; + /** Something the sender said instead of typing it (a voice message): its words become part of the message. */ + spoken?: boolean; download: () => Promise; } diff --git a/packages/core/src/voice/voice.ts b/packages/core/src/voice/voice.ts index 8d01b598..907b222a 100644 --- a/packages/core/src/voice/voice.ts +++ b/packages/core/src/voice/voice.ts @@ -3,7 +3,8 @@ * The "browser" providers run entirely in the UI (Web Speech API) and never reach these functions. * API keys are vault app secrets: `openai_api_key`, `elevenlabs_api_key`. */ -import type { Settings } from "@godmode/shared"; +import type { Agent, Settings } from "@godmode/shared"; +import { apiToolKey, apiToolsForAgent, markApiToolUsed } from "../integrations/apiTools"; import { getSettings } from "../services/settings"; import * as vault from "../vault/vault"; import { badRequest, HttpError } from "../util"; @@ -12,11 +13,15 @@ export const OPENAI_API_KEY_SECRET = "openai_api_key"; export const ELEVENLABS_API_KEY_SECRET = "elevenlabs_api_key"; export const ELEVENLABS_BASE_URL = "https://api.elevenlabs.io"; export const ELEVENLABS_MODEL = "eleven_flash_v2_5"; +export const ELEVENLABS_STT_MODEL = "scribe_v2"; +const OPENAI_STT_MODEL = "gpt-4o-mini-transcribe"; /** OpenAI's upload limit for transcriptions. */ export const MAX_AUDIO_BYTES = 25 * 1024 * 1024; export const MAX_SPEAK_CHARS = 4000; const STT_TIMEOUT_MS = 120_000; +/** Time allowed for a transcript to arrive once the provider started answering. */ +const STT_BODY_TIMEOUT_MS = 30_000; /** Time allowed until the TTS provider starts answering; the audio itself then streams without a deadline. */ const TTS_TIMEOUT_MS = 60_000; @@ -139,23 +144,131 @@ export async function transcribe(audio: Blob, opts: { language?: string | null; if (!audio || audio.size === 0) throw badRequest("The recording is empty"); if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large"); - const key = apiKey(OPENAI_API_KEY_SECRET, "OpenAI"); + return { + text: await openaiTranscript(audio, { + key: apiKey(OPENAI_API_KEY_SECRET, "OpenAI"), + baseUrl: openaiBaseUrl(voice), + model: voice.sttModel || OPENAI_STT_MODEL, + language: isoLanguage(opts.language || voice.language), + filename: opts.filename, + }), + }; +} + +async function openaiTranscript( + audio: Blob, + opts: { key: string; baseUrl: string; model: string; language?: string; filename?: string | null }, +): Promise { const form = new FormData(); form.set("file", audio, audioFilename(audio.type, opts.filename)); - form.set("model", voice.sttModel || "gpt-4o-mini-transcribe"); + form.set("model", opts.model); form.set("response_format", "json"); - const language = isoLanguage(opts.language || voice.language); - if (language) form.set("language", language); + if (opts.language) form.set("language", opts.language); const res = await providerFetch( "OpenAI", - `${openaiBaseUrl(voice)}/audio/transcriptions`, - { method: "POST", headers: { authorization: `Bearer ${key}`, accept: "application/json" }, body: form }, + `${opts.baseUrl}/audio/transcriptions`, + { method: "POST", headers: { authorization: `Bearer ${opts.key}`, accept: "application/json" }, body: form }, STT_TIMEOUT_MS, ); - const data = (await res.json().catch(() => null)) as { text?: unknown } | null; - if (!data || typeof data.text !== "string") throw new HttpError(502, "OpenAI returned an unexpected transcription response", "voice_upstream"); - return { text: data.text.trim() }; + return transcriptIn("OpenAI", res); +} + +async function transcriptIn(provider: Provider, res: Response): Promise { + let timer: ReturnType | undefined; + const late = new Promise((_, reject) => { + timer = setTimeout(() => reject(new HttpError(502, `${provider} did not finish its answer within ${STT_BODY_TIMEOUT_MS / 1000} seconds`, "voice_unreachable")), STT_BODY_TIMEOUT_MS); + }); + try { + const data = (await Promise.race([res.json().catch(() => null), late])) as { text?: unknown } | null; + if (!data || typeof data.text !== "string") throw new HttpError(502, `${provider} returned an unexpected transcription response`, "voice_upstream"); + return data.text.trim(); + } finally { + clearTimeout(timer); + } +} + +async function elevenlabsTranscript(audio: Blob, opts: { key: string; filename?: string | null }): Promise { + const form = new FormData(); + form.set("file", audio, audioFilename(audio.type, opts.filename)); + form.set("model_id", ELEVENLABS_STT_MODEL); + form.set("tag_audio_events", "false"); + form.set("timestamps_granularity", "none"); + + const res = await providerFetch( + "ElevenLabs", + `${ELEVENLABS_BASE_URL}/v1/speech-to-text`, + { method: "POST", headers: { "xi-api-key": opts.key, accept: "application/json" }, body: form }, + STT_TIMEOUT_MS, + ); + return transcriptIn("ElevenLabs", res); +} + +type RecordingAgent = Pick; + +interface Transcriber { + provider: "openai" | "elevenlabs"; + key: string; + /** The API tool the key belongs to, when it isn't one of the Voice keys. */ + toolId?: string; +} + +const TOOL_HOSTS: Record = { "api.openai.com": "openai", "api.elevenlabs.io": "elevenlabs" }; + +function toolProvider(baseUrl: string): Transcriber["provider"] | null { + try { + const u = new URL(baseUrl); + return u.protocol === "https:" ? (TOOL_HOSTS[u.host] ?? null) : null; + } catch { + return null; + } +} + +/** + * The key a recording someone sent is transcribed with: the Voice keys first, then an OpenAI or ElevenLabs API tool + * `agent` may use (the one closest to the agent). Null when there is none. + */ +function transcriberFor(agent: RecordingAgent | null): Transcriber | null { + for (const [secret, provider] of [ + [OPENAI_API_KEY_SECRET, "openai"], + [ELEVENLABS_API_KEY_SECRET, "elevenlabs"], + ] as const) { + const key = vault.getAppSecret(secret); + if (key) return { provider, key }; + } + if (!agent) return null; + for (const tool of apiToolsForAgent(agent).reverse()) { + const provider = tool.hasKey ? toolProvider(tool.baseUrl) : null; + const key = provider ? apiToolKey(tool.id) : null; + if (provider && key) return { provider, key, toolId: tool.id }; + } + return null; +} + +export const NO_TRANSCRIPTION_KEY = "No OpenAI or ElevenLabs key is set up (Settings β†’ Integrations)"; + +/** + * Text of a recording that reached Godmode from outside the app (a voice message in a messaging chat). The language + * is detected. Throws an HttpError a human can read when it can't be transcribed. + */ +export async function transcribeRecording(audio: Blob, opts: { filename?: string | null; agent?: RecordingAgent | null } = {}): Promise { + if (!audio || audio.size === 0) throw new HttpError(400, "The recording is empty", "voice_empty"); + if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large"); + const source = transcriberFor(opts.agent ?? null); + if (!source) throw new HttpError(400, NO_TRANSCRIPTION_KEY, "voice_no_key"); + const voice = getSettings().voice; + const text = + source.provider === "elevenlabs" + ? await elevenlabsTranscript(audio, { key: source.key, filename: opts.filename }) + : await openaiTranscript(audio, { + key: source.key, + // A tool's key only ever goes to the tool's own address. + baseUrl: source.toolId ? "https://api.openai.com/v1" : openaiBaseUrl(voice), + model: (!source.toolId && voice.sttModel) || OPENAI_STT_MODEL, + filename: opts.filename, + }); + if (source.toolId) markApiToolUsed(source.toolId); + return text; } export interface SpeechAudio { diff --git a/packages/core/test/messaging.test.ts b/packages/core/test/messaging.test.ts index 3ef5f548..27695f64 100644 --- a/packages/core/test/messaging.test.ts +++ b/packages/core/test/messaging.test.ts @@ -6,6 +6,7 @@ import { all, get } from "../src/db"; import { getRun, waitForRun } from "../src/runner/runner"; import { openQuestionOf, listQuestions } from "../src/services/questions"; import * as vault from "../src/vault/vault"; +import { createApiTool, deleteApiTool } from "../src/integrations/apiTools"; import { issueGrant } from "../src/server/grants"; import { createApp } from "../src/server/app"; import { splitMessage, toPlainText, toSlackMrkdwn, toTelegramHtml, fromSlackText } from "../src/messaging/format"; @@ -33,6 +34,7 @@ interface Call { method: string; url: URL; body: Record; + text: string; headers: Headers; } @@ -61,7 +63,7 @@ function installFetchMock() { body = Object.fromEntries(new URLSearchParams(text)); } } - const call = { method: req.method, url, body, headers: req.headers }; + const call = { method: req.method, url, body, text, headers: req.headers }; calls.push(call); return handler(call); }) as typeof fetch; @@ -83,7 +85,10 @@ function fakeTelegram() { onHost("api.telegram.org", async (call) => { const method = tgMethod(call); const ok = (result: unknown) => Response.json({ ok: true, result }); + if (call.url.pathname.startsWith("/file/")) return new Response(new Uint8Array([79, 103, 103, 83])); switch (method) { + case "getFile": + return ok({ file_path: "voice/file_7.oga", file_size: 4 }); case "getMe": return ok({ id: 4242, is_bot: true, first_name: "Godmode", username: "godmode_test_bot" }); case "getWebhookInfo": @@ -114,8 +119,33 @@ function tgMessage(from: { id: number; first_name: string; username?: string }, }); } +function tgVoice(from: { id: number; first_name: string; username?: string }, extra: Record = {}) { + updates.push({ + update_id: updateId++, + message: { + message_id: messageId++, + from: { is_bot: false, ...from }, + chat: { id: from.id, type: "private", first_name: from.first_name }, + voice: { file_id: "voice-file", file_size: 4, mime_type: "audio/ogg", duration: 3 }, + ...extra, + }, + }); +} + const sentTexts = () => tgCalls("sendMessage").map((c) => String(c.body.text)); +/* --------------------------- fake ElevenLabs ---------------------------- */ + +let heard = ""; +let sttDown = false; +const sttCalls = () => calls.filter((c) => c.url.host === "api.elevenlabs.io"); + +function fakeElevenLabs() { + onHost("api.elevenlabs.io", () => + sttDown ? Response.json({ detail: { message: "Invalid API key xi-tool-key-123" } }, { status: 401 }) : Response.json({ text: heard, language_code: "deu" }), + ); +} + /* -------------------------------- setup --------------------------------- */ let env: TestEnv; @@ -130,6 +160,7 @@ beforeAll(async () => { researcher = await makeAgent({ name: "Researcher", avatar: "πŸ”Ž", description: "Digs into things" }); installFetchMock(); fakeTelegram(); + fakeElevenLabs(); startMessaging(); }); @@ -421,6 +452,103 @@ describe("Telegram bot", () => { expect(sends[1]!.body.text).toBe("BROKEN answer"); }); + describe("voice messages", () => { + const dana = { id: 7004, first_name: "Dana" }; + const elevenLabs = () => + createApiTool({ name: "ElevenLabs", baseUrl: "https://api.elevenlabs.io", apiKey: "xi-tool-key-123", auth: { in: "header", name: "xi-api-key", prefix: "" } }); + const danasId = () => listUsers(connectionId).find((u) => u.name === "Dana")!.id; + const conversationId = () => listChats(connectionId).find((c) => c.title === "Dana")!.conversationId!; + const lastUserMessage = () => + get<{ content: string; attachments: string }>( + "SELECT content, attachments FROM messages WHERE conversation_id = ? AND role = 'user' ORDER BY created_at DESC, rowid DESC LIMIT 1", + conversationId(), + )!; + + test("they reach the agent as text, with the recording attached", async () => { + tgMessage(dana, "hi"); + await until(() => listUsers(connectionId).some((u) => u.name === "Dana"), 5000, "dana recorded"); + await setUserStatus(connectionId, danasId(), "approved"); + + // No key to transcribe with: the agent still gets the recording and learns why there are no words. + let before = sentTexts().length; + tgVoice(dana); + await until(() => sentTexts().slice(before).includes("Hello, nice to meet you!"), 10_000, "the answer to an untranscribed voice message"); + expect(sttCalls()).toHaveLength(0); + expect(lastUserMessage().content).toContain("couldn't transcribe: no OpenAI or ElevenLabs key is set up"); + expect(lastUserMessage().attachments).toContain(".ogg"); + + const tool = elevenLabs(); + try { + heard = "Was gibt's Neues?"; + before = sentTexts().length; + tgVoice(dana); + await until(() => sentTexts().length > before, 10_000, "the answer to a voice message"); + expect(sentTexts().at(-1)).toBe("Hello, nice to meet you!"); + expect(sttCalls()).toHaveLength(1); + expect(sttCalls()[0]!.headers.get("xi-api-key")).toBe("xi-tool-key-123"); + expect(sttCalls()[0]!.text).toContain("scribe_v2"); + const message = lastUserMessage(); + expect(message.content.startsWith("Was gibt's Neues?")).toBe(true); + expect(message.content).toContain("Sent as a voice message"); + expect(message.attachments).toContain(".ogg"); + + // A caption and the spoken words belong together; silence is named as such. + heard = ""; + before = sentTexts().length; + tgVoice(dana, { caption: "Listen to this" }); + await until(() => sentTexts().length > before, 10_000, "the answer to a silent voice message"); + expect(lastUserMessage().content).toBe("Listen to this\n\n(A voice message is attached that Godmode couldn't transcribe: nothing audible in it.)"); + + // Spoken words are never a slash command. + heard = "/model opus"; + before = sentTexts().length; + tgVoice(dana); + await until(() => sentTexts().length > before, 10_000, "the answer to a spoken slash"); + expect(lastUserMessage().content.startsWith("model opus\n\n(Sent as a voice message")).toBe(true); + + // What the provider answers stays out of the chat and the agent's prompt. + sttDown = true; + before = sentTexts().length; + tgVoice(dana); + await until(() => sentTexts().length > before, 10_000, "the answer when the provider refuses"); + expect(lastUserMessage().content).toBe("(A voice message is attached that Godmode couldn't transcribe: the speech-to-text key was rejected.)"); + expect(sentTexts().join("\n")).not.toContain("xi-tool-key-123"); + } finally { + sttDown = false; + deleteApiTool(tool.id); + } + }, 40_000); + + test("the owner can answer an agent's question by voice", async () => { + await setUserOwner(connectionId, danasId(), true); + const tool = elevenLabs(); + try { + const before = sentTexts().length; + tgMessage(dana, "ASK_APPROVAL for the reminder"); + await until(() => sentTexts().slice(before).some((t) => t.includes("needs an OK")), 15_000, "the approval request"); + const approval = openQuestionOf(conversationId())!; + + // An answer nobody could hear leaves the question open. + sttDown = true; + tgVoice(dana); + await until(() => sentTexts().some((t) => t.includes("I couldn't understand your voice message (the speech-to-text key was rejected)")), 10_000, "the request to type"); + expect(openQuestionOf(conversationId())?.id).toBe(approval.id); + + sttDown = false; + heard = "Yes."; + tgVoice(dana); + await until(() => listQuestions({ status: "approved" }).some((q) => q.id === approval.id), 10_000, "the spoken approval"); + await until(() => sentTexts().includes("I heard: β€œYes.”"), 5000, "what was heard"); + await waitForRun(approval.runId, 20_000); + } finally { + sttDown = false; + deleteApiTool(tool.id); + await setUserOwner(connectionId, danasId(), false); + await until(() => !all<{ id: string }>("SELECT id FROM runs WHERE status IN ('queued', 'running', 'paused')").length, 20_000, "the chat to settle"); + } + }, 60_000); + }); + test("turning a bot off stops polling; deleting removes people and chats", async () => { await updateConnection(connectionId, { enabled: false }); await syncMessaging(); diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts index 131060f6..7c83ea4b 100644 --- a/packages/core/test/voice.test.ts +++ b/packages/core/test/voice.test.ts @@ -8,7 +8,8 @@ import { closeDb, openDb } from "../src/db"; import { setLogLevel } from "../src/log"; import { resetSettingsCache, updateSettings } from "../src/services/settings"; import * as vault from "../src/vault/vault"; -import { audioFilename, isoLanguage, speak, transcribe } from "../src/voice/voice"; +import { audioFilename, isoLanguage, NO_TRANSCRIPTION_KEY, speak, transcribe, transcribeRecording } from "../src/voice/voice"; +import { createApiTool, deleteApiTool } from "../src/integrations/apiTools"; import { registerVoiceRoutes } from "../src/server/routes/voice"; import { HttpError } from "../src/util"; @@ -215,3 +216,92 @@ describe("routes", () => { expect([...new Uint8Array(await res.arrayBuffer())]).toEqual([9, 9]); }); }); + +describe("transcribeRecording (voice messages)", () => { + const audio = new Blob([new Uint8Array([1, 2, 3, 4])], { type: "audio/ogg" }); + const agent = { id: "agt_listener", workspaceId: null, inheritMcp: true }; + + beforeEach(() => { + vault.setAppSecret("openai_api_key", null); + vault.setAppSecret("elevenlabs_api_key", null); + updateSettings({ voice: { sttProvider: "browser", sttModel: "", language: "de-DE", openaiBaseUrl: "" } }); + }); + + test("without any key it says what is missing and calls nobody", async () => { + const err = await expectHttpError(() => transcribeRecording(audio, { filename: "voice-1.ogg", agent }), 400); + expect(err.message).toBe(NO_TRANSCRIPTION_KEY); + expect(err.code).toBe("voice_no_key"); + expect(captured).toHaveLength(0); + }); + + test("the OpenAI voice key is used first, and the language is detected", async () => { + vault.setAppSecret("openai_api_key", "sk-openai-test"); + vault.setAppSecret("elevenlabs_api_key", "xi-test-key"); + respond = () => Response.json({ text: " Was gibt's Neues? " }); + expect(await transcribeRecording(audio, { filename: "voice-1.ogg" })).toBe("Was gibt's Neues?"); + const c = captured[0]!; + expect(c.url).toBe("https://api.openai.com/v1/audio/transcriptions"); + expect(c.headers.get("authorization")).toBe("Bearer sk-openai-test"); + expect(c.form!.get("model")).toBe("gpt-4o-mini-transcribe"); + expect(c.form!.has("language")).toBe(false); + expect((c.form!.get("file") as File).name).toBe("voice-1.ogg"); + }); + + test("ElevenLabs transcribes when it is the only voice key", async () => { + vault.setAppSecret("elevenlabs_api_key", "xi-test-key"); + respond = () => Response.json({ text: "Hello there", language_code: "eng" }); + expect(await transcribeRecording(audio, { filename: "voice-2.ogg" })).toBe("Hello there"); + const c = captured[0]!; + expect(c.url).toBe("https://api.elevenlabs.io/v1/speech-to-text"); + expect(c.headers.get("xi-api-key")).toBe("xi-test-key"); + expect(c.form!.get("model_id")).toBe("scribe_v2"); + expect(c.form!.get("tag_audio_events")).toBe("false"); + expect((c.form!.get("file") as File).name).toBe("voice-2.ogg"); + }); + + test("an API tool the agent may use lends its key, only to the provider's own address", async () => { + const elsewhere = createApiTool({ name: "Proxy", baseUrl: "https://api.openai.com.example.net", apiKey: "sk-proxy-123456" }); + const insecure = createApiTool({ name: "Plain", baseUrl: "http://api.openai.com", apiKey: "sk-plain-123456" }); + try { + await expectHttpError(() => transcribeRecording(audio, { agent }), 400); + expect(captured).toHaveLength(0); + + const tool = createApiTool({ name: "ElevenLabs", baseUrl: "https://api.elevenlabs.io", apiKey: "xi-tool-key-123", auth: { in: "header", name: "xi-api-key", prefix: "" } }); + respond = () => Response.json({ text: "From the tool" }); + expect(await transcribeRecording(audio, { filename: "voice-3.ogg", agent })).toBe("From the tool"); + expect(captured[0]!.url).toBe("https://api.elevenlabs.io/v1/speech-to-text"); + expect(captured[0]!.headers.get("xi-api-key")).toBe("xi-tool-key-123"); + + // Nobody to borrow from: no agent, or an agent that doesn't get shared tools. + await expectHttpError(() => transcribeRecording(audio), 400); + await expectHttpError(() => transcribeRecording(audio, { agent: { ...agent, inheritMcp: false } }), 400); + expect(captured).toHaveLength(1); + deleteApiTool(tool.id); + + updateSettings({ voice: { sttModel: "local-whisper", openaiBaseUrl: "https://proxy.example/v1" } }); + const openai = createApiTool({ name: "OpenAI", baseUrl: "https://api.openai.com", apiKey: "sk-tool-key-123" }); + respond = () => Response.json({ text: "From OpenAI" }); + expect(await transcribeRecording(audio, { agent })).toBe("From OpenAI"); + expect(captured[1]!.url).toBe("https://api.openai.com/v1/audio/transcriptions"); + expect(captured[1]!.headers.get("authorization")).toBe("Bearer sk-tool-key-123"); + expect(captured[1]!.form!.get("model")).toBe("gpt-4o-mini-transcribe"); + deleteApiTool(openai.id); + } finally { + deleteApiTool(elsewhere.id); + deleteApiTool(insecure.id); + } + }); + + test("empty and oversized recordings are refused, provider errors stay readable", async () => { + vault.setAppSecret("elevenlabs_api_key", "xi-test-key"); + expect((await expectHttpError(() => transcribeRecording(new Blob([])), 400)).code).toBe("voice_empty"); + expect((await expectHttpError(() => transcribeRecording(new Blob([new Uint8Array(25 * 1024 * 1024 + 1)])), 413)).code).toBe("too_large"); + expect(captured).toHaveLength(0); + respond = () => new Response(JSON.stringify({ detail: { message: "Invalid API key" } }), { status: 401 }); + const rejected = await expectHttpError(() => transcribeRecording(audio), 400); + expect(rejected.message).toContain("ElevenLabs rejected the API key"); + expect(rejected.code).toBe("voice_invalid_key"); + respond = () => Response.json({ words: [] }); + expect((await expectHttpError(() => transcribeRecording(audio), 502)).message).toContain("ElevenLabs returned an unexpected transcription response"); + }); +});