From 907779287cb4239ebbba301555c7c73157f3ab70 Mon Sep 17 00:00:00 2001
From: Daniel Ehrhardt <daniel.ehrhardt@codext.de>
Date: Sun, 11 Oct 2026 16:33:44 +0200
Subject: [PATCH 1/2] Telegram voice messages are transcribed and reach the
 agent as text

---
 docs/ARCHITECTURE.md                    |   6 +-
 packages/core/src/messaging/bridge.ts   |  54 +++++++++--
 packages/core/src/messaging/telegram.ts |  14 ++-
 packages/core/src/messaging/types.ts    |   2 +
 packages/core/src/voice/voice.ts        | 117 ++++++++++++++++++++++--
 packages/core/test/messaging.test.ts    | 103 ++++++++++++++++++++-
 packages/core/test/voice.test.ts        |  85 ++++++++++++++++-
 7 files changed, 360 insertions(+), 21 deletions(-)

diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index c83967fa..16afda36 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -1319,7 +1319,11 @@ unverified). Messages of a chat are accepted in order; each starts a normal chat
 an 👀 reaction) and the answer is converted (Telegram HTML, Slack mrkdwn, Teams Markdown) and split. Chat commands:
 `/help`, `/agents`, `/agent <name>` (switch; in a Slack thread the channel follows), `/new`, `/stop`; Slack uses
 `/godmode <command>`. Claude Code's own slash commands are not available from chats. Attachments (any number;
-≤ 25 MB each, Telegram ≤ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
+≤ 25 MB each, Telegram ≤ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Telegram voice and video
+messages (`InboundFile.spoken`) are transcribed first (`transcribeRecording` in `voice/voice.ts`): their words become
+the message, also as the owner's answer to a question, and the recording stays attached. The key is the OpenAI or
+ElevenLabs voice key, else an OpenAI or ElevenLabs API tool the chat's agent may use (sent to that provider's own
+address only); without one the agent gets the recording and a note saying why there are no words. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
 connections but restore them turned off, so two machines never answer for one bot.
 
 ## Phone app
diff --git a/packages/core/src/messaging/bridge.ts b/packages/core/src/messaging/bridge.ts
index 08df8919..bf4a5205 100644
--- a/packages/core/src/messaging/bridge.ts
+++ b/packages/core/src/messaging/bridge.ts
@@ -15,6 +15,7 @@ import { notify } from "../services/notifications";
 import { getSettings } from "../services/settings";
 import { bus } from "../events/bus";
 import { HttpError, newId, now, parseJson } from "../util";
+import { transcribeRecording } from "../voice/voice";
 import {
   agentIdsOf,
   chatRow,
@@ -361,8 +362,13 @@ function conversationFor(conn: ConnectionRow, msg: InboundMessage, chat: ChatRow
   return conversation.id;
 }
 
-async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: string; mime: string; data: string }[]; problems: string[] }> {
-  const files: { name: string; mime: string; data: string }[] = [];
+type Attachment = { name: string; mime: string; data: string };
+/** A voice message: the attachment it arrived as, and its bytes. */
+type Recording = { file: Attachment; bytes: Uint8Array };
+
+async function attachmentsOf(msg: InboundMessage): Promise<{ files: Attachment[]; recordings: Recording[]; problems: string[] }> {
+  const files: Attachment[] = [];
+  const recordings: Recording[] = [];
   const problems: string[] = [];
   let total = 0;
   let over = 0;
@@ -385,13 +391,39 @@ async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: stri
         continue;
       }
       total += bytes.byteLength;
-      files.push({ name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") });
+      const file = { name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") };
+      files.push(file);
+      if (f.spoken) recordings.push({ file, bytes });
     } catch (err) {
       problems.push(`${f.name}: ${err instanceof Error ? err.message : String(err)}`);
     }
   }
   if (over) problems.push(`${over} more didn't fit, a message carries up to ${MAX_MESSAGE_ATTACHMENT_BYTES / 1024 / 1024} MB of files`);
-  return { files, problems };
+  return { files, recordings, problems };
+}
+
+/** What was said in the voice messages, which of them were understood, and why the others weren't. */
+async function heardIn(recordings: Recording[], agent: Agent): Promise<{ said: string; heard: Set<Attachment>; unheard: string[] }> {
+  const said: string[] = [];
+  const heard = new Set<Attachment>();
+  const unheard: string[] = [];
+  for (const { file, bytes } of recordings) {
+    try {
+      const transcript = await transcribeRecording(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: file.mime }), { filename: file.name, agent });
+      // Spoken words never count as a slash command.
+      const text = transcript.replace(/^[\s/]+/, "");
+      if (!text) {
+        unheard.push("nothing audible in it");
+        continue;
+      }
+      said.push(text);
+      heard.add(file);
+    } catch (err) {
+      log.warn(`could not transcribe ${file.name}`, err instanceof Error ? err.message : err);
+      unheard.push(err instanceof HttpError ? err.message : "the transcription failed");
+    }
+  }
+  return { said: said.join("\n\n"), heard, unheard: [...new Set(unheard)] };
 }
 
 async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: InboundMessage, chatId: string, from: { userId: string; owner: boolean }) {
@@ -406,12 +438,18 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
     await say(adapter, msg, `${agentLabel(agent)} is turned off right now.`);
     return;
   }
-  const { files, problems } = await attachmentsOf(msg);
+  const { files, recordings, problems } = await attachmentsOf(msg);
   if (!msg.text.trim() && !files.length) {
     await say(adapter, msg, `I couldn't open that file (${problems.join("; ")}).`);
     return;
   }
-  let content = msg.kind === "group" ? `${msg.user.name}: ${msg.text}` : msg.text;
+  // Listening takes a moment: the chat already shows that something is happening.
+  const stopListening = recordings.length ? await adapter.working(msg.target, msg.messageId).catch(() => async () => {}) : async () => {};
+  const { said, heard, unheard } = await heardIn(recordings, agent).finally(() => stopListening().catch(() => undefined));
+  const words = [msg.text.trim(), said].filter(Boolean).join("\n\n");
+  let content = msg.kind === "group" ? `${msg.user.name}: ${words}` : words;
+  if (said) content += "\n\n(Sent as a voice message and transcribed by Godmode; the recording is attached.)";
+  if (unheard.length) content += `\n\n(A voice message is attached that Godmode couldn't transcribe: ${unheard.join("; ")}.)`;
   if (problems.length) content += `\n\n(Some attachments couldn't be downloaded: ${problems.join("; ")}.)`;
 
   chat = chatById(chatId);
@@ -423,8 +461,10 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
   if (from.owner) await untilAsked(conversationId);
   if (from.owner && openQuestionOf(conversationId)) {
     let answered: ReturnType<typeof answerByMessage> = null;
+    // A spoken answer counts like a typed one ("yes", "2"): its recording isn't sent along.
+    const sent = files.filter((f) => !heard.has(f));
     try {
-      answered = answerByMessage(conversationId, { content: msg.text, attachments: files.length ? files : undefined }, {
+      answered = answerByMessage(conversationId, { content: words, attachments: sent.length ? sent : undefined }, {
         actor: `messaging:${from.userId}`,
         via: conn.provider,
       });
diff --git a/packages/core/src/messaging/telegram.ts b/packages/core/src/messaging/telegram.ts
index b1e30c94..4f1fc491 100644
--- a/packages/core/src/messaging/telegram.ts
+++ b/packages/core/src/messaging/telegram.ts
@@ -85,6 +85,7 @@ interface TgMessage {
   voice?: TgFileRef;
   audio?: TgFileRef;
   video?: TgFileRef;
+  video_note?: TgFileRef;
 }
 
 interface TgUpdate {
@@ -260,16 +261,23 @@ export class TelegramAdapter implements MessagingAdapter {
 
   private files(m: TgMessage): InboundFile[] {
     const out: InboundFile[] = [];
-    const add = (ref: TgFileRef | undefined, name: string, mime: string) => {
+    const add = (ref: TgFileRef | undefined, name: string, mime: string, spoken = false) => {
       if (!ref) return;
-      out.push({ name: ref.file_name || name, mime: ref.mime_type || mime, size: ref.file_size ?? null, download: () => this.download(ref.file_id) });
+      out.push({
+        name: ref.file_name || name,
+        mime: ref.mime_type || mime,
+        size: ref.file_size ?? null,
+        ...(spoken ? { spoken } : {}),
+        download: () => this.download(ref.file_id),
+      });
     };
     const photo = m.photo?.[m.photo.length - 1];
     add(photo, `photo-${m.message_id}.jpg`, "image/jpeg");
     add(m.document, `file-${m.message_id}`, "application/octet-stream");
-    add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg");
+    add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg", true);
     add(m.audio, `audio-${m.message_id}.mp3`, "audio/mpeg");
     add(m.video, `video-${m.message_id}.mp4`, "video/mp4");
+    add(m.video_note, `video-message-${m.message_id}.mp4`, "video/mp4", true);
     return out;
   }
 
diff --git a/packages/core/src/messaging/types.ts b/packages/core/src/messaging/types.ts
index 1713b620..7a7b4d86 100644
--- a/packages/core/src/messaging/types.ts
+++ b/packages/core/src/messaging/types.ts
@@ -5,6 +5,8 @@ export interface InboundFile {
   mime: string;
   /** Bytes, when the platform says. */
   size: number | null;
+  /** Something the sender said instead of typing it (a voice message): its words become part of the message. */
+  spoken?: boolean;
   download: () => Promise<Uint8Array>;
 }
 
diff --git a/packages/core/src/voice/voice.ts b/packages/core/src/voice/voice.ts
index 8d01b598..26e5bb91 100644
--- a/packages/core/src/voice/voice.ts
+++ b/packages/core/src/voice/voice.ts
@@ -3,7 +3,8 @@
  * The "browser" providers run entirely in the UI (Web Speech API) and never reach these functions.
  * API keys are vault app secrets: `openai_api_key`, `elevenlabs_api_key`.
  */
-import type { Settings } from "@godmode/shared";
+import type { Agent, Settings } from "@godmode/shared";
+import { apiToolKey, apiToolsForAgent, markApiToolUsed } from "../integrations/apiTools";
 import { getSettings } from "../services/settings";
 import * as vault from "../vault/vault";
 import { badRequest, HttpError } from "../util";
@@ -12,6 +13,8 @@ export const OPENAI_API_KEY_SECRET = "openai_api_key";
 export const ELEVENLABS_API_KEY_SECRET = "elevenlabs_api_key";
 export const ELEVENLABS_BASE_URL = "https://api.elevenlabs.io";
 export const ELEVENLABS_MODEL = "eleven_flash_v2_5";
+export const ELEVENLABS_STT_MODEL = "scribe_v2";
+const OPENAI_STT_MODEL = "gpt-4o-mini-transcribe";
 /** OpenAI's upload limit for transcriptions. */
 export const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
 export const MAX_SPEAK_CHARS = 4000;
@@ -139,23 +142,121 @@ export async function transcribe(audio: Blob, opts: { language?: string | null;
   if (!audio || audio.size === 0) throw badRequest("The recording is empty");
   if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large");
 
-  const key = apiKey(OPENAI_API_KEY_SECRET, "OpenAI");
+  return {
+    text: await openaiTranscript(audio, {
+      key: apiKey(OPENAI_API_KEY_SECRET, "OpenAI"),
+      baseUrl: openaiBaseUrl(voice),
+      model: voice.sttModel || OPENAI_STT_MODEL,
+      language: isoLanguage(opts.language || voice.language),
+      filename: opts.filename,
+    }),
+  };
+}
+
+async function openaiTranscript(
+  audio: Blob,
+  opts: { key: string; baseUrl: string; model: string; language?: string; filename?: string | null },
+): Promise<string> {
   const form = new FormData();
   form.set("file", audio, audioFilename(audio.type, opts.filename));
-  form.set("model", voice.sttModel || "gpt-4o-mini-transcribe");
+  form.set("model", opts.model);
   form.set("response_format", "json");
-  const language = isoLanguage(opts.language || voice.language);
-  if (language) form.set("language", language);
+  if (opts.language) form.set("language", opts.language);
 
   const res = await providerFetch(
     "OpenAI",
-    `${openaiBaseUrl(voice)}/audio/transcriptions`,
-    { method: "POST", headers: { authorization: `Bearer ${key}`, accept: "application/json" }, body: form },
+    `${opts.baseUrl}/audio/transcriptions`,
+    { method: "POST", headers: { authorization: `Bearer ${opts.key}`, accept: "application/json" }, body: form },
     STT_TIMEOUT_MS,
   );
   const data = (await res.json().catch(() => null)) as { text?: unknown } | null;
   if (!data || typeof data.text !== "string") throw new HttpError(502, "OpenAI returned an unexpected transcription response", "voice_upstream");
-  return { text: data.text.trim() };
+  return data.text.trim();
+}
+
+async function elevenlabsTranscript(audio: Blob, opts: { key: string; filename?: string | null }): Promise<string> {
+  const form = new FormData();
+  form.set("file", audio, audioFilename(audio.type, opts.filename));
+  form.set("model_id", ELEVENLABS_STT_MODEL);
+  form.set("tag_audio_events", "false");
+  form.set("timestamps_granularity", "none");
+
+  const res = await providerFetch(
+    "ElevenLabs",
+    `${ELEVENLABS_BASE_URL}/v1/speech-to-text`,
+    { method: "POST", headers: { "xi-api-key": opts.key, accept: "application/json" }, body: form },
+    STT_TIMEOUT_MS,
+  );
+  const data = (await res.json().catch(() => null)) as { text?: unknown } | null;
+  if (!data || typeof data.text !== "string") throw new HttpError(502, "ElevenLabs returned an unexpected transcription response", "voice_upstream");
+  return data.text.trim();
+}
+
+type RecordingAgent = Pick<Agent, "id" | "workspaceId" | "inheritMcp">;
+
+interface Transcriber {
+  provider: "openai" | "elevenlabs";
+  key: string;
+  /** The API tool the key belongs to, when it isn't one of the Voice keys. */
+  toolId?: string;
+}
+
+const TOOL_HOSTS: Record<string, Transcriber["provider"]> = { "api.openai.com": "openai", "api.elevenlabs.io": "elevenlabs" };
+
+function toolProvider(baseUrl: string): Transcriber["provider"] | null {
+  try {
+    const u = new URL(baseUrl);
+    return u.protocol === "https:" ? (TOOL_HOSTS[u.host] ?? null) : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The key a recording someone sent is transcribed with: the Voice keys first, then an OpenAI or ElevenLabs API tool
+ * `agent` may use (the one closest to the agent). Null when there is none.
+ */
+function transcriberFor(agent: RecordingAgent | null): Transcriber | null {
+  for (const [secret, provider] of [
+    [OPENAI_API_KEY_SECRET, "openai"],
+    [ELEVENLABS_API_KEY_SECRET, "elevenlabs"],
+  ] as const) {
+    const key = vault.getAppSecret(secret);
+    if (key) return { provider, key };
+  }
+  if (!agent) return null;
+  for (const tool of apiToolsForAgent(agent).reverse()) {
+    const provider = tool.hasKey ? toolProvider(tool.baseUrl) : null;
+    const key = provider ? apiToolKey(tool.id) : null;
+    if (provider && key) return { provider, key, toolId: tool.id };
+  }
+  return null;
+}
+
+export const NO_TRANSCRIPTION_KEY = "no OpenAI or ElevenLabs key is set up (Settings → Integrations)";
+
+/**
+ * Text of a recording that reached Godmode from outside the app (a voice message in a messaging chat). The language
+ * is detected. Throws an HttpError a human can read when it can't be transcribed.
+ */
+export async function transcribeRecording(audio: Blob, opts: { filename?: string | null; agent?: RecordingAgent | null } = {}): Promise<string> {
+  if (!audio || audio.size === 0) throw badRequest("the recording is empty");
+  if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "recordings are limited to 25 MB", "too_large");
+  const source = transcriberFor(opts.agent ?? null);
+  if (!source) throw badRequest(NO_TRANSCRIPTION_KEY, { secret: OPENAI_API_KEY_SECRET });
+  const voice = getSettings().voice;
+  const text =
+    source.provider === "elevenlabs"
+      ? await elevenlabsTranscript(audio, { key: source.key, filename: opts.filename })
+      : await openaiTranscript(audio, {
+          key: source.key,
+          // A tool's key only ever goes to the tool's own address.
+          baseUrl: source.toolId ? "https://api.openai.com/v1" : openaiBaseUrl(voice),
+          model: (!source.toolId && voice.sttModel) || OPENAI_STT_MODEL,
+          filename: opts.filename,
+        });
+  if (source.toolId) markApiToolUsed(source.toolId);
+  return text;
 }
 
 export interface SpeechAudio {
diff --git a/packages/core/test/messaging.test.ts b/packages/core/test/messaging.test.ts
index 3ef5f548..110bc209 100644
--- a/packages/core/test/messaging.test.ts
+++ b/packages/core/test/messaging.test.ts
@@ -6,6 +6,7 @@ import { all, get } from "../src/db";
 import { getRun, waitForRun } from "../src/runner/runner";
 import { openQuestionOf, listQuestions } from "../src/services/questions";
 import * as vault from "../src/vault/vault";
+import { createApiTool, deleteApiTool } from "../src/integrations/apiTools";
 import { issueGrant } from "../src/server/grants";
 import { createApp } from "../src/server/app";
 import { splitMessage, toPlainText, toSlackMrkdwn, toTelegramHtml, fromSlackText } from "../src/messaging/format";
@@ -33,6 +34,7 @@ interface Call {
   method: string;
   url: URL;
   body: Record<string, unknown>;
+  text: string;
   headers: Headers;
 }
 
@@ -61,7 +63,7 @@ function installFetchMock() {
         body = Object.fromEntries(new URLSearchParams(text));
       }
     }
-    const call = { method: req.method, url, body, headers: req.headers };
+    const call = { method: req.method, url, body, text, headers: req.headers };
     calls.push(call);
     return handler(call);
   }) as typeof fetch;
@@ -83,7 +85,10 @@ function fakeTelegram() {
   onHost("api.telegram.org", async (call) => {
     const method = tgMethod(call);
     const ok = (result: unknown) => Response.json({ ok: true, result });
+    if (call.url.pathname.startsWith("/file/")) return new Response(new Uint8Array([79, 103, 103, 83]));
     switch (method) {
+      case "getFile":
+        return ok({ file_path: "voice/file_7.oga", file_size: 4 });
       case "getMe":
         return ok({ id: 4242, is_bot: true, first_name: "Godmode", username: "godmode_test_bot" });
       case "getWebhookInfo":
@@ -114,8 +119,30 @@ function tgMessage(from: { id: number; first_name: string; username?: string },
   });
 }
 
+function tgVoice(from: { id: number; first_name: string; username?: string }, extra: Record<string, unknown> = {}) {
+  updates.push({
+    update_id: updateId++,
+    message: {
+      message_id: messageId++,
+      from: { is_bot: false, ...from },
+      chat: { id: from.id, type: "private", first_name: from.first_name },
+      voice: { file_id: "voice-file", file_size: 4, mime_type: "audio/ogg", duration: 3 },
+      ...extra,
+    },
+  });
+}
+
 const sentTexts = () => tgCalls("sendMessage").map((c) => String(c.body.text));
 
+/* --------------------------- fake ElevenLabs ---------------------------- */
+
+let heard = "";
+const sttCalls = () => calls.filter((c) => c.url.host === "api.elevenlabs.io");
+
+function fakeElevenLabs() {
+  onHost("api.elevenlabs.io", () => Response.json({ text: heard, language_code: "deu" }));
+}
+
 /* -------------------------------- setup --------------------------------- */
 
 let env: TestEnv;
@@ -130,6 +157,7 @@ beforeAll(async () => {
   researcher = await makeAgent({ name: "Researcher", avatar: "🔎", description: "Digs into things" });
   installFetchMock();
   fakeTelegram();
+  fakeElevenLabs();
   startMessaging();
 });
 
@@ -421,6 +449,79 @@ describe("Telegram bot", () => {
     expect(sends[1]!.body.text).toBe("BROKEN answer");
   });
 
+  describe("voice messages", () => {
+    const dana = { id: 7004, first_name: "Dana" };
+    const elevenLabs = () =>
+      createApiTool({ name: "ElevenLabs", baseUrl: "https://api.elevenlabs.io", apiKey: "xi-tool-key-123", auth: { in: "header", name: "xi-api-key", prefix: "" } });
+    const danasId = () => listUsers(connectionId).find((u) => u.name === "Dana")!.id;
+    const conversationId = () => listChats(connectionId).find((c) => c.title === "Dana")!.conversationId!;
+    const lastUserMessage = () =>
+      get<{ content: string; attachments: string }>(
+        "SELECT content, attachments FROM messages WHERE conversation_id = ? AND role = 'user' ORDER BY created_at DESC, rowid DESC LIMIT 1",
+        conversationId(),
+      )!;
+
+    test("they reach the agent as text, with the recording attached", async () => {
+      tgMessage(dana, "hi");
+      await until(() => listUsers(connectionId).some((u) => u.name === "Dana"), 5000, "dana recorded");
+      await setUserStatus(connectionId, danasId(), "approved");
+
+      // No key to transcribe with: the agent still gets the recording and learns why there are no words.
+      let before = sentTexts().length;
+      tgVoice(dana);
+      await until(() => sentTexts().slice(before).includes("Hello, nice to meet you!"), 10_000, "the answer to an untranscribed voice message");
+      expect(sttCalls()).toHaveLength(0);
+      expect(lastUserMessage().content).toContain("couldn't transcribe: no OpenAI or ElevenLabs key is set up");
+      expect(lastUserMessage().attachments).toContain(".ogg");
+
+      const tool = elevenLabs();
+      try {
+        heard = "Was gibt's Neues?";
+        before = sentTexts().length;
+        const typing = tgCalls("sendChatAction").length;
+        tgVoice(dana);
+        await until(() => sentTexts().length > before, 10_000, "the answer to a voice message");
+        expect(sentTexts().at(-1)).toBe("Hello, nice to meet you!");
+        expect(sttCalls()).toHaveLength(1);
+        expect(sttCalls()[0]!.headers.get("xi-api-key")).toBe("xi-tool-key-123");
+        expect(sttCalls()[0]!.text).toContain("scribe_v2");
+        expect(tgCalls("sendChatAction").length).toBeGreaterThan(typing);
+        const message = lastUserMessage();
+        expect(message.content.startsWith("Was gibt's Neues?")).toBe(true);
+        expect(message.content).toContain("Sent as a voice message");
+        expect(message.attachments).toContain(".ogg");
+
+        // A caption and the spoken words belong together; silence is named as such.
+        heard = "";
+        before = sentTexts().length;
+        tgVoice(dana, { caption: "Listen to this" });
+        await until(() => sentTexts().length > before, 10_000, "the answer to a silent voice message");
+        expect(lastUserMessage().content).toBe("Listen to this\n\n(A voice message is attached that Godmode couldn't transcribe: nothing audible in it.)");
+      } finally {
+        deleteApiTool(tool.id);
+      }
+    }, 40_000);
+
+    test("the owner can answer an agent's question by voice", async () => {
+      await setUserOwner(connectionId, danasId(), true);
+      const tool = elevenLabs();
+      try {
+        const before = sentTexts().length;
+        tgMessage(dana, "ASK_APPROVAL for the reminder");
+        await until(() => sentTexts().slice(before).some((t) => t.includes("needs an OK")), 15_000, "the approval request");
+        const approval = openQuestionOf(conversationId())!;
+        heard = "Yes.";
+        tgVoice(dana);
+        await until(() => listQuestions({ status: "approved" }).some((q) => q.id === approval.id), 10_000, "the spoken approval");
+        await waitForRun(approval.runId, 20_000);
+      } finally {
+        deleteApiTool(tool.id);
+        await setUserOwner(connectionId, danasId(), false);
+        await until(() => !all<{ id: string }>("SELECT id FROM runs WHERE status IN ('queued', 'running', 'paused')").length, 20_000, "the chat to settle");
+      }
+    }, 60_000);
+  });
+
   test("turning a bot off stops polling; deleting removes people and chats", async () => {
     await updateConnection(connectionId, { enabled: false });
     await syncMessaging();
diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts
index 131060f6..90d1f495 100644
--- a/packages/core/test/voice.test.ts
+++ b/packages/core/test/voice.test.ts
@@ -8,7 +8,8 @@ import { closeDb, openDb } from "../src/db";
 import { setLogLevel } from "../src/log";
 import { resetSettingsCache, updateSettings } from "../src/services/settings";
 import * as vault from "../src/vault/vault";
-import { audioFilename, isoLanguage, speak, transcribe } from "../src/voice/voice";
+import { audioFilename, isoLanguage, NO_TRANSCRIPTION_KEY, speak, transcribe, transcribeRecording } from "../src/voice/voice";
+import { createApiTool, deleteApiTool } from "../src/integrations/apiTools";
 import { registerVoiceRoutes } from "../src/server/routes/voice";
 import { HttpError } from "../src/util";
 
@@ -215,3 +216,85 @@ describe("routes", () => {
     expect([...new Uint8Array(await res.arrayBuffer())]).toEqual([9, 9]);
   });
 });
+
+describe("transcribeRecording (voice messages)", () => {
+  const audio = new Blob([new Uint8Array([1, 2, 3, 4])], { type: "audio/ogg" });
+  const agent = { id: "agt_listener", workspaceId: null, inheritMcp: true };
+
+  beforeEach(() => {
+    vault.setAppSecret("openai_api_key", null);
+    vault.setAppSecret("elevenlabs_api_key", null);
+    updateSettings({ voice: { sttProvider: "browser", sttModel: "", language: "de-DE", openaiBaseUrl: "" } });
+  });
+
+  test("without any key it says what is missing and calls nobody", async () => {
+    const err = await expectHttpError(() => transcribeRecording(audio, { filename: "voice-1.ogg", agent }), 400);
+    expect(err.message).toBe(NO_TRANSCRIPTION_KEY);
+    expect(captured).toHaveLength(0);
+  });
+
+  test("the OpenAI voice key is used first, and the language is detected", async () => {
+    vault.setAppSecret("openai_api_key", "sk-openai-test");
+    vault.setAppSecret("elevenlabs_api_key", "xi-test-key");
+    respond = () => Response.json({ text: " Was gibt's Neues? " });
+    expect(await transcribeRecording(audio, { filename: "voice-1.ogg" })).toBe("Was gibt's Neues?");
+    const c = captured[0]!;
+    expect(c.url).toBe("https://api.openai.com/v1/audio/transcriptions");
+    expect(c.headers.get("authorization")).toBe("Bearer sk-openai-test");
+    expect(c.form!.get("model")).toBe("gpt-4o-mini-transcribe");
+    expect(c.form!.has("language")).toBe(false);
+    expect((c.form!.get("file") as File).name).toBe("voice-1.ogg");
+  });
+
+  test("ElevenLabs transcribes when it is the only voice key", async () => {
+    vault.setAppSecret("elevenlabs_api_key", "xi-test-key");
+    respond = () => Response.json({ text: "Hello there", language_code: "eng" });
+    expect(await transcribeRecording(audio, { filename: "voice-2.ogg" })).toBe("Hello there");
+    const c = captured[0]!;
+    expect(c.url).toBe("https://api.elevenlabs.io/v1/speech-to-text");
+    expect(c.headers.get("xi-api-key")).toBe("xi-test-key");
+    expect(c.form!.get("model_id")).toBe("scribe_v2");
+    expect(c.form!.get("tag_audio_events")).toBe("false");
+    expect((c.form!.get("file") as File).name).toBe("voice-2.ogg");
+  });
+
+  test("an API tool the agent may use lends its key, only to the provider's own address", async () => {
+    const elsewhere = createApiTool({ name: "Proxy", baseUrl: "https://api.openai.com.example.net", apiKey: "sk-proxy-123456" });
+    const insecure = createApiTool({ name: "Plain", baseUrl: "http://api.openai.com", apiKey: "sk-plain-123456" });
+    try {
+      await expectHttpError(() => transcribeRecording(audio, { agent }), 400);
+      expect(captured).toHaveLength(0);
+
+      const tool = createApiTool({ name: "ElevenLabs", baseUrl: "https://api.elevenlabs.io", apiKey: "xi-tool-key-123", auth: { in: "header", name: "xi-api-key", prefix: "" } });
+      respond = () => Response.json({ text: "From the tool" });
+      expect(await transcribeRecording(audio, { filename: "voice-3.ogg", agent })).toBe("From the tool");
+      expect(captured[0]!.url).toBe("https://api.elevenlabs.io/v1/speech-to-text");
+      expect(captured[0]!.headers.get("xi-api-key")).toBe("xi-tool-key-123");
+
+      // Nobody to borrow from: no agent, or an agent that doesn't get shared tools.
+      await expectHttpError(() => transcribeRecording(audio), 400);
+      await expectHttpError(() => transcribeRecording(audio, { agent: { ...agent, inheritMcp: false } }), 400);
+      expect(captured).toHaveLength(1);
+      deleteApiTool(tool.id);
+
+      updateSettings({ voice: { sttModel: "local-whisper", openaiBaseUrl: "https://proxy.example/v1" } });
+      const openai = createApiTool({ name: "OpenAI", baseUrl: "https://api.openai.com", apiKey: "sk-tool-key-123" });
+      respond = () => Response.json({ text: "From OpenAI" });
+      expect(await transcribeRecording(audio, { agent })).toBe("From OpenAI");
+      expect(captured[1]!.url).toBe("https://api.openai.com/v1/audio/transcriptions");
+      expect(captured[1]!.headers.get("authorization")).toBe("Bearer sk-tool-key-123");
+      expect(captured[1]!.form!.get("model")).toBe("gpt-4o-mini-transcribe");
+      deleteApiTool(openai.id);
+    } finally {
+      deleteApiTool(elsewhere.id);
+      deleteApiTool(insecure.id);
+    }
+  });
+
+  test("empty and oversized recordings are refused, provider errors stay readable", async () => {
+    vault.setAppSecret("elevenlabs_api_key", "xi-test-key");
+    await expectHttpError(() => transcribeRecording(new Blob([])), 400);
+    respond = () => new Response(JSON.stringify({ detail: { message: "Invalid API key" } }), { status: 401 });
+    expect((await expectHttpError(() => transcribeRecording(audio), 400)).message).toContain("ElevenLabs rejected the API key");
+  });
+});

From 171c12ca536b93bdd084a16c6c0a8ce8ee57426f Mon Sep 17 00:00:00 2001
From: Daniel Ehrhardt <daniel.ehrhardt@codext.de>
Date: Sun, 11 Oct 2026 16:44:09 +0200
Subject: [PATCH 2/2] Voice messages: neutral reasons, unheard answers keep the
 question open, transcript deadline

---
 docs/ARCHITECTURE.md                  |  4 +++-
 packages/core/src/messaging/bridge.ts | 21 ++++++++++++++++-
 packages/core/src/voice/voice.ts      | 32 ++++++++++++++++++--------
 packages/core/test/messaging.test.ts  | 33 ++++++++++++++++++++++++---
 packages/core/test/voice.test.ts      | 11 +++++++--
 5 files changed, 84 insertions(+), 17 deletions(-)

diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 16afda36..3432ec0a 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -1323,7 +1323,9 @@ an 👀 reaction) and the answer is converted (Telegram HTML, Slack mrkdwn, Team
 messages (`InboundFile.spoken`) are transcribed first (`transcribeRecording` in `voice/voice.ts`): their words become
 the message, also as the owner's answer to a question, and the recording stays attached. The key is the OpenAI or
 ElevenLabs voice key, else an OpenAI or ElevenLabs API tool the chat's agent may use (sent to that provider's own
-address only); without one the agent gets the recording and a note saying why there are no words. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
+address only); without one, or when the provider fails, the agent gets the recording and a note with a neutral reason
+(the provider's own words stay in the log). A spoken answer to a question is echoed ("I heard: …"); one that couldn't
+be understood leaves the question open and asks for a typed answer. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
 connections but restore them turned off, so two machines never answer for one bot.
 
 ## Phone app
diff --git a/packages/core/src/messaging/bridge.ts b/packages/core/src/messaging/bridge.ts
index bf4a5205..8b2f6e0e 100644
--- a/packages/core/src/messaging/bridge.ts
+++ b/packages/core/src/messaging/bridge.ts
@@ -402,6 +402,19 @@ async function attachmentsOf(msg: InboundMessage): Promise<{ files: Attachment[]
   return { files, recordings, problems };
 }
 
+/** Why a voice message has no words, for the agent and the chat: what the provider answered stays in the log. */
+function unheardReason(err: unknown): string {
+  const code = err instanceof HttpError ? err.code : undefined;
+  if (code === "voice_no_key") return "no OpenAI or ElevenLabs key is set up in Godmode (Settings → Integrations)";
+  if (code === "voice_invalid_key") return "the speech-to-text key was rejected";
+  if (code === "voice_rate_limited") return "the speech-to-text limit is reached";
+  if (code === "voice_unreachable") return "the speech-to-text service didn't answer";
+  if (code === "voice_empty") return "it is empty";
+  if (code === "too_large") return "it is longer than speech-to-text takes (25 MB)";
+  if (code === "vault_locked") return "Godmode's vault is locked";
+  return "the transcription failed";
+}
+
 /** What was said in the voice messages, which of them were understood, and why the others weren't. */
 async function heardIn(recordings: Recording[], agent: Agent): Promise<{ said: string; heard: Set<Attachment>; unheard: string[] }> {
   const said: string[] = [];
@@ -420,7 +433,7 @@ async function heardIn(recordings: Recording[], agent: Agent): Promise<{ said: s
       heard.add(file);
     } catch (err) {
       log.warn(`could not transcribe ${file.name}`, err instanceof Error ? err.message : err);
-      unheard.push(err instanceof HttpError ? err.message : "the transcription failed");
+      unheard.push(unheardReason(err));
     }
   }
   return { said: said.join("\n\n"), heard, unheard: [...new Set(unheard)] };
@@ -460,6 +473,11 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
   // not prefixed with their name — and the files they sent).
   if (from.owner) await untilAsked(conversationId);
   if (from.owner && openQuestionOf(conversationId)) {
+    // An answer nobody could hear is no answer: the question stays open.
+    if (unheard.length) {
+      await say(adapter, msg, `I couldn't understand your voice message (${unheard.join("; ")}). Please type your answer.`);
+      return;
+    }
     let answered: ReturnType<typeof answerByMessage> = null;
     // A spoken answer counts like a typed one ("yes", "2"): its recording isn't sent along.
     const sent = files.filter((f) => !heard.has(f));
@@ -474,6 +492,7 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
     }
     if (answered) {
       patchChat(chatId, { last_message_at: now() });
+      if (said) await say(adapter, msg, `I heard: “${said}”`);
       const stopAnswered = await adapter.working(msg.target, msg.messageId).catch(() => async () => {});
       void deliver(conn, adapter, msg, answered.run.id, stopAnswered, true);
       return;
diff --git a/packages/core/src/voice/voice.ts b/packages/core/src/voice/voice.ts
index 26e5bb91..907b222a 100644
--- a/packages/core/src/voice/voice.ts
+++ b/packages/core/src/voice/voice.ts
@@ -20,6 +20,8 @@ export const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
 export const MAX_SPEAK_CHARS = 4000;
 
 const STT_TIMEOUT_MS = 120_000;
+/** Time allowed for a transcript to arrive once the provider started answering. */
+const STT_BODY_TIMEOUT_MS = 30_000;
 /** Time allowed until the TTS provider starts answering; the audio itself then streams without a deadline. */
 const TTS_TIMEOUT_MS = 60_000;
 
@@ -169,9 +171,21 @@ async function openaiTranscript(
     { method: "POST", headers: { authorization: `Bearer ${opts.key}`, accept: "application/json" }, body: form },
     STT_TIMEOUT_MS,
   );
-  const data = (await res.json().catch(() => null)) as { text?: unknown } | null;
-  if (!data || typeof data.text !== "string") throw new HttpError(502, "OpenAI returned an unexpected transcription response", "voice_upstream");
-  return data.text.trim();
+  return transcriptIn("OpenAI", res);
+}
+
+async function transcriptIn(provider: Provider, res: Response): Promise<string> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const late = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new HttpError(502, `${provider} did not finish its answer within ${STT_BODY_TIMEOUT_MS / 1000} seconds`, "voice_unreachable")), STT_BODY_TIMEOUT_MS);
+  });
+  try {
+    const data = (await Promise.race([res.json().catch(() => null), late])) as { text?: unknown } | null;
+    if (!data || typeof data.text !== "string") throw new HttpError(502, `${provider} returned an unexpected transcription response`, "voice_upstream");
+    return data.text.trim();
+  } finally {
+    clearTimeout(timer);
+  }
 }
 
 async function elevenlabsTranscript(audio: Blob, opts: { key: string; filename?: string | null }): Promise<string> {
@@ -187,9 +201,7 @@ async function elevenlabsTranscript(audio: Blob, opts: { key: string; filename?:
     { method: "POST", headers: { "xi-api-key": opts.key, accept: "application/json" }, body: form },
     STT_TIMEOUT_MS,
   );
-  const data = (await res.json().catch(() => null)) as { text?: unknown } | null;
-  if (!data || typeof data.text !== "string") throw new HttpError(502, "ElevenLabs returned an unexpected transcription response", "voice_upstream");
-  return data.text.trim();
+  return transcriptIn("ElevenLabs", res);
 }
 
 type RecordingAgent = Pick<Agent, "id" | "workspaceId" | "inheritMcp">;
@@ -233,17 +245,17 @@ function transcriberFor(agent: RecordingAgent | null): Transcriber | null {
   return null;
 }
 
-export const NO_TRANSCRIPTION_KEY = "no OpenAI or ElevenLabs key is set up (Settings → Integrations)";
+export const NO_TRANSCRIPTION_KEY = "No OpenAI or ElevenLabs key is set up (Settings → Integrations)";
 
 /**
  * Text of a recording that reached Godmode from outside the app (a voice message in a messaging chat). The language
  * is detected. Throws an HttpError a human can read when it can't be transcribed.
  */
 export async function transcribeRecording(audio: Blob, opts: { filename?: string | null; agent?: RecordingAgent | null } = {}): Promise<string> {
-  if (!audio || audio.size === 0) throw badRequest("the recording is empty");
-  if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "recordings are limited to 25 MB", "too_large");
+  if (!audio || audio.size === 0) throw new HttpError(400, "The recording is empty", "voice_empty");
+  if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large");
   const source = transcriberFor(opts.agent ?? null);
-  if (!source) throw badRequest(NO_TRANSCRIPTION_KEY, { secret: OPENAI_API_KEY_SECRET });
+  if (!source) throw new HttpError(400, NO_TRANSCRIPTION_KEY, "voice_no_key");
   const voice = getSettings().voice;
   const text =
     source.provider === "elevenlabs"
diff --git a/packages/core/test/messaging.test.ts b/packages/core/test/messaging.test.ts
index 110bc209..27695f64 100644
--- a/packages/core/test/messaging.test.ts
+++ b/packages/core/test/messaging.test.ts
@@ -137,10 +137,13 @@ const sentTexts = () => tgCalls("sendMessage").map((c) => String(c.body.text));
 /* --------------------------- fake ElevenLabs ---------------------------- */
 
 let heard = "";
+let sttDown = false;
 const sttCalls = () => calls.filter((c) => c.url.host === "api.elevenlabs.io");
 
 function fakeElevenLabs() {
-  onHost("api.elevenlabs.io", () => Response.json({ text: heard, language_code: "deu" }));
+  onHost("api.elevenlabs.io", () =>
+    sttDown ? Response.json({ detail: { message: "Invalid API key xi-tool-key-123" } }, { status: 401 }) : Response.json({ text: heard, language_code: "deu" }),
+  );
 }
 
 /* -------------------------------- setup --------------------------------- */
@@ -478,14 +481,12 @@ describe("Telegram bot", () => {
       try {
         heard = "Was gibt's Neues?";
         before = sentTexts().length;
-        const typing = tgCalls("sendChatAction").length;
         tgVoice(dana);
         await until(() => sentTexts().length > before, 10_000, "the answer to a voice message");
         expect(sentTexts().at(-1)).toBe("Hello, nice to meet you!");
         expect(sttCalls()).toHaveLength(1);
         expect(sttCalls()[0]!.headers.get("xi-api-key")).toBe("xi-tool-key-123");
         expect(sttCalls()[0]!.text).toContain("scribe_v2");
-        expect(tgCalls("sendChatAction").length).toBeGreaterThan(typing);
         const message = lastUserMessage();
         expect(message.content.startsWith("Was gibt's Neues?")).toBe(true);
         expect(message.content).toContain("Sent as a voice message");
@@ -497,7 +498,23 @@ describe("Telegram bot", () => {
         tgVoice(dana, { caption: "Listen to this" });
         await until(() => sentTexts().length > before, 10_000, "the answer to a silent voice message");
         expect(lastUserMessage().content).toBe("Listen to this\n\n(A voice message is attached that Godmode couldn't transcribe: nothing audible in it.)");
+
+        // Spoken words are never a slash command.
+        heard = "/model opus";
+        before = sentTexts().length;
+        tgVoice(dana);
+        await until(() => sentTexts().length > before, 10_000, "the answer to a spoken slash");
+        expect(lastUserMessage().content.startsWith("model opus\n\n(Sent as a voice message")).toBe(true);
+
+        // What the provider answers stays out of the chat and the agent's prompt.
+        sttDown = true;
+        before = sentTexts().length;
+        tgVoice(dana);
+        await until(() => sentTexts().length > before, 10_000, "the answer when the provider refuses");
+        expect(lastUserMessage().content).toBe("(A voice message is attached that Godmode couldn't transcribe: the speech-to-text key was rejected.)");
+        expect(sentTexts().join("\n")).not.toContain("xi-tool-key-123");
       } finally {
+        sttDown = false;
         deleteApiTool(tool.id);
       }
     }, 40_000);
@@ -510,11 +527,21 @@ describe("Telegram bot", () => {
         tgMessage(dana, "ASK_APPROVAL for the reminder");
         await until(() => sentTexts().slice(before).some((t) => t.includes("needs an OK")), 15_000, "the approval request");
         const approval = openQuestionOf(conversationId())!;
+
+        // An answer nobody could hear leaves the question open.
+        sttDown = true;
+        tgVoice(dana);
+        await until(() => sentTexts().some((t) => t.includes("I couldn't understand your voice message (the speech-to-text key was rejected)")), 10_000, "the request to type");
+        expect(openQuestionOf(conversationId())?.id).toBe(approval.id);
+
+        sttDown = false;
         heard = "Yes.";
         tgVoice(dana);
         await until(() => listQuestions({ status: "approved" }).some((q) => q.id === approval.id), 10_000, "the spoken approval");
+        await until(() => sentTexts().includes("I heard: “Yes.”"), 5000, "what was heard");
         await waitForRun(approval.runId, 20_000);
       } finally {
+        sttDown = false;
         deleteApiTool(tool.id);
         await setUserOwner(connectionId, danasId(), false);
         await until(() => !all<{ id: string }>("SELECT id FROM runs WHERE status IN ('queued', 'running', 'paused')").length, 20_000, "the chat to settle");
diff --git a/packages/core/test/voice.test.ts b/packages/core/test/voice.test.ts
index 90d1f495..7c83ea4b 100644
--- a/packages/core/test/voice.test.ts
+++ b/packages/core/test/voice.test.ts
@@ -230,6 +230,7 @@ describe("transcribeRecording (voice messages)", () => {
   test("without any key it says what is missing and calls nobody", async () => {
     const err = await expectHttpError(() => transcribeRecording(audio, { filename: "voice-1.ogg", agent }), 400);
     expect(err.message).toBe(NO_TRANSCRIPTION_KEY);
+    expect(err.code).toBe("voice_no_key");
     expect(captured).toHaveLength(0);
   });
 
@@ -293,8 +294,14 @@ describe("transcribeRecording (voice messages)", () => {
 
   test("empty and oversized recordings are refused, provider errors stay readable", async () => {
     vault.setAppSecret("elevenlabs_api_key", "xi-test-key");
-    await expectHttpError(() => transcribeRecording(new Blob([])), 400);
+    expect((await expectHttpError(() => transcribeRecording(new Blob([])), 400)).code).toBe("voice_empty");
+    expect((await expectHttpError(() => transcribeRecording(new Blob([new Uint8Array(25 * 1024 * 1024 + 1)])), 413)).code).toBe("too_large");
+    expect(captured).toHaveLength(0);
     respond = () => new Response(JSON.stringify({ detail: { message: "Invalid API key" } }), { status: 401 });
-    expect((await expectHttpError(() => transcribeRecording(audio), 400)).message).toContain("ElevenLabs rejected the API key");
+    const rejected = await expectHttpError(() => transcribeRecording(audio), 400);
+    expect(rejected.message).toContain("ElevenLabs rejected the API key");
+    expect(rejected.code).toBe("voice_invalid_key");
+    respond = () => Response.json({ words: [] });
+    expect((await expectHttpError(() => transcribeRecording(audio), 502)).message).toContain("ElevenLabs returned an unexpected transcription response");
   });
 });