Telegram voice messages are transcribed and reach the agent as text

docs/ARCHITECTURE.md

@@ -1355,7 +1355,13 @@ unverified). Messages of a chat are accepted in order; each starts a normal chat
 an 👀 reaction) and the answer is converted (Telegram HTML, Slack mrkdwn, Teams Markdown) and split. Chat commands:
 `/help`, `/agents`, `/agent <name>` (switch; in a Slack thread the channel follows), `/new`, `/stop`; Slack uses
 `/godmode <command>`. Claude Code's own slash commands are not available from chats. Attachments (any number;
-≤ 25 MB each, Telegram ≤ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
+≤ 25 MB each, Telegram ≤ 20 MB, 250 MB per message) are downloaded into the agent's uploads. Telegram voice and video
+messages (`InboundFile.spoken`) are transcribed first (`transcribeRecording` in `voice/voice.ts`): their words become
+the message, also as the owner's answer to a question, and the recording stays attached. The key is the OpenAI or
+ElevenLabs voice key, else an OpenAI or ElevenLabs API tool the chat's agent may use (sent to that provider's own
+address only); without one, or when the provider fails, the agent gets the recording and a note with a neutral reason
+(the provider's own words stay in the log). A spoken answer to a question is echoed ("I heard: …"); one that couldn't
+be understood leaves the question open and asks for a typed answer. Limits: 20 messages per chat and 120 per bot per minute. Backups carry
 connections but restore them turned off, so two machines never answer for one bot.
 
 ## Phone app

packages/core/src/messaging/bridge.ts

@@ -16,6 +16,7 @@ import { notify } from "../services/notifications";
 import { getSettings } from "../services/settings";
 import { bus } from "../events/bus";
 import { HttpError, newId, now, parseJson } from "../util";
+import { transcribeRecording } from "../voice/voice";
 import {
   agentIdsOf,
   chatRow,
@@ -362,8 +363,13 @@ function conversationFor(conn: ConnectionRow, msg: InboundMessage, chat: ChatRow
   return conversation.id;
 }
 
-async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: string; mime: string; data: string }[]; problems: string[] }> {
-  const files: { name: string; mime: string; data: string }[] = [];
+type Attachment = { name: string; mime: string; data: string };
+/** A voice message: the attachment it arrived as, and its bytes. */
+type Recording = { file: Attachment; bytes: Uint8Array };
+
+async function attachmentsOf(msg: InboundMessage): Promise<{ files: Attachment[]; recordings: Recording[]; problems: string[] }> {
+  const files: Attachment[] = [];
+  const recordings: Recording[] = [];
   const problems: string[] = [];
   let total = 0;
   let over = 0;
@@ -386,13 +392,52 @@ async function attachmentsOf(msg: InboundMessage): Promise<{ files: { name: stri
         continue;
       }
       total += bytes.byteLength;
-      files.push({ name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") });
+      const file = { name: f.name, mime: f.mime, data: Buffer.from(bytes).toString("base64") };
+      files.push(file);
+      if (f.spoken) recordings.push({ file, bytes });
     } catch (err) {
       problems.push(`${f.name}: ${err instanceof Error ? err.message : String(err)}`);
     }
   }
   if (over) problems.push(`${over} more didn't fit, a message carries up to ${MAX_MESSAGE_ATTACHMENT_BYTES / 1024 / 1024} MB of files`);
-  return { files, problems };
+  return { files, recordings, problems };
+}
+
+/** Why a voice message has no words, for the agent and the chat: what the provider answered stays in the log. */
+function unheardReason(err: unknown): string {
+  const code = err instanceof HttpError ? err.code : undefined;
+  if (code === "voice_no_key") return "no OpenAI or ElevenLabs key is set up in Godmode (Settings → Integrations)";
+  if (code === "voice_invalid_key") return "the speech-to-text key was rejected";
+  if (code === "voice_rate_limited") return "the speech-to-text limit is reached";
+  if (code === "voice_unreachable") return "the speech-to-text service didn't answer";
+  if (code === "voice_empty") return "it is empty";
+  if (code === "too_large") return "it is longer than speech-to-text takes (25 MB)";
+  if (code === "vault_locked") return "Godmode's vault is locked";
+  return "the transcription failed";
+}
+
+/** What was said in the voice messages, which of them were understood, and why the others weren't. */
+async function heardIn(recordings: Recording[], agent: Agent): Promise<{ said: string; heard: Set<Attachment>; unheard: string[] }> {
+  const said: string[] = [];
+  const heard = new Set<Attachment>();
+  const unheard: string[] = [];
+  for (const { file, bytes } of recordings) {
+    try {
+      const transcript = await transcribeRecording(new Blob([bytes as Uint8Array<ArrayBuffer>], { type: file.mime }), { filename: file.name, agent });
+      // Spoken words never count as a slash command.
+      const text = transcript.replace(/^[\s/]+/, "");
+      if (!text) {
+        unheard.push("nothing audible in it");
+        continue;
+      }
+      said.push(text);
+      heard.add(file);
+    } catch (err) {
+      log.warn(`could not transcribe ${file.name}`, err instanceof Error ? err.message : err);
+      unheard.push(unheardReason(err));
+    }
+  }
+  return { said: said.join("\n\n"), heard, unheard: [...new Set(unheard)] };
 }
 
 async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: InboundMessage, chatId: string, from: { userId: string; owner: boolean }) {
@@ -407,12 +452,18 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
     await say(adapter, msg, `${agentLabel(agent)} is turned off right now.`);
     return;
   }
-  const { files, problems } = await attachmentsOf(msg);
+  const { files, recordings, problems } = await attachmentsOf(msg);
   if (!msg.text.trim() && !files.length) {
     await say(adapter, msg, `I couldn't open that file (${problems.join("; ")}).`);
     return;
   }
-  let content = msg.kind === "group" ? `${msg.user.name}: ${msg.text}` : msg.text;
+  // Listening takes a moment: the chat already shows that something is happening.
+  const stopListening = recordings.length ? await adapter.working(msg.target, msg.messageId).catch(() => async () => {}) : async () => {};
+  const { said, heard, unheard } = await heardIn(recordings, agent).finally(() => stopListening().catch(() => undefined));
+  const words = [msg.text.trim(), said].filter(Boolean).join("\n\n");
+  let content = msg.kind === "group" ? `${msg.user.name}: ${words}` : words;
+  if (said) content += "\n\n(Sent as a voice message and transcribed by Godmode; the recording is attached.)";
+  if (unheard.length) content += `\n\n(A voice message is attached that Godmode couldn't transcribe: ${unheard.join("; ")}.)`;
   if (problems.length) content += `\n\n(Some attachments couldn't be downloaded: ${problems.join("; ")}.)`;
 
   chat = chatById(chatId);
@@ -423,9 +474,16 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
   // not prefixed with their name — and the files they sent).
   if (from.owner) await untilAsked(conversationId);
   if (from.owner && openQuestionOf(conversationId)) {
+    // An answer nobody could hear is no answer: the question stays open.
+    if (unheard.length) {
+      await say(adapter, msg, `I couldn't understand your voice message (${unheard.join("; ")}). Please type your answer.`);
+      return;
+    }
     let answered: ReturnType<typeof answerByMessage> = null;
+    // A spoken answer counts like a typed one ("yes", "2"): its recording isn't sent along.
+    const sent = files.filter((f) => !heard.has(f));
     try {
-      answered = answerByMessage(conversationId, { content: msg.text, attachments: files.length ? files : undefined }, {
+      answered = answerByMessage(conversationId, { content: words, attachments: sent.length ? sent : undefined }, {
         actor: `messaging:${from.userId}`,
         via: conn.provider,
       });
@@ -435,6 +493,7 @@ async function startTurn(conn: ConnectionRow, adapter: MessagingAdapter, msg: In
     }
     if (answered) {
       patchChat(chatId, { last_message_at: now() });
+      if (said) await say(adapter, msg, `I heard: “${said}”`);
       const stopAnswered = await adapter.working(msg.target, msg.messageId).catch(() => async () => {});
       void deliver(conn, adapter, msg, answered.run.id, stopAnswered, true);
       return;

packages/core/src/messaging/telegram.ts

@@ -85,6 +85,7 @@ interface TgMessage {
   voice?: TgFileRef;
   audio?: TgFileRef;
   video?: TgFileRef;
+  video_note?: TgFileRef;
 }
 
 interface TgUpdate {
@@ -260,16 +261,23 @@ export class TelegramAdapter implements MessagingAdapter {
 
   private files(m: TgMessage): InboundFile[] {
     const out: InboundFile[] = [];
-    const add = (ref: TgFileRef | undefined, name: string, mime: string) => {
+    const add = (ref: TgFileRef | undefined, name: string, mime: string, spoken = false) => {
       if (!ref) return;
-      out.push({ name: ref.file_name || name, mime: ref.mime_type || mime, size: ref.file_size ?? null, download: () => this.download(ref.file_id) });
+      out.push({
+        name: ref.file_name || name,
+        mime: ref.mime_type || mime,
+        size: ref.file_size ?? null,
+        ...(spoken ? { spoken } : {}),
+        download: () => this.download(ref.file_id),
+      });
     };
     const photo = m.photo?.[m.photo.length - 1];
     add(photo, `photo-${m.message_id}.jpg`, "image/jpeg");
     add(m.document, `file-${m.message_id}`, "application/octet-stream");
-    add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg");
+    add(m.voice, `voice-${m.message_id}.ogg`, "audio/ogg", true);
     add(m.audio, `audio-${m.message_id}.mp3`, "audio/mpeg");
     add(m.video, `video-${m.message_id}.mp4`, "video/mp4");
+    add(m.video_note, `video-message-${m.message_id}.mp4`, "video/mp4", true);
     return out;
   }
 

packages/core/src/messaging/types.ts

@@ -5,6 +5,8 @@ export interface InboundFile {
   mime: string;
   /** Bytes, when the platform says. */
   size: number | null;
+  /** Something the sender said instead of typing it (a voice message): its words become part of the message. */
+  spoken?: boolean;
   download: () => Promise<Uint8Array>;
 }
 

packages/core/src/voice/voice.ts

@@ -3,7 +3,8 @@
  * The "browser" providers run entirely in the UI (Web Speech API) and never reach these functions.
  * API keys are vault app secrets: `openai_api_key`, `elevenlabs_api_key`.
  */
-import type { Settings } from "@godmode/shared";
+import type { Agent, Settings } from "@godmode/shared";
+import { apiToolKey, apiToolsForAgent, markApiToolUsed } from "../integrations/apiTools";
 import { getSettings } from "../services/settings";
 import * as vault from "../vault/vault";
 import { badRequest, HttpError } from "../util";
@@ -12,11 +13,15 @@ export const OPENAI_API_KEY_SECRET = "openai_api_key";
 export const ELEVENLABS_API_KEY_SECRET = "elevenlabs_api_key";
 export const ELEVENLABS_BASE_URL = "https://api.elevenlabs.io";
 export const ELEVENLABS_MODEL = "eleven_flash_v2_5";
+export const ELEVENLABS_STT_MODEL = "scribe_v2";
+const OPENAI_STT_MODEL = "gpt-4o-mini-transcribe";
 /** OpenAI's upload limit for transcriptions. */
 export const MAX_AUDIO_BYTES = 25 * 1024 * 1024;
 export const MAX_SPEAK_CHARS = 4000;
 
 const STT_TIMEOUT_MS = 120_000;
+/** Time allowed for a transcript to arrive once the provider started answering. */
+const STT_BODY_TIMEOUT_MS = 30_000;
 /** Time allowed until the TTS provider starts answering; the audio itself then streams without a deadline. */
 const TTS_TIMEOUT_MS = 60_000;
 
@@ -139,23 +144,131 @@ export async function transcribe(audio: Blob, opts: { language?: string | null;
   if (!audio || audio.size === 0) throw badRequest("The recording is empty");
   if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large");
 
-  const key = apiKey(OPENAI_API_KEY_SECRET, "OpenAI");
+  return {
+    text: await openaiTranscript(audio, {
+      key: apiKey(OPENAI_API_KEY_SECRET, "OpenAI"),
+      baseUrl: openaiBaseUrl(voice),
+      model: voice.sttModel || OPENAI_STT_MODEL,
+      language: isoLanguage(opts.language || voice.language),
+      filename: opts.filename,
+    }),
+  };
+}
+
+async function openaiTranscript(
+  audio: Blob,
+  opts: { key: string; baseUrl: string; model: string; language?: string; filename?: string | null },
+): Promise<string> {
   const form = new FormData();
   form.set("file", audio, audioFilename(audio.type, opts.filename));
-  form.set("model", voice.sttModel || "gpt-4o-mini-transcribe");
+  form.set("model", opts.model);
   form.set("response_format", "json");
-  const language = isoLanguage(opts.language || voice.language);
-  if (language) form.set("language", language);
+  if (opts.language) form.set("language", opts.language);
 
   const res = await providerFetch(
     "OpenAI",
-    `${openaiBaseUrl(voice)}/audio/transcriptions`,
-    { method: "POST", headers: { authorization: `Bearer ${key}`, accept: "application/json" }, body: form },
+    `${opts.baseUrl}/audio/transcriptions`,
+    { method: "POST", headers: { authorization: `Bearer ${opts.key}`, accept: "application/json" }, body: form },
     STT_TIMEOUT_MS,
   );
-  const data = (await res.json().catch(() => null)) as { text?: unknown } | null;
-  if (!data || typeof data.text !== "string") throw new HttpError(502, "OpenAI returned an unexpected transcription response", "voice_upstream");
-  return { text: data.text.trim() };
+  return transcriptIn("OpenAI", res);
+}
+
+async function transcriptIn(provider: Provider, res: Response): Promise<string> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const late = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new HttpError(502, `${provider} did not finish its answer within ${STT_BODY_TIMEOUT_MS / 1000} seconds`, "voice_unreachable")), STT_BODY_TIMEOUT_MS);
+  });
+  try {
+    const data = (await Promise.race([res.json().catch(() => null), late])) as { text?: unknown } | null;
+    if (!data || typeof data.text !== "string") throw new HttpError(502, `${provider} returned an unexpected transcription response`, "voice_upstream");
+    return data.text.trim();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function elevenlabsTranscript(audio: Blob, opts: { key: string; filename?: string | null }): Promise<string> {
+  const form = new FormData();
+  form.set("file", audio, audioFilename(audio.type, opts.filename));
+  form.set("model_id", ELEVENLABS_STT_MODEL);
+  form.set("tag_audio_events", "false");
+  form.set("timestamps_granularity", "none");
+
+  const res = await providerFetch(
+    "ElevenLabs",
+    `${ELEVENLABS_BASE_URL}/v1/speech-to-text`,
+    { method: "POST", headers: { "xi-api-key": opts.key, accept: "application/json" }, body: form },
+    STT_TIMEOUT_MS,
+  );
+  return transcriptIn("ElevenLabs", res);
+}
+
+type RecordingAgent = Pick<Agent, "id" | "workspaceId" | "inheritMcp">;
+
+interface Transcriber {
+  provider: "openai" | "elevenlabs";
+  key: string;
+  /** The API tool the key belongs to, when it isn't one of the Voice keys. */
+  toolId?: string;
+}
+
+const TOOL_HOSTS: Record<string, Transcriber["provider"]> = { "api.openai.com": "openai", "api.elevenlabs.io": "elevenlabs" };
+
+function toolProvider(baseUrl: string): Transcriber["provider"] | null {
+  try {
+    const u = new URL(baseUrl);
+    return u.protocol === "https:" ? (TOOL_HOSTS[u.host] ?? null) : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The key a recording someone sent is transcribed with: the Voice keys first, then an OpenAI or ElevenLabs API tool
+ * `agent` may use (the one closest to the agent). Null when there is none.
+ */
+function transcriberFor(agent: RecordingAgent | null): Transcriber | null {
+  for (const [secret, provider] of [
+    [OPENAI_API_KEY_SECRET, "openai"],
+    [ELEVENLABS_API_KEY_SECRET, "elevenlabs"],
+  ] as const) {
+    const key = vault.getAppSecret(secret);
+    if (key) return { provider, key };
+  }
+  if (!agent) return null;
+  for (const tool of apiToolsForAgent(agent).reverse()) {
+    const provider = tool.hasKey ? toolProvider(tool.baseUrl) : null;
+    const key = provider ? apiToolKey(tool.id) : null;
+    if (provider && key) return { provider, key, toolId: tool.id };
+  }
+  return null;
+}
+
+export const NO_TRANSCRIPTION_KEY = "No OpenAI or ElevenLabs key is set up (Settings → Integrations)";
+
+/**
+ * Text of a recording that reached Godmode from outside the app (a voice message in a messaging chat). The language
+ * is detected. Throws an HttpError a human can read when it can't be transcribed.
+ */
+export async function transcribeRecording(audio: Blob, opts: { filename?: string | null; agent?: RecordingAgent | null } = {}): Promise<string> {
+  if (!audio || audio.size === 0) throw new HttpError(400, "The recording is empty", "voice_empty");
+  if (audio.size > MAX_AUDIO_BYTES) throw new HttpError(413, "Recordings are limited to 25 MB", "too_large");
+  const source = transcriberFor(opts.agent ?? null);
+  if (!source) throw new HttpError(400, NO_TRANSCRIPTION_KEY, "voice_no_key");
+  const voice = getSettings().voice;
+  const text =
+    source.provider === "elevenlabs"
+      ? await elevenlabsTranscript(audio, { key: source.key, filename: opts.filename })
+      : await openaiTranscript(audio, {
+          key: source.key,
+          // A tool's key only ever goes to the tool's own address.
+          baseUrl: source.toolId ? "https://api.openai.com/v1" : openaiBaseUrl(voice),
+          model: (!source.toolId && voice.sttModel) || OPENAI_STT_MODEL,
+          filename: opts.filename,
+        });
+  if (source.toolId) markApiToolUsed(source.toolId);
+  return text;
 }
 
 export interface SpeechAudio {