Promote to production: Voice to Text Fast engine + live dictation

.prettierignore

@@ -4,3 +4,4 @@ node_modules/
 package-lock.json
 pnpm-lock.yaml
 yarn.lock
+src/vendor/

e2e/tools/voice-to-text.spec.ts

@@ -0,0 +1,102 @@
+import { test, expect, type Page } from '@playwright/test';
+
+// `?e2e` swaps the Whistle engine for a dev-only scripted stub (see
+// src/tools/media/whistle.stub.ts), so no model is downloaded in CI.
+
+/** A tiny synthetic WAV: 1 s of a 440 Hz tone, 16 kHz mono 16-bit. */
+function toneWav(seconds = 1, rate = 16000): Buffer {
+  const n = seconds * rate;
+  const buf = Buffer.alloc(44 + n * 2);
+  buf.write('RIFF', 0); buf.writeUInt32LE(36 + n * 2, 4); buf.write('WAVE', 8);
+  buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22);
+  buf.writeUInt32LE(rate, 24); buf.writeUInt32LE(rate * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34);
+  buf.write('data', 36); buf.writeUInt32LE(n * 2, 40);
+  for (let i = 0; i < n; i++) buf.writeInt16LE(Math.round(Math.sin((2 * Math.PI * 440 * i) / rate) * 8000), 44 + i * 2);
+  return buf;
+}
+
+/** Replace the microphone with an oscillator so live dictation runs headless. */
+async function fakeMicrophone(page: Page) {
+  await page.addInitScript(() => {
+    const md = navigator.mediaDevices;
+    if (!md) return;
+    md.getUserMedia = async () => {
+      const ctx = new AudioContext();
+      const osc = ctx.createOscillator();
+      const dest = ctx.createMediaStreamDestination();
+      osc.connect(dest);
+      osc.start();
+      return dest.stream;
+    };
+  });
+}
+
+/** The island lazy-hydrates; wait until React owns the language picker. */
+async function waitForIsland(page: Page) {
+  await page.waitForLoadState('networkidle').catch(() => {});
+  await page.waitForFunction(
+    () => {
+      const el = document.querySelector('select:has(option[value="indonesian"])');
+      return !!el && Object.keys(el).some(k => k.startsWith('__reactProps'));
+    },
+    undefined,
+    { timeout: 45_000 },
+  );
+}
+
+const languagePicker = (page: Page) => page.locator('select:has(option[value="indonesian"])');
+
+test('defaults to the Fast model and switches to Whisper for Indonesian', async ({ page }) => {
+  test.setTimeout(90_000);
+  await page.goto('/tools/voice-to-text?e2e');
+  await waitForIsland(page);
+
+  await expect(page.getByRole('button', { name: 'Fast · 7 languages' })).toHaveAttribute('aria-pressed', 'true');
+  await expect(page.getByRole('checkbox', { name: 'Live dictation' })).toBeEnabled();
+
+  await languagePicker(page).selectOption('indonesian');
+  await expect(page.getByRole('button', { name: 'Multilingual', exact: true })).toHaveAttribute('aria-pressed', 'true');
+  await expect(page.getByText('Fast doesn’t support Indonesian (Bahasa) — switched to Multilingual.')).toBeVisible();
+  await expect(page.getByRole('checkbox', { name: 'Live dictation' })).toBeDisabled();
+
+  // Back to Fast: the unsupported language resets to auto-detect.
+  await page.getByRole('button', { name: 'Fast · 7 languages' }).click();
+  await expect(languagePicker(page)).toHaveValue('');
+});
+
+test('Indonesian pages default to Whisper with Bahasa selected', async ({ page }) => {
+  test.setTimeout(90_000);
+  await page.goto('/id/tools/voice-to-text?e2e');
+  await waitForIsland(page);
+  await expect(languagePicker(page)).toHaveValue('indonesian');
+  await expect(page.getByRole('button', { name: 'Multibahasa', exact: true })).toHaveAttribute('aria-pressed', 'true');
+});
+
+test('transcribes a file with the Fast engine', async ({ page }) => {
+  test.setTimeout(90_000);
+  await page.goto('/tools/voice-to-text?e2e');
+  await waitForIsland(page);
+  await page.locator('input[type="file"]').setInputFiles({ name: 'tone.wav', mimeType: 'audio/wav', buffer: toneWav() });
+  await page.getByRole('button', { name: 'Transcribe', exact: true }).click();
+  await expect(page.locator('textarea')).toHaveValue('Stub transcript from the fast engine.', { timeout: 30_000 });
+
+  await page.getByRole('button', { name: 'Subtitles' }).click();
+  await expect(page.locator('pre', { hasText: '-->' })).toContainText('00:00:00,000 -->');
+});
+
+test('live dictation streams words and keeps the transcript', async ({ page }) => {
+  test.setTimeout(90_000);
+  await fakeMicrophone(page);
+  await page.goto('/tools/voice-to-text?e2e');
+  await waitForIsland(page);
+
+  await page.getByRole('checkbox', { name: 'Live dictation' }).check();
+  await page.getByRole('button', { name: 'Live dictation' }).click();
+  await expect(page.getByTestId('live-transcript')).toContainText('live1', { timeout: 30_000 });
+  await expect(page.getByTestId('live-transcript')).toContainText('listening…');
+
+  await page.getByRole('button', { name: /Stop live/ }).click();
+  await expect(page.locator('textarea')).toHaveValue(/^live1.* done\.$/, { timeout: 30_000 });
+  // The live recording is kept for replay.
+  await expect(page.locator('audio')).toBeVisible();
+});

eslint.config.mjs

@@ -19,6 +19,7 @@ export default [
       'coverage/**',
       '**/*.config.{js,mjs,cjs,ts}',
       'src/env.d.ts', // Astro-generated (triple-slash reference)
+      'src/vendor/**', // third-party code vendored unmodified (see scripts/vendor-needle.mjs)
     ],
   },
   js.configs.recommended,

scripts/vendor-needle.mjs

@@ -0,0 +1,32 @@
+// Vendors the Needle engine's Emscripten JS wrapper (used by the Voice to Text
+// "Fast" Whistle model) as an ES module, so no remote JS is executed at runtime.
+// The matching needle.wasm and whistle.cact are fetched at runtime through the
+// /hf proxy, pinned to the same commits and SHA-256 verified (see whistle.engine.ts).
+//
+//   node scripts/vendor-needle.mjs
+import { writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+
+const COMMIT = '2ae11323dc000f5e70c49f7403efa6af12ba9e67';
+const SHA256 = '964681b2a5ec3c4db2f06e45a5b60c8981a7d4ab4cac16ef6bf5b1988657a5f1';
+const URL_ = `https://huggingface.co/Cactus-Compute/needle3/resolve/${COMMIT}/wasm/needle.js`;
+
+const res = await fetch(URL_);
+if (!res.ok) throw new Error(`Download failed: ${res.status}`);
+const src = Buffer.from(await res.arrayBuffer());
+const hash = createHash('sha256').update(src).digest('hex');
+if (hash !== SHA256) throw new Error(`needle.js hash mismatch: ${hash}`);
+
+const header = `/* eslint-disable */
+// @ts-nocheck
+// Needle engine JS wrapper (Emscripten), vendored unmodified except for the
+// trailing ES export. Source: huggingface.co/Cactus-Compute/needle3 @ ${COMMIT}
+// (wasm/needle.js, sha256 ${SHA256}). Licensed under Apache-2.0 — see ./LICENSE.
+// Regenerate with: node scripts/vendor-needle.mjs
+`;
+writeFileSync('src/vendor/needle/needle.mjs', header + src.toString('utf8') + '\nexport default createNeedle;\n');
+
+const lic = await fetch(`https://huggingface.co/Cactus-Compute/needle3/resolve/${COMMIT}/LICENSE`);
+if (!lic.ok) throw new Error(`LICENSE download failed: ${lic.status}`);
+writeFileSync('src/vendor/needle/LICENSE', Buffer.from(await lic.arrayBuffer()));
+console.log('Vendored needle.mjs + LICENSE');

src/hooks/useLiveDictation.ts

@@ -0,0 +1,183 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { liveStream } from '@/tools/media/stt.client';
+import { downsampleTo16k, type WhistleWord } from '@/tools/media/whistle.lib';
+
+export type LiveErrorReason = 'denied' | 'unsupported' | 'engine';
+export interface LiveError { reason: LiveErrorReason; message?: string }
+
+/** Send roughly this much audio per streaming step. */
+const STEP_SEC = 1;
+/** Never send more than the engine's per-call limit, even after a slow pass. */
+const MAX_PENDING_SEC = 29;
+
+// Captures mic frames on the audio thread and posts copies to the main thread.
+const WORKLET = `
+class GwtCapture extends AudioWorkletProcessor {
+  process(inputs) {
+    const ch = inputs[0] && inputs[0][0];
+    if (ch) this.port.postMessage(ch.slice(0));
+    return true;
+  }
+}
+registerProcessor('gwt-capture', GwtCapture);
+`;
+
+/**
+ * Live dictation: streams the microphone to the worker's Whistle engine about
+ * once a second; committed words accumulate, the unconfirmed tail is `pending`.
+ * A MediaRecorder keeps the audio so it can be replayed or re-transcribed.
+ */
+export function useLiveDictation() {
+  const [live, setLive] = useState(false);
+  const [starting, setStarting] = useState(false);
+  const [progress, setProgress] = useState<number | null>(null);
+  const [seconds, setSeconds] = useState(0);
+  const [words, setWords] = useState<WhistleWord[]>([]);
+  const [pending, setPending] = useState('');
+  const [blob, setBlob] = useState<Blob | null>(null);
+  const [error, setError] = useState<LiveError | null>(null);
+
+  const streamRef = useRef<MediaStream | null>(null);
+  const ctxRef = useRef<AudioContext | null>(null);
+  const recorderRef = useRef<MediaRecorder | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
+  const bufferRef = useRef<Float32Array[]>([]);
+  const bufferedRef = useRef(0);
+  const inflightRef = useRef<Promise<void> | null>(null);
+  const langRef = useRef<string | undefined>(undefined);
+  const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const activeRef = useRef(false);
+  /** Every committed word so far (state updates are async; stop() returns this). */
+  const allWordsRef = useRef<WhistleWord[]>([]);
+
+  const teardown = useCallback(() => {
+    activeRef.current = false;
+    if (timerRef.current) clearInterval(timerRef.current);
+    timerRef.current = null;
+    streamRef.current?.getTracks().forEach(t => t.stop());
+    streamRef.current = null;
+    void ctxRef.current?.close().catch(() => {});
+    ctxRef.current = null;
+  }, []);
+
+  useEffect(() => teardown, [teardown]);
+
+  /** Drain the buffered native-rate audio into one 16 kHz chunk. */
+  const takeBuffered = (rate: number): Float32Array => {
+    const total = bufferedRef.current;
+    const merged = new Float32Array(total);
+    let o = 0;
+    for (const b of bufferRef.current) { merged.set(b, o); o += b.length; }
+    bufferRef.current = [];
+    bufferedRef.current = 0;
+    return downsampleTo16k(merged, rate);
+  };
+
+  const send = useCallback((pcm: Float32Array) => {
+    const run = liveStream.push(pcm, langRef.current).then(u => {
+      if (u.words.length) {
+        allWordsRef.current = [...allWordsRef.current, ...u.words];
+        setWords(allWordsRef.current);
+      }
+      setPending(u.pending);
+    });
+    inflightRef.current = run.catch(err => {
+      setError({ reason: 'engine', message: err instanceof Error ? err.message : undefined });
+    }).finally(() => { inflightRef.current = null; });
+  }, []);
+
+  const start = useCallback(async (language: string | undefined) => {
+    setError(null);
+    setBlob(null);
+    setWords([]);
+    allWordsRef.current = [];
+    setPending('');
+    langRef.current = language;
+    if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia || typeof AudioWorkletNode === 'undefined') {
+      setError({ reason: 'unsupported' });
+      return;
+    }
+    setStarting(true);
+    try {
+      await liveStream.start(r => setProgress(r));
+      setProgress(null);
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
+      streamRef.current = stream;
+
+      const ctx = new AudioContext();
+      ctxRef.current = ctx;
+      const url = URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' }));
+      try { await ctx.audioWorklet.addModule(url); } finally { URL.revokeObjectURL(url); }
+      const source = ctx.createMediaStreamSource(stream);
+      const node = new AudioWorkletNode(ctx, 'gwt-capture');
+      const mute = ctx.createGain();
+      mute.gain.value = 0;
+      source.connect(node).connect(mute).connect(ctx.destination);
+
+      const rate = ctx.sampleRate;
+      node.port.onmessage = (e: MessageEvent<Float32Array>) => {
+        if (!activeRef.current) return;
+        bufferRef.current.push(e.data);
+        bufferedRef.current += e.data.length;
+        const ready = bufferedRef.current >= STEP_SEC * rate;
+        // Back-pressure: while a pass is running keep buffering (up to the engine limit).
+        if (ready && !inflightRef.current) send(takeBuffered(rate));
+        else if (bufferedRef.current >= MAX_PENDING_SEC * rate) {
+          bufferRef.current.shift();
+          bufferedRef.current = bufferRef.current.reduce((s, b) => s + b.length, 0);
+        }
+      };
+
+      chunksRef.current = [];
+      if (typeof MediaRecorder !== 'undefined') {
+        const rec = new MediaRecorder(stream);
+        recorderRef.current = rec;
+        rec.ondataavailable = ev => { if (ev.data.size > 0) chunksRef.current.push(ev.data); };
+        rec.onstop = () => setBlob(new Blob(chunksRef.current, { type: chunksRef.current[0]?.type || 'audio/webm' }));
+        rec.start();
+      }
+
+      activeRef.current = true;
+      setSeconds(0);
+      timerRef.current = setInterval(() => setSeconds(s => s + 1), 1000);
+      setLive(true);
+    } catch (err) {
+      teardown();
+      const name = err instanceof Error ? err.name : '';
+      setError(name === 'NotAllowedError' || name === 'SecurityError'
+        ? { reason: 'denied' }
+        : { reason: 'engine', message: err instanceof Error ? err.message : undefined });
+    } finally {
+      setStarting(false);
+      setProgress(null);
+    }
+  }, [send, teardown]);
+
+  /** Stop listening, flush the last audio and finalise the transcript. */
+  const stop = useCallback(async (): Promise<WhistleWord[]> => {
+    const rate = ctxRef.current?.sampleRate ?? 16000;
+    const rec = recorderRef.current;
+    if (rec && rec.state !== 'inactive') rec.stop();
+    recorderRef.current = null;
+    teardown();
+    setLive(false);
+
+    try {
+      if (inflightRef.current) await inflightRef.current;
+      if (bufferedRef.current > 0) {
+        send(takeBuffered(rate));
+        if (inflightRef.current) await inflightRef.current;
+      }
+      const fin = await liveStream.stop();
+      allWordsRef.current = [...allWordsRef.current, ...fin.words];
+      setWords(allWordsRef.current);
+      setPending('');
+    } catch (err) {
+      setError({ reason: 'engine', message: err instanceof Error ? err.message : undefined });
+    }
+    return allWordsRef.current;
+  }, [send, teardown]);
+
+  return { live, starting, progress, seconds, words, pending, blob, error, start, stop };
+}