diff --git a/.prettierignore b/.prettierignore index 2a968f8..ea102cb 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,3 +4,4 @@ node_modules/ package-lock.json pnpm-lock.yaml yarn.lock +src/vendor/ diff --git a/e2e/tools/voice-to-text.spec.ts b/e2e/tools/voice-to-text.spec.ts new file mode 100644 index 0000000..6689cda --- /dev/null +++ b/e2e/tools/voice-to-text.spec.ts @@ -0,0 +1,102 @@ +import { test, expect, type Page } from '@playwright/test'; + +// `?e2e` swaps the Whistle engine for a dev-only scripted stub (see +// src/tools/media/whistle.stub.ts), so no model is downloaded in CI. + +/** A tiny synthetic WAV: 1 s of a 440 Hz tone, 16 kHz mono 16-bit. */ +function toneWav(seconds = 1, rate = 16000): Buffer { + const n = seconds * rate; + const buf = Buffer.alloc(44 + n * 2); + buf.write('RIFF', 0); buf.writeUInt32LE(36 + n * 2, 4); buf.write('WAVE', 8); + buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22); + buf.writeUInt32LE(rate, 24); buf.writeUInt32LE(rate * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34); + buf.write('data', 36); buf.writeUInt32LE(n * 2, 40); + for (let i = 0; i < n; i++) buf.writeInt16LE(Math.round(Math.sin((2 * Math.PI * 440 * i) / rate) * 8000), 44 + i * 2); + return buf; +} + +/** Replace the microphone with an oscillator so live dictation runs headless. */ +async function fakeMicrophone(page: Page) { + await page.addInitScript(() => { + const md = navigator.mediaDevices; + if (!md) return; + md.getUserMedia = async () => { + const ctx = new AudioContext(); + const osc = ctx.createOscillator(); + const dest = ctx.createMediaStreamDestination(); + osc.connect(dest); + osc.start(); + return dest.stream; + }; + }); +} + +/** The island lazy-hydrates; wait until React owns the language picker. */ +async function waitForIsland(page: Page) { + await page.waitForLoadState('networkidle').catch(() => {}); + await page.waitForFunction( + () => { + const el = document.querySelector('select:has(option[value="indonesian"])'); + return !!el && Object.keys(el).some(k => k.startsWith('__reactProps')); + }, + undefined, + { timeout: 45_000 }, + ); +} + +const languagePicker = (page: Page) => page.locator('select:has(option[value="indonesian"])'); + +test('defaults to the Fast model and switches to Whisper for Indonesian', async ({ page }) => { + test.setTimeout(90_000); + await page.goto('/tools/voice-to-text?e2e'); + await waitForIsland(page); + + await expect(page.getByRole('button', { name: 'Fast · 7 languages' })).toHaveAttribute('aria-pressed', 'true'); + await expect(page.getByRole('checkbox', { name: 'Live dictation' })).toBeEnabled(); + + await languagePicker(page).selectOption('indonesian'); + await expect(page.getByRole('button', { name: 'Multilingual', exact: true })).toHaveAttribute('aria-pressed', 'true'); + await expect(page.getByText('Fast doesn’t support Indonesian (Bahasa) — switched to Multilingual.')).toBeVisible(); + await expect(page.getByRole('checkbox', { name: 'Live dictation' })).toBeDisabled(); + + // Back to Fast: the unsupported language resets to auto-detect. + await page.getByRole('button', { name: 'Fast · 7 languages' }).click(); + await expect(languagePicker(page)).toHaveValue(''); +}); + +test('Indonesian pages default to Whisper with Bahasa selected', async ({ page }) => { + test.setTimeout(90_000); + await page.goto('/id/tools/voice-to-text?e2e'); + await waitForIsland(page); + await expect(languagePicker(page)).toHaveValue('indonesian'); + await expect(page.getByRole('button', { name: 'Multibahasa', exact: true })).toHaveAttribute('aria-pressed', 'true'); +}); + +test('transcribes a file with the Fast engine', async ({ page }) => { + test.setTimeout(90_000); + await page.goto('/tools/voice-to-text?e2e'); + await waitForIsland(page); + await page.locator('input[type="file"]').setInputFiles({ name: 'tone.wav', mimeType: 'audio/wav', buffer: toneWav() }); + await page.getByRole('button', { name: 'Transcribe', exact: true }).click(); + await expect(page.locator('textarea')).toHaveValue('Stub transcript from the fast engine.', { timeout: 30_000 }); + + await page.getByRole('button', { name: 'Subtitles' }).click(); + await expect(page.locator('pre', { hasText: '-->' })).toContainText('00:00:00,000 -->'); +}); + +test('live dictation streams words and keeps the transcript', async ({ page }) => { + test.setTimeout(90_000); + await fakeMicrophone(page); + await page.goto('/tools/voice-to-text?e2e'); + await waitForIsland(page); + + await page.getByRole('checkbox', { name: 'Live dictation' }).check(); + await page.getByRole('button', { name: 'Live dictation' }).click(); + await expect(page.getByTestId('live-transcript')).toContainText('live1', { timeout: 30_000 }); + await expect(page.getByTestId('live-transcript')).toContainText('listening…'); + + await page.getByRole('button', { name: /Stop live/ }).click(); + await expect(page.locator('textarea')).toHaveValue(/^live1.* done\.$/, { timeout: 30_000 }); + // The live recording is kept for replay. + await expect(page.locator('audio')).toBeVisible(); +}); diff --git a/eslint.config.mjs b/eslint.config.mjs index aaf213a..0bf8e4f 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -19,6 +19,7 @@ export default [ 'coverage/**', '**/*.config.{js,mjs,cjs,ts}', 'src/env.d.ts', // Astro-generated (triple-slash reference) + 'src/vendor/**', // third-party code vendored unmodified (see scripts/vendor-needle.mjs) ], }, js.configs.recommended, diff --git a/scripts/vendor-needle.mjs b/scripts/vendor-needle.mjs new file mode 100644 index 0000000..1b05732 --- /dev/null +++ b/scripts/vendor-needle.mjs @@ -0,0 +1,32 @@ +// Vendors the Needle engine's Emscripten JS wrapper (used by the Voice to Text +// "Fast" Whistle model) as an ES module, so no remote JS is executed at runtime. +// The matching needle.wasm and whistle.cact are fetched at runtime through the +// /hf proxy, pinned to the same commits and SHA-256 verified (see whistle.engine.ts). +// +// node scripts/vendor-needle.mjs +import { writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; + +const COMMIT = '2ae11323dc000f5e70c49f7403efa6af12ba9e67'; +const SHA256 = '964681b2a5ec3c4db2f06e45a5b60c8981a7d4ab4cac16ef6bf5b1988657a5f1'; +const URL_ = `https://huggingface.co/Cactus-Compute/needle3/resolve/${COMMIT}/wasm/needle.js`; + +const res = await fetch(URL_); +if (!res.ok) throw new Error(`Download failed: ${res.status}`); +const src = Buffer.from(await res.arrayBuffer()); +const hash = createHash('sha256').update(src).digest('hex'); +if (hash !== SHA256) throw new Error(`needle.js hash mismatch: ${hash}`); + +const header = `/* eslint-disable */ +// @ts-nocheck +// Needle engine JS wrapper (Emscripten), vendored unmodified except for the +// trailing ES export. Source: huggingface.co/Cactus-Compute/needle3 @ ${COMMIT} +// (wasm/needle.js, sha256 ${SHA256}). Licensed under Apache-2.0 — see ./LICENSE. +// Regenerate with: node scripts/vendor-needle.mjs +`; +writeFileSync('src/vendor/needle/needle.mjs', header + src.toString('utf8') + '\nexport default createNeedle;\n'); + +const lic = await fetch(`https://huggingface.co/Cactus-Compute/needle3/resolve/${COMMIT}/LICENSE`); +if (!lic.ok) throw new Error(`LICENSE download failed: ${lic.status}`); +writeFileSync('src/vendor/needle/LICENSE', Buffer.from(await lic.arrayBuffer())); +console.log('Vendored needle.mjs + LICENSE'); diff --git a/src/hooks/useLiveDictation.ts b/src/hooks/useLiveDictation.ts new file mode 100644 index 0000000..edb4a46 --- /dev/null +++ b/src/hooks/useLiveDictation.ts @@ -0,0 +1,183 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { liveStream } from '@/tools/media/stt.client'; +import { downsampleTo16k, type WhistleWord } from '@/tools/media/whistle.lib'; + +export type LiveErrorReason = 'denied' | 'unsupported' | 'engine'; +export interface LiveError { reason: LiveErrorReason; message?: string } + +/** Send roughly this much audio per streaming step. */ +const STEP_SEC = 1; +/** Never send more than the engine's per-call limit, even after a slow pass. */ +const MAX_PENDING_SEC = 29; + +// Captures mic frames on the audio thread and posts copies to the main thread. +const WORKLET = ` +class GwtCapture extends AudioWorkletProcessor { + process(inputs) { + const ch = inputs[0] && inputs[0][0]; + if (ch) this.port.postMessage(ch.slice(0)); + return true; + } +} +registerProcessor('gwt-capture', GwtCapture); +`; + +/** + * Live dictation: streams the microphone to the worker's Whistle engine about + * once a second; committed words accumulate, the unconfirmed tail is `pending`. + * A MediaRecorder keeps the audio so it can be replayed or re-transcribed. + */ +export function useLiveDictation() { + const [live, setLive] = useState(false); + const [starting, setStarting] = useState(false); + const [progress, setProgress] = useState(null); + const [seconds, setSeconds] = useState(0); + const [words, setWords] = useState([]); + const [pending, setPending] = useState(''); + const [blob, setBlob] = useState(null); + const [error, setError] = useState(null); + + const streamRef = useRef(null); + const ctxRef = useRef(null); + const recorderRef = useRef(null); + const chunksRef = useRef([]); + const bufferRef = useRef([]); + const bufferedRef = useRef(0); + const inflightRef = useRef | null>(null); + const langRef = useRef(undefined); + const timerRef = useRef | null>(null); + const activeRef = useRef(false); + /** Every committed word so far (state updates are async; stop() returns this). */ + const allWordsRef = useRef([]); + + const teardown = useCallback(() => { + activeRef.current = false; + if (timerRef.current) clearInterval(timerRef.current); + timerRef.current = null; + streamRef.current?.getTracks().forEach(t => t.stop()); + streamRef.current = null; + void ctxRef.current?.close().catch(() => {}); + ctxRef.current = null; + }, []); + + useEffect(() => teardown, [teardown]); + + /** Drain the buffered native-rate audio into one 16 kHz chunk. */ + const takeBuffered = (rate: number): Float32Array => { + const total = bufferedRef.current; + const merged = new Float32Array(total); + let o = 0; + for (const b of bufferRef.current) { merged.set(b, o); o += b.length; } + bufferRef.current = []; + bufferedRef.current = 0; + return downsampleTo16k(merged, rate); + }; + + const send = useCallback((pcm: Float32Array) => { + const run = liveStream.push(pcm, langRef.current).then(u => { + if (u.words.length) { + allWordsRef.current = [...allWordsRef.current, ...u.words]; + setWords(allWordsRef.current); + } + setPending(u.pending); + }); + inflightRef.current = run.catch(err => { + setError({ reason: 'engine', message: err instanceof Error ? err.message : undefined }); + }).finally(() => { inflightRef.current = null; }); + }, []); + + const start = useCallback(async (language: string | undefined) => { + setError(null); + setBlob(null); + setWords([]); + allWordsRef.current = []; + setPending(''); + langRef.current = language; + if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia || typeof AudioWorkletNode === 'undefined') { + setError({ reason: 'unsupported' }); + return; + } + setStarting(true); + try { + await liveStream.start(r => setProgress(r)); + setProgress(null); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }); + streamRef.current = stream; + + const ctx = new AudioContext(); + ctxRef.current = ctx; + const url = URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' })); + try { await ctx.audioWorklet.addModule(url); } finally { URL.revokeObjectURL(url); } + const source = ctx.createMediaStreamSource(stream); + const node = new AudioWorkletNode(ctx, 'gwt-capture'); + const mute = ctx.createGain(); + mute.gain.value = 0; + source.connect(node).connect(mute).connect(ctx.destination); + + const rate = ctx.sampleRate; + node.port.onmessage = (e: MessageEvent) => { + if (!activeRef.current) return; + bufferRef.current.push(e.data); + bufferedRef.current += e.data.length; + const ready = bufferedRef.current >= STEP_SEC * rate; + // Back-pressure: while a pass is running keep buffering (up to the engine limit). + if (ready && !inflightRef.current) send(takeBuffered(rate)); + else if (bufferedRef.current >= MAX_PENDING_SEC * rate) { + bufferRef.current.shift(); + bufferedRef.current = bufferRef.current.reduce((s, b) => s + b.length, 0); + } + }; + + chunksRef.current = []; + if (typeof MediaRecorder !== 'undefined') { + const rec = new MediaRecorder(stream); + recorderRef.current = rec; + rec.ondataavailable = ev => { if (ev.data.size > 0) chunksRef.current.push(ev.data); }; + rec.onstop = () => setBlob(new Blob(chunksRef.current, { type: chunksRef.current[0]?.type || 'audio/webm' })); + rec.start(); + } + + activeRef.current = true; + setSeconds(0); + timerRef.current = setInterval(() => setSeconds(s => s + 1), 1000); + setLive(true); + } catch (err) { + teardown(); + const name = err instanceof Error ? err.name : ''; + setError(name === 'NotAllowedError' || name === 'SecurityError' + ? { reason: 'denied' } + : { reason: 'engine', message: err instanceof Error ? err.message : undefined }); + } finally { + setStarting(false); + setProgress(null); + } + }, [send, teardown]); + + /** Stop listening, flush the last audio and finalise the transcript. */ + const stop = useCallback(async (): Promise => { + const rate = ctxRef.current?.sampleRate ?? 16000; + const rec = recorderRef.current; + if (rec && rec.state !== 'inactive') rec.stop(); + recorderRef.current = null; + teardown(); + setLive(false); + + try { + if (inflightRef.current) await inflightRef.current; + if (bufferedRef.current > 0) { + send(takeBuffered(rate)); + if (inflightRef.current) await inflightRef.current; + } + const fin = await liveStream.stop(); + allWordsRef.current = [...allWordsRef.current, ...fin.words]; + setWords(allWordsRef.current); + setPending(''); + } catch (err) { + setError({ reason: 'engine', message: err instanceof Error ? err.message : undefined }); + } + return allWordsRef.current; + }, [send, teardown]); + + return { live, starting, progress, seconds, words, pending, blob, error, start, stop }; +} diff --git a/src/islands/media/VoiceToText.tsx b/src/islands/media/VoiceToText.tsx index f7dacd5..dfc2074 100644 --- a/src/islands/media/VoiceToText.tsx +++ b/src/islands/media/VoiceToText.tsx @@ -1,5 +1,5 @@ import { useEffect, useMemo, useRef, useState } from 'react'; -import { Mic, Square, Upload } from 'lucide-react'; +import { Mic, Radio, Square, Upload } from 'lucide-react'; import { Dropzone } from '@/components/ui/Dropzone'; import { Button } from '@/components/ui/Button'; import { Alert } from '@/components/ui/Alert'; @@ -7,11 +7,14 @@ import { ProgressBar } from '@/components/ui/ProgressBar'; import { CopyButton } from '@/components/ui/CopyButton'; import { downloadService } from '@/services/download'; import { useAudioRecorder } from '@/hooks/useAudioRecorder'; +import { useLiveDictation } from '@/hooks/useLiveDictation'; import { useWakeLock } from '@/hooks/useWakeLock'; import { decodeToMono16k } from '@/tools/media/stt-audio.lib'; import { transcribeInWorker } from '@/tools/media/stt.client'; import { saveRecording, loadRecording } from '@/tools/media/recording-store'; import { type SttModelId } from '@/tools/media/stt.engine'; +import { resolveModel, defaultLanguageFor, isWhistleLanguage, wordsToSegments } from '@/tools/media/whistle.lib'; +import { WHISTLE_CACHE } from '@/tools/media/whistle.engine'; import { segmentsToText, segmentsToSrt, @@ -34,12 +37,14 @@ const TR: Record string; fastReset: string; + live: string; liveHint: string; liveFastOnly: string; listening: string; liveStop: (clock: string) => string; + liveStarting: string; liveDenied: string; liveUnsupported: string; liveFailed: string; }> = { en: { models: { - 'onnx-community/whisper-tiny.en': { label: 'English · Fast', note: 'smallest download' }, - 'onnx-community/whisper-base.en': { label: 'English · Accurate', note: 'larger, more accurate' }, - 'onnx-community/whisper-base': { label: 'Multilingual', note: 'auto-detects language' }, + whistle: { label: 'Fast · 7 languages', note: '17 MB, quick on phones — English, German, French, Spanish, Italian, Dutch, Polish' }, + 'onnx-community/whisper-base': { label: 'Multilingual', note: 'auto-detects language; use for Bahasa and other languages' }, 'onnx-community/whisper-small': { label: 'Multilingual · Better', note: 'much better for non-English (e.g. Bahasa); larger download' }, }, languages: { @@ -47,7 +52,7 @@ const TR: Record `Stop (${c})`, record: 'Record', or: 'or', dropTitle: 'Drop an audio or video file', @@ -65,12 +70,22 @@ const TR: Record `Fast doesn’t support ${l} — switched to Multilingual.`, + fastReset: 'Fast handles English, German, French, Spanish, Italian, Dutch and Polish — language reset to Auto-detect.', + live: 'Live dictation', + liveHint: 'See words appear as you speak (Fast model).', + liveFastOnly: 'Live dictation needs the Fast model and one of its languages.', + listening: 'Listening…', + liveStop: c => `Stop live (${c})`, + liveStarting: 'Starting live dictation…', + liveDenied: 'Microphone access was blocked — allow it in your browser settings, or upload a file instead.', + liveUnsupported: 'This browser can’t do live dictation — record or upload instead.', + liveFailed: 'Live dictation stopped', }, id: { models: { - 'onnx-community/whisper-tiny.en': { label: 'Inggris · Cepat', note: 'unduhan terkecil' }, - 'onnx-community/whisper-base.en': { label: 'Inggris · Akurat', note: 'lebih besar, lebih akurat' }, - 'onnx-community/whisper-base': { label: 'Multibahasa', note: 'mendeteksi bahasa otomatis' }, + whistle: { label: 'Cepat · 7 bahasa', note: '17 MB, cepat di ponsel — Inggris, Jerman, Prancis, Spanyol, Italia, Belanda, Polandia' }, + 'onnx-community/whisper-base': { label: 'Multibahasa', note: 'mendeteksi bahasa otomatis; gunakan untuk Bahasa Indonesia dan bahasa lain' }, 'onnx-community/whisper-small': { label: 'Multibahasa · Lebih Baik', note: 'jauh lebih baik untuk non-Inggris (mis. Bahasa Indonesia); unduhan lebih besar' }, }, languages: { @@ -78,7 +93,7 @@ const TR: Record `Berhenti (${c})`, record: 'Rekam', or: 'atau', dropTitle: 'Letakkan berkas audio atau video', @@ -96,14 +111,24 @@ const TR: Record `Model Cepat tidak mendukung ${l} — dialihkan ke Multibahasa.`, + fastReset: 'Model Cepat mendukung Inggris, Jerman, Prancis, Spanyol, Italia, Belanda, dan Polandia — bahasa diatur ke Deteksi otomatis.', + live: 'Dikte langsung', + liveHint: 'Lihat kata muncul saat Anda berbicara (model Cepat).', + liveFastOnly: 'Dikte langsung membutuhkan model Cepat dan salah satu bahasanya.', + listening: 'Mendengarkan…', + liveStop: c => `Hentikan dikte (${c})`, + liveStarting: 'Memulai dikte langsung…', + liveDenied: 'Akses mikrofon diblokir — izinkan di pengaturan browser, atau unggah berkas.', + liveUnsupported: 'Browser ini tidak mendukung dikte langsung — rekam atau unggah berkas saja.', + liveFailed: 'Dikte langsung berhenti', }, }; -const MODELS: { value: SttModelId; label: string; note: string; multilingual?: boolean }[] = [ - { value: 'onnx-community/whisper-tiny.en', label: 'English · Fast', note: 'smallest download' }, - { value: 'onnx-community/whisper-base.en', label: 'English · Accurate', note: 'larger, more accurate' }, - { value: 'onnx-community/whisper-base', label: 'Multilingual', note: 'auto-detects language', multilingual: true }, - { value: 'onnx-community/whisper-small', label: 'Multilingual · Better', note: 'much better for non-English (e.g. Bahasa); larger download', multilingual: true }, +const MODELS: { value: SttModelId; label: string; note: string }[] = [ + { value: 'whistle', label: 'Fast · 7 languages', note: '17 MB, quick on phones' }, + { value: 'onnx-community/whisper-base', label: 'Multilingual', note: 'auto-detects language' }, + { value: 'onnx-community/whisper-small', label: 'Multilingual · Better', note: 'much better for non-English (e.g. Bahasa); larger download' }, ]; // Whisper source-language options (value = the lowercase name Whisper expects). @@ -129,6 +154,7 @@ const LANGUAGES: { value: string; label: string }[] = [ { value: 'german', label: 'German' }, { value: 'italian', label: 'Italian' }, { value: 'dutch', label: 'Dutch' }, + { value: 'polish', label: 'Polish' }, { value: 'russian', label: 'Russian' }, { value: 'turkish', label: 'Turkish' }, ]; @@ -138,12 +164,16 @@ type Tab = 'text' | 'timestamped' | 'subtitles'; export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { const t = TR[lang] ?? TR.en; const recorder = useAudioRecorder(); + const dictation = useLiveDictation(); const wakeLock = useWakeLock(); const [restored, setRestored] = useState(false); const [audioBlob, setAudioBlob] = useState(null); const [audioUrl, setAudioUrl] = useState(''); - const [model, setModel] = useState('onnx-community/whisper-tiny.en'); - const [language, setLanguage] = useState(''); + // Default: the Fast engine, unless the locale's language needs Whisper (Indonesian pages). + const [language, setLanguage] = useState(() => defaultLanguageFor(lang)); + const [model, setModel] = useState(() => resolveModel('whistle', defaultLanguageFor(lang)).model); + const [note, setNote] = useState(''); + const [liveMode, setLiveMode] = useState(false); const [modelProgress, setModelProgress] = useState(null); const [transcribing, setTranscribing] = useState(false); const [elapsed, setElapsed] = useState(0); @@ -179,13 +209,13 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { return () => clearInterval(id); }, [transcribing]); - const setAudio = (blob: Blob, persist = true) => { + const setAudio = (blob: Blob, persist = true, keepTranscript = false) => { if (urlRef.current) URL.revokeObjectURL(urlRef.current); const url = URL.createObjectURL(blob); urlRef.current = url; setAudioUrl(url); setAudioBlob(blob); - setSegments(null); + if (!keepTranscript) setSegments(null); setError(''); if (persist) { setRestored(false); void saveRecording(blob); } }; @@ -230,6 +260,11 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { // shows even on a cache read, so this tells the user whether it's a real download.) const refreshModelCached = async () => { try { + if (model === 'whistle') { + const keys = await (await caches.open(WHISTLE_CACHE)).keys(); + setModelCached(keys.some(r => r.url.endsWith('whistle.cact'))); + return; + } const cache = await caches.open('transformers-cache'); const keys = await cache.keys(); setModelCached(keys.some(r => r.url.includes(`${model}/resolve`))); @@ -250,11 +285,10 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { void wakeLock.request(); // keep the screen on so the phone doesn't lock + discard the tab try { const audio = await decodeToMono16k(audioBlob); - const isMultilingual = MODELS.find(m => m.value === model)?.multilingual; const segs = await transcribeInWorker( audio, model, - isMultilingual ? language || undefined : undefined, + language || undefined, r => setModelProgress(r), ); setModelProgress(null); // model ready (or cached) — now inference (indeterminate) @@ -271,6 +305,55 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { } }; + // Language/model pairing: Fast only covers its 7 languages, so picking another + // language switches to Whisper; picking Fast with such a language resets it. + const chooseLanguage = (value: string) => { + setLanguage(value); + const r = resolveModel(model, value); + if (r.switched) { + setModel(r.model); + setLiveMode(false); + setNote(t.switched(t.languages[value] ?? value)); + } else setNote(''); + }; + const chooseModel = (value: SttModelId) => { + setModel(value); + if (value === 'whistle' && !isWhistleLanguage(language)) { + setLanguage(''); + setNote(t.fastReset); + } else setNote(''); + if (value !== 'whistle') setLiveMode(false); + }; + + // Live dictation: words stream in while recording; on stop they become the result. + const toggleLive = async () => { + if (dictation.live) { + const words = await dictation.stop(); + const segs = wordsToSegments(words); + setSegments(segs); + setEditedText(segmentsToText(segs)); + setTab('text'); + wakeLock.release(); + } else { + audioRef.current?.pause(); + setSegments(null); + setError(''); + void wakeLock.request(); + await dictation.start(language || undefined); + } + }; + // Keep the live recording (replay / re-transcribe) without wiping its transcript. + useEffect(() => { + if (dictation.blob) setAudio(dictation.blob, true, true); + }, [dictation.blob]); + useEffect(() => { + if (!dictation.error) return; + const { reason, message } = dictation.error; + setError(reason === 'denied' ? t.liveDenied : reason === 'unsupported' ? t.liveUnsupported : `${t.liveFailed}${message ? `: ${message}` : ''}`); + wakeLock.release(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [dictation.error]); + const srt = useMemo(() => (segments ? segmentsToSrt(segments) : ''), [segments]); const vtt = useMemo(() => (segments ? segmentsToVtt(segments) : ''), [segments]); const currentSubs = subFormat === 'srt' ? srt : vtt; @@ -287,18 +370,47 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { downloadService.download(new Blob([content], { type: mime }), `transcript.${kind}`); }; - const busy = transcribing; + const busy = transcribing || dictation.starting; + const liveAvailable = model === 'whistle' && isWhistleLanguage(language); return (
{/* Input: record or upload */}
- + {liveMode && liveAvailable ? ( + + ) : ( + + )} + {t.or}
+

{liveAvailable ? t.liveHint : t.liveFastOnly}

+ + {dictation.progress !== null && ( + + )} + {(dictation.live || dictation.words.length > 0 || dictation.pending) && !segments && ( +
+ {dictation.live && {t.listening}} + {dictation.words.map(w => w.word).join(' ')} + {dictation.pending && {dictation.pending}} +
+ )}
@@ -327,7 +439,7 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) { key={m.value} variant={model === m.value ? 'primary' : 'secondary'} aria-pressed={model === m.value} - onClick={() => setModel(m.value)} + onClick={() => chooseModel(m.value)} disabled={busy} title={t.models[m.value]?.note ?? m.note} > @@ -337,23 +449,22 @@ export default function VoiceToText({ lang = 'en' }: { lang?: Lang }) {
- {MODELS.find(m => m.value === model)?.multilingual && ( -