From 6e70178d921c9a81f50e21d05c6062b145019e18 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:49:38 +0000 Subject: [PATCH 01/11] feat: sketch a call timeline schema for voice traces Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 249 +++++++ .../call-timeline/examples/cascaded.json | 623 ++++++++++++++++++ .../call-timeline/examples/realtime.json | 592 +++++++++++++++++ .../voice-schema/call-timeline.test.ts | 43 ++ .../livekit/voice-schema/call-timeline.ts | 106 +++ 5 files changed, 1613 insertions(+) create mode 100644 docs/voice-schema/call-timeline/README.md create mode 100644 docs/voice-schema/call-timeline/examples/cascaded.json create mode 100644 docs/voice-schema/call-timeline/examples/realtime.json create mode 100644 js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts create mode 100644 js/src/instrumentation/livekit/voice-schema/call-timeline.ts diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md new file mode 100644 index 000000000..1ccb54c10 --- /dev/null +++ b/docs/voice-schema/call-timeline/README.md @@ -0,0 +1,249 @@ +# Voice schema option: call timeline + +> AI-created or modified and not human-reviewed in its current form; treat this artifact as provisional. Updated 2026-10-12. + +In this option, integrations log what they observe during a call as an ordered, append-only list of timestamped events. Lingua builds speech, turns, latencies and overlaps from that list. The SDK records observations only; it does not decide what a turn or an utterance is. + +Implemented in this PR: the zod schema (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`), the two example traces (`examples/`), and a test that parses them and checks ordering and start/end pairing. Everything else here is a sketch: the LiveKit and Pipecat capture, and the Lingua mapping. The Lingua side is specified in the companion Lingua PR (`specs/voice-schema-call-timeline.md`). + +## Customer problem + +A customer debugging a voice agent needs to answer five questions: + +- What did each side say? +- What did it sound like? +- Where did time go between the caller finishing and the agent speaking? +- Who talked over whom? +- Which tool call caused a pause? + +Today the evidence is scattered. It sits in span names (`user_turn`, `livekit.agent_speaking`), vendor keys (`contrib.livekit.*`, `contrib.pipecat.*`), span start times on different clocks, and selections that exist only after upload. Realtime pipelines make this worse: transcripts arrive late or never, and the provider's view of an interruption differs from what the caller heard. + +## Contract + +### Where it lives + +The integration buffers events and flushes them as **timeline rows**. These are child spans of the call span (the span that logs `audio.recordings`). + +| Field | Location | Type | Req. | Meaning | +| ------------------------------- | ---------------------- | --------------------- | ---- | --------------------------------------------------------------------------------- | +| span name | `span_attributes.name` | `"voice.timeline"` | yes | Stable name; one row per chunk | +| span type | `span_attributes.type` | `"task"` | yes | | +| parent | `span_parents[0]` | span id | yes | The call span | +| `voice.timeline.call_span_id` | `metadata` | string | yes | Span id of the call span (repeated so Lingua can index chunks from headers alone) | +| `voice.timeline.chunk_index` | `metadata` | int ≥ 0 | yes | 0, 1, 2… per call, no gaps | +| `voice.timeline.final` | `metadata` | bool | yes | `true` only on the last chunk, written at call end | +| `voice.events` | `metadata` | `VoiceEvent[]`, 1–256 | yes | Events with consecutive `seq` | +| `metrics.start` / `metrics.end` | row | unix s | yes | First and last event's wall time; used only for ordering spans in the trace tree | + +**Flush rule:** + +- Flush when 64 events are buffered, when the oldest buffered event is 5 s old, or at call end. +- A chunk is never rewritten. Late information becomes a new event in a later chunk. + +**Why child rows, not the call span:** + +- Rewriting one growing array on the call span would make each merge O(n). +- It would also hit row-size limits on long calls. +- It would also hide the timeline until the call ends. + +**Why `metadata`, not `input`/`output`:** Lingua builds `voice_calls` from span headers, without fetching payloads (`TRAJECTORY_METADATA_FIELDS`). A 64-event chunk is about 15–25 KB with transcripts and about 8 KB without. + +### `VoiceEvent` + +Common fields: + +| Field | Type | Req. | Meaning | +| ------- | ---------------------------------- | ---- | ------------------------------------------------------------------------------------------------------- | +| `seq` | int ≥ 0 | yes | Per-call sequence, consecutive across chunks. Defines order when `at_ms` ties; a gap means a lost chunk | +| `at_ms` | number | yes | **Call ms**: milliseconds since `audio.recordings[*].timeline.origin_unix_ms` | +| `basis` | `"audio" \| "local" \| "provider"` | yes | How `at_ms` was obtained (see Timing) | +| `kind` | enum below | yes | | + +Kinds: + +| `kind` | Extra fields (req. unless `?`) | Emitted when | +| ---------------- | -------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `speech_start` | `speaker`, `utterance_id`, `response_id?` | Caller speech detected (VAD), or agent audio starts playing out | +| `speech_end` | `speaker`, `utterance_id`, `interrupted?`, `played_ms?` | Caller speech stops, or agent playout stops (finished or cut) | +| `transcript` | `speaker`, `utterance_id`, `final`, `source: stt\|provider\|playout`, `text?`, `message_id?` | STT or provider transcription arrives; for the agent, the text actually played | +| `end_of_turn` | `utterance_ids[]` | The turn detector commits the caller's turn | +| `response_start` | `response_id`, `span_id`, `in_reply_to?[]` | An LLM or realtime generation starts | +| `tool_call` | `response_id`, `tool_call_id`, `name`, `span_id` | A tool starts running | +| `tool_result` | `tool_call_id`, `span_id`, `outcome: ok\|error\|cancelled` | The tool finishes, fails, or is cancelled | +| `truncate` | `utterance_id`, `audio_end_ms` | Realtime provider truncation of an agent item; `audio_end_ms` is the item-relative position reported to the provider | +| `missing` | `evidence: transcript\|speech_end\|audio\|tool_result`, `reason`, `utterance_id?` | The SDK knows that expected evidence will never arrive | +| `call_end` | `reason` | Session closed; must be the last event | + +`text` is omitted when content capture is off. Every other field is timing or identity and is always logged. + +## IDs and relationships + +- **`utterance_id`:** one contiguous stretch of speech by one speaker, unique per call. It is the join key for `speech_start`, `speech_end`, `transcript`, `truncate` and `missing`. + - Caller: the provider item id when one exists (realtime), otherwise an SDK counter `u`. + - Agent: the realtime item id, or the LiveKit `speech_id` or Pipecat TTS `context_id` with a counter suffix if one handle plays several stretches. +- **`response_id`:** one model generation. + - Realtime uses the provider response id; LiveKit cascaded uses `generation_id`; Pipecat uses an SDK counter. + - `response_start.span_id` is the Braintrust span id of the `llm_response` span. This is the only link from the timeline into the step tree. +- **`tool_call_id`:** the provider tool call id. `span_id` on `tool_call` and `tool_result` is the tool span. +- **`in_reply_to`:** the caller utterances whose turn triggered this generation, when the framework knows them. LiveKit knows the committed user message for both pipelines; Pipecat knows the aggregated user turn. +- **Agent speech to a generation:** `speech_start.response_id` ties agent speech to the generation that produced it. +- **Messages:** `message_id` on `transcript` links to the framework chat item (LiveKit `ChatMessage.id`, realtime item id). Lingua can use it to match conversation messages; it is optional. +- **Turns are not emitted.** A turn is inferred by Lingua: one `end_of_turn`, the responses that reply to it, and the agent speech they produce. + +## Timing semantics + +- **One clock.** `at_ms` is on the call clock, the same one the recording uses: `origin_unix_ms` comes from `snapshot().origin` in `@braintrust/audio`. A call ms `t` maps to segment `s` at offset `t - s.timeline.recording_start_offset_ms` when `s` is ready and covers `t`. +- **`basis` values:** + - `audio`: derived from sample positions the recorder placed on the timeline (the caller frame anchor plus samples, or playout ranges from `playbackProgressed`). This is the best value and matches the recording by construction. + - `local`: `Date.now() - origin` when a hook fires. Callbacks are delayed by event-loop latency, typically under 20 ms. + - `provider`: a provider-reported time converted to the call clock. For example, OpenAI `audio_start_ms`/`audio_end_ms` are relative to the input audio buffer the SDK streamed; the SDK adds the call ms at which that buffer began. Use it when the provider's VAD defines the boundary. +- **Delay meanings:** + - `played_ms` on an agent `speech_end` is audio actually played out. + - `truncate.audio_end_ms` is what the provider was told. The two can differ by the output buffer: in the realtime example, 2600 vs 2900. +- **No timeline without a recording.** If no recording exists, there is no origin. Integrations still emit the timeline using `origin = session start Date.now()` and log `voice.timeline_origin_unix_ms` on the call span; without a recording, Lingua treats `audio` basis as `local`. + +## What integrations capture, and when + +### LiveKit (JS, `js/src/instrumentation/livekit`) + +| Event | Cascaded (`AgentActivity._pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| user `speech_start` / `speech_end` | `EndOfTurnInfo.startedSpeakingAt` / `stoppedSpeakingAt` in `onEndOfTurn` (`local`); refined to `audio` from capture anchors when audio capture is on | `input_speech_started` / `input_speech_stopped` (`local`), or provider `audio_start_ms`/`audio_end_ms` (`provider`) when exposed | +| user `transcript` | `user_input_transcribed` (final only), `source: stt` | `input_audio_transcription_completed`, `source: provider`; may arrive after `response_start` | +| `end_of_turn` | `onEndOfTurn` hook | `input_audio_buffer.committed` / `userTurnCompleted` | +| `response_start` | `llm_response` span start (`processor.ts` `onStart`), `response_id = generation_id` | realtime `llm_response` start, `response_id = contrib.livekit.response_id` | +| `tool_call` / `tool_result` | `function_tool` span start / end; `outcome` from `function_tool.is_error`, or `cancelled` when the speech handle was interrupted before the result | same | +| agent `speech_start` / `speech_end` | `playback.ts`: first `playbackProgressed.startedAt` / `playbackFinished` (`audio`), `interrupted`, `played_ms = playbackPosition` | same | +| agent `transcript` | `_conversationItemAdded` assistant item, `source: playout` (synchronized transcript) | same, `source: provider` | +| `truncate` | — | `RealtimeSession.truncate` call (hook) | +| `missing` | at close: user utterances with no final transcript | same; common for barge-ins | +| `call_end` | `closeImpl` | same | + +Starting point: add a `Timeline` class beside `turns.ts`. It holds `seq`, a buffer and the flush rule, and is owned by the per-session `Capture`/processor state. `TurnTracker` grouping is no longer needed for emission: the timeline logs raw fragments and Lingua groups them. + +### Pipecat (Python, sketch against `sdk-python` `delner/pipecat-voice-recording`) + +| Event | Cascaded | Realtime (OpenAI/Gemini realtime services) | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | +| user `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`; `audio` basis from input sample clock (`_audio` alignment) | provider `input_audio_buffer.speech_started/stopped` (`provider`) | +| user `transcript` | `TranscriptionFrame` (`finalized`), `source: stt` | `TranscriptionFrame` from the service's input transcription, `source: provider` | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits | service commit event | +| `response_start` | `LLMFullResponseStartFrame` (`llm_response` span) | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `BotStartedSpeakingFrame` / `BotStoppedSpeakingFrame` at transport output (`audio` from output write observation); `interrupted` when an `InterruptionFrame` preceded the stop | same | +| agent `transcript` | `TTSTextFrame` text accumulated per `context_id`, cut at interruption, `source: playout` | provider transcript, `source: provider` | +| `truncate` | — | service truncation call | + +## What Lingua records versus infers + +**Recorded (copied, normalized):** + +- Events sorted by `seq` into `VoiceCall.events`. +- Recordings, unchanged from #486. + +**Inferred:** + +- `Speech` entries: one per `utterance_id` with a `speech_start`. Selections come from `[start, end]` intersected with ready recordings. `interrupted` comes from `speech_end`. +- `Speech.step_ids`: + - Agent: the `response_start.span_id` of its `response_id`. + - Caller: the `span_id` of each `response_start` whose `in_reply_to` contains it. + - Without `in_reply_to`: the first `response_start` after its `end_of_turn`, flagged as inferred. +- Latencies and overlaps (see `VoiceCall.events` consumers below). + +**Not inferred:** + +- Text that was never transcribed. +- A `speech_end` that never arrived. Lingua reports a warning and clips the speech at the next event or at `call_end`. + +## Trajectory and app mapping + +`Trajectory.voice_calls[*]` keeps the #486 fields. `speech` and `recordings` are built from the timeline instead of turn spans, so #22034 and #22036 work unchanged: the voice bar, click-to-step, play-from-message and highlighting all read `speech[].step_ids` and `selections`. + +The new field `VoiceCall.events` exposes the normalized log for: + +- a latency strip showing caller end → `end_of_turn` → `response_start` → first agent `speech_start`; +- overlap shading, from intersecting caller and agent speech; +- "no transcript" badges, from `missing` events. + +Thread messages still come from the LLM spans' inputs and outputs, as today. + +## Worked example + +`examples/cascaded.json` and `examples/realtime.json` encode the same fictional call (see the scenario in the comparison note). Row tree: + +``` +livekit.agent_session audio.recordings: call-0000 (13 s) +├── llm_response llm-1 / rt-1 → find_slots +├── find_slots tool-1 +├── llm_response llm-2 / rt-2 "I have nine thirty or eleven on Friday…" +├── tts tts-1 (cascaded only) +├── llm_response llm-3 / rt-3 → reschedule +├── reschedule tool-2 +├── llm_response llm-4 / rt-4 "Done — you're booked…" +├── tts tts-2 (cascaded only) +├── voice.timeline timeline-0 seq 0–11 +└── voice.timeline timeline-1 seq 12–22/23, final +``` + +Concrete differences, same audio: + +| | Cascaded | Realtime | +| ------------------------------------------------ | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Caller transcript of turn 1 | `transcript` at 3450 (`stt`), before `response_start` | `transcript` at 4300 (`provider`), **after** `tool_call` at 4100 | +| Turn-1 latency (end_of_turn → first agent audio) | 3400 → 5300 = 1900 ms: 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms: 200 ms to response, 500 ms to the tool call, 500 ms tool, 400 ms to audio | +| Barge-in | agent `speech_end` 7900, `interrupted`, `played_ms` 2600 | `truncate` at 7600 (`audio_end_ms` 2600) **and** `speech_end` at 7900 (`played_ms` 2900); a 300 ms disagreement is visible | +| Agent text actually said | `playout` transcript "…on Fri" vs planned text in `llm-2` output | provider transcript of the truncated item; `rt-2` output already holds the truncated text | +| Caller's barge-in text | `stt` transcript at 8600 | **none**: `missing{evidence: transcript}` at 13000; `rt-3` input user content is `""` | +| IDs | `u1`, `a1`, `r1` SDK ids | provider item and response ids (`item_u1`, `resp_1`) | +| Separate STT/TTS steps | `tts-1`, `tts-2` | none | + +## Customer usefulness + +- **What was said: good.** + - Final transcripts with provenance. + - Agent text that was actually played, separate from generated text. + - Missing transcripts are explicit. +- **Playing matching audio: good.** + - Every speech interval is on the recording clock. + - Selections are computed by Lingua, so they work across segment boundaries and before upload metadata exists. +- **Diagnosing delays: strongest of the options.** + - Every gap between caller end, end-of-turn, generation, tool and first audio is a subtraction on one clock, including realtime. +- **Interruptions and overlap: strong.** + - Overlap comes from intervals, interruption from `speech_end`. + - The provider-versus-playout truncation gap is visible. +- **Following tools: adequate.** + - The timeline places tools in time and links them to spans. + - Tool arguments and results stay on tool spans. +- **Missing or uncertain evidence: strong.** + - `basis` on every timestamp. + - `missing` events. + - `seq` gaps reveal lost chunks. + - `final` reveals an in-progress call. + +## Edge cases and weaknesses + +- **Barge-in truncation:** handled, as above. The planned text remains only on the LLM span. +- **Overlap:** intervals can overlap freely, and Lingua does not merge them. +- **Late or missing realtime transcripts:** a late transcript is a later event with the same `utterance_id`. Missing ones get `missing`. Weakness: if the provider never reports which item was being transcribed, the SDK must guess the `utterance_id`, as `TurnTracker.transcript` does today. +- **Failed or omitted segments:** speech still has times but no selection in that range. Lingua emits partial selections and the app shows silence. This is better than #486, where a failed segment drops the turn's selections. +- **Unknown clock offset:** `basis: local` events skew relative to `audio` events, and nothing measures that skew. A `clock_offset` event (local vs audio at the same instant) could carry it; this is not specified. +- **Call in progress:** chunks appear every 5 s or 64 events, so the latest ≤5 s are invisible. Lingua warns when no chunk is `final`. +- **Content capture off:** `text` is dropped and everything else stays, so timing analysis still works. +- **Tool cancelled by interruption:** `tool_result{outcome: cancelled}`. The cancellation's cause is not linked; it is inferred from an agent `speech_end{interrupted}` near the same time. +- **Agent handoff:** handoffs are not modeled. Events from a new agent activity continue the same log. A `speaker`-level agent name would be needed to show which agent spoke. This is a weakness. +- **Duplicate observers:** two hooks for the same moment (LiveKit native `user_turn` plus realtime events) must emit one event. The SDK owns dedup, and Lingua does not dedup. +- **Main weakness overall: indirection.** Speech, turns and the step links all depend on Lingua's derivation being right, and the raw spans in the trace tree no longer show speech on their own. Debugging the derivation means reading an event log. Row volume is low (one row per 64 events), but it is a new kind of row in the trace tree. + +## Implementation starting point + +1. Lock the event schema from `call-timeline.ts`, including `basis` and the flush rule. +2. LiveKit: add `timeline.ts` (buffer, `seq`, flush) to the session state in `processor.ts`. Emit from the hooks already present: `onEndOfTurn`, `RealtimeObserver` listeners, `playback.ts` `settle`, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Keep existing spans and selections during the transition. +3. Add `missing` at close: utterances without a final transcript, and starts without ends. +4. Lingua: implement `specs/voice-schema-call-timeline.md`, starting with speech and selections from the timeline, then `step_ids`, then `events` on `VoiceCall`. +5. Pipecat: port `timeline.ts` to the Python `voice` package using the frame table above. + +## Open questions + +- Should `end_of_turn` carry the detector's delay and probability (LiveKit `eou.*`), or should those stay on the native `eou` spans? +- Are 5 s and 64 events the right flush defaults for live viewing versus row count? +- Should agent `transcript` events carry word-level timings when the TTS provides alignments? diff --git a/docs/voice-schema/call-timeline/examples/cascaded.json b/docs/voice-schema/call-timeline/examples/cascaded.json new file mode 100644 index 000000000..231f91e63 --- /dev/null +++ b/docs/voice-schema/call-timeline/examples/cascaded.json @@ -0,0 +1,623 @@ +[ + { + "id": "call-span", + "span_id": "call-span", + "root_span_id": "call-span", + "span_parents": [], + "span_attributes": { + "name": "livekit.agent_session", + "type": "task" + }, + "metrics": { + "start": 1791830000.0, + "end": 1791830013.2 + }, + "input": { + "audio": { + "call-0000": { + "type": "braintrust_attachment", + "filename": "call-0000.ogg", + "content_type": "audio/ogg", + "key": "fictional/call-0000.ogg" + } + } + }, + "metadata": { + "audio.recordings": [ + { + "id": "call-0000", + "recording_group_id": "call", + "state": "ready", + "attachment": { + "span_id": "call-span", + "ref": "/input/audio/call-0000" + }, + "mime_type": "audio/ogg", + "duration_ms": 13000, + "channel_count": 2, + "sample_rate_hz": 48000, + "sources": [ + { + "channel_index": 0, + "boundary": "session_input" + }, + { + "channel_index": 1, + "boundary": "local_audio_output" + } + ], + "timeline": { + "origin_unix_ms": 1791830000000, + "recording_start_offset_ms": 0, + "basis": "local_input_arrival_and_output_playout" + } + } + ], + "voice.timeline_version": 1, + "pipeline": "cascaded" + } + }, + { + "id": "llm-1", + "span_id": "llm-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830003.45, + "end": 1791830004.1 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "find_slots", + "arguments": "{\"date\": \"2026-10-16\", \"window\": \"morning\"}" + } + } + ] + } + ], + "metadata": { + "model": "gpt-4.1-nano", + "provider": "openai" + } + }, + { + "id": "tool-1", + "span_id": "tool-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "find_slots", + "type": "tool" + }, + "metrics": { + "start": 1791830004.1, + "end": 1791830004.6 + }, + "input": { + "date": "2026-10-16", + "window": "morning" + }, + "output": ["09:30", "11:00"], + "metadata": { + "gen_ai.tool.call.id": "call_1", + "gen_ai.tool.name": "find_slots", + "contrib.livekit.request_span_id": "llm-1" + } + }, + { + "id": "llm-2", + "span_id": "llm-2", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830004.6, + "end": 1791830005.1 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "find_slots", + "arguments": "{\"date\": \"2026-10-16\", \"window\": \"morning\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "[\"09:30\",\"11:00\"]" + } + ], + "output": [ + { + "role": "assistant", + "content": "I have nine thirty or eleven on Friday. Which would you like?" + } + ], + "metadata": { + "model": "gpt-4.1-nano", + "provider": "openai" + } + }, + { + "id": "tts-1", + "span_id": "tts-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "tts", + "type": "task" + }, + "metrics": { + "start": 1791830005.05, + "end": 1791830007.9 + }, + "input": { + "text": "I have nine thirty or eleven on Friday. Which would you like?" + }, + "metadata": { + "model": "gpt-4o-mini-tts", + "provider": "openai", + "contrib.livekit.response.ttfb": 0.25 + } + }, + { + "id": "llm-3", + "span_id": "llm-3", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830008.7, + "end": 1791830009.0 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": "I have nine thirty or eleven on Fri" + }, + { + "role": "user", + "content": "Nine thirty works." + } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "reschedule", + "arguments": "{\"appointment_id\": \"apt_481\", \"slot\": \"2026-10-16T09:30\"}" + } + } + ] + } + ], + "metadata": { + "model": "gpt-4.1-nano", + "provider": "openai" + } + }, + { + "id": "tool-2", + "span_id": "tool-2", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "reschedule", + "type": "tool" + }, + "metrics": { + "start": 1791830009.0, + "end": 1791830009.4 + }, + "input": { + "appointment_id": "apt_481", + "slot": "2026-10-16T09:30" + }, + "output": { + "ok": true + }, + "metadata": { + "gen_ai.tool.call.id": "call_2", + "gen_ai.tool.name": "reschedule", + "contrib.livekit.request_span_id": "llm-3" + } + }, + { + "id": "llm-4", + "span_id": "llm-4", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830009.4, + "end": 1791830009.7 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": "I have nine thirty or eleven on Fri" + }, + { + "role": "user", + "content": "Nine thirty works." + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "reschedule", + "arguments": "{\"appointment_id\": \"apt_481\", \"slot\": \"2026-10-16T09:30\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_2", + "content": "{\"ok\":true}" + } + ], + "output": [ + { + "role": "assistant", + "content": "Done — you're booked Friday at nine thirty." + } + ], + "metadata": { + "model": "gpt-4.1-nano", + "provider": "openai" + } + }, + { + "id": "tts-2", + "span_id": "tts-2", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "tts", + "type": "task" + }, + "metrics": { + "start": 1791830009.65, + "end": 1791830012.6 + }, + "input": { + "text": "Done — you're booked Friday at nine thirty." + }, + "metadata": { + "model": "gpt-4o-mini-tts", + "provider": "openai", + "contrib.livekit.response.ttfb": 0.25 + } + }, + { + "id": "timeline-0", + "span_id": "timeline-0", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "voice.timeline", + "type": "task" + }, + "metrics": { + "start": 1791830000.3, + "end": 1791830007.9 + }, + "metadata": { + "voice.timeline": { + "call_span_id": "call-span", + "chunk_index": 0, + "final": false + }, + "voice.events": [ + { + "seq": 0, + "at_ms": 300, + "basis": "audio", + "kind": "speech_start", + "speaker": "user", + "utterance_id": "u1" + }, + { + "seq": 1, + "at_ms": 2900, + "basis": "audio", + "kind": "speech_end", + "speaker": "user", + "utterance_id": "u1" + }, + { + "seq": 2, + "at_ms": 3400, + "basis": "local", + "kind": "end_of_turn", + "utterance_ids": ["u1"] + }, + { + "seq": 3, + "at_ms": 3450, + "basis": "local", + "kind": "transcript", + "speaker": "user", + "utterance_id": "u1", + "final": true, + "text": "Hi, can I move my Thursday cleaning to Friday morning?", + "message_id": "item_u1", + "source": "stt" + }, + { + "seq": 4, + "at_ms": 3450, + "basis": "local", + "kind": "response_start", + "response_id": "r1", + "span_id": "llm-1", + "in_reply_to": ["u1"] + }, + { + "seq": 5, + "at_ms": 4100, + "basis": "local", + "kind": "tool_call", + "response_id": "r1", + "tool_call_id": "call_1", + "name": "find_slots", + "span_id": "tool-1" + }, + { + "seq": 6, + "at_ms": 4600, + "basis": "local", + "kind": "tool_result", + "tool_call_id": "call_1", + "span_id": "tool-1", + "outcome": "ok" + }, + { + "seq": 7, + "at_ms": 4600, + "basis": "local", + "kind": "response_start", + "response_id": "r2", + "span_id": "llm-2", + "in_reply_to": ["u1"] + }, + { + "seq": 8, + "at_ms": 5300, + "basis": "audio", + "kind": "speech_start", + "speaker": "agent", + "utterance_id": "a1", + "response_id": "r2" + }, + { + "seq": 9, + "at_ms": 7600, + "basis": "audio", + "kind": "speech_start", + "speaker": "user", + "utterance_id": "u2" + }, + { + "seq": 10, + "at_ms": 7900, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "a1", + "interrupted": true, + "played_ms": 2600 + }, + { + "seq": 11, + "at_ms": 7900, + "basis": "local", + "kind": "transcript", + "speaker": "agent", + "utterance_id": "a1", + "final": true, + "text": "I have nine thirty or eleven on Fri", + "message_id": "item_a1", + "source": "playout" + } + ] + } + }, + { + "id": "timeline-1", + "span_id": "timeline-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "voice.timeline", + "type": "task" + }, + "metrics": { + "start": 1791830008.5, + "end": 1791830013.0 + }, + "metadata": { + "voice.timeline": { + "call_span_id": "call-span", + "chunk_index": 1, + "final": true + }, + "voice.events": [ + { + "seq": 12, + "at_ms": 8500, + "basis": "audio", + "kind": "speech_end", + "speaker": "user", + "utterance_id": "u2" + }, + { + "seq": 13, + "at_ms": 8600, + "basis": "local", + "kind": "transcript", + "speaker": "user", + "utterance_id": "u2", + "final": true, + "text": "Nine thirty works.", + "message_id": "item_u2", + "source": "stt" + }, + { + "seq": 14, + "at_ms": 8700, + "basis": "local", + "kind": "end_of_turn", + "utterance_ids": ["u2"] + }, + { + "seq": 15, + "at_ms": 8700, + "basis": "local", + "kind": "response_start", + "response_id": "r3", + "span_id": "llm-3", + "in_reply_to": ["u2"] + }, + { + "seq": 16, + "at_ms": 9000, + "basis": "local", + "kind": "tool_call", + "response_id": "r3", + "tool_call_id": "call_2", + "name": "reschedule", + "span_id": "tool-2" + }, + { + "seq": 17, + "at_ms": 9400, + "basis": "local", + "kind": "tool_result", + "tool_call_id": "call_2", + "span_id": "tool-2", + "outcome": "ok" + }, + { + "seq": 18, + "at_ms": 9400, + "basis": "local", + "kind": "response_start", + "response_id": "r4", + "span_id": "llm-4", + "in_reply_to": ["u2"] + }, + { + "seq": 19, + "at_ms": 9900, + "basis": "audio", + "kind": "speech_start", + "speaker": "agent", + "utterance_id": "a2", + "response_id": "r4" + }, + { + "seq": 20, + "at_ms": 12600, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "a2", + "interrupted": false, + "played_ms": 2700 + }, + { + "seq": 21, + "at_ms": 12600, + "basis": "local", + "kind": "transcript", + "speaker": "agent", + "utterance_id": "a2", + "final": true, + "text": "Done — you're booked Friday at nine thirty.", + "message_id": "item_a2", + "source": "playout" + }, + { + "seq": 22, + "at_ms": 13000, + "basis": "local", + "kind": "call_end", + "reason": "participant_disconnected" + } + ] + } + } +] diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json new file mode 100644 index 000000000..f7c21164a --- /dev/null +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -0,0 +1,592 @@ +[ + { + "id": "call-span", + "span_id": "call-span", + "root_span_id": "call-span", + "span_parents": [], + "span_attributes": { + "name": "livekit.agent_session", + "type": "task" + }, + "metrics": { + "start": 1791830000.0, + "end": 1791830013.2 + }, + "input": { + "audio": { + "call-0000": { + "type": "braintrust_attachment", + "filename": "call-0000.ogg", + "content_type": "audio/ogg", + "key": "fictional/call-0000.ogg" + } + } + }, + "metadata": { + "audio.recordings": [ + { + "id": "call-0000", + "recording_group_id": "call", + "state": "ready", + "attachment": { + "span_id": "call-span", + "ref": "/input/audio/call-0000" + }, + "mime_type": "audio/ogg", + "duration_ms": 13000, + "channel_count": 2, + "sample_rate_hz": 48000, + "sources": [ + { + "channel_index": 0, + "boundary": "session_input" + }, + { + "channel_index": 1, + "boundary": "local_audio_output" + } + ], + "timeline": { + "origin_unix_ms": 1791830000000, + "recording_start_offset_ms": 0, + "basis": "local_input_arrival_and_output_playout" + } + } + ], + "voice.timeline_version": 1, + "pipeline": "realtime" + } + }, + { + "id": "rt-1", + "span_id": "rt-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830003.6, + "end": 1791830004.1 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "find_slots", + "arguments": "{\"date\": \"2026-10-16\", \"window\": \"morning\"}" + } + } + ] + } + ], + "metadata": { + "model": "gpt-realtime-mini", + "provider": "openai", + "contrib.livekit.input_scope": "realtime_session_context", + "contrib.livekit.response_id": "resp_1" + } + }, + { + "id": "tool-1", + "span_id": "tool-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "find_slots", + "type": "tool" + }, + "metrics": { + "start": 1791830004.1, + "end": 1791830004.6 + }, + "input": { + "date": "2026-10-16", + "window": "morning" + }, + "output": ["09:30", "11:00"], + "metadata": { + "gen_ai.tool.call.id": "call_1", + "gen_ai.tool.name": "find_slots", + "contrib.livekit.request_span_id": "rt-1" + } + }, + { + "id": "rt-2", + "span_id": "rt-2", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830004.6, + "end": 1791830007.9 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "find_slots", + "arguments": "{\"date\": \"2026-10-16\", \"window\": \"morning\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "[\"09:30\",\"11:00\"]" + } + ], + "output": [ + { + "role": "assistant", + "content": "I have nine thirty or eleven on Fri" + } + ], + "metadata": { + "model": "gpt-realtime-mini", + "provider": "openai", + "contrib.livekit.input_scope": "realtime_session_context", + "contrib.livekit.response_id": "resp_2" + } + }, + { + "id": "rt-3", + "span_id": "rt-3", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830008.6, + "end": 1791830008.9 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": "I have nine thirty or eleven on Fri" + }, + { + "role": "user", + "content": "" + } + ], + "output": [ + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "reschedule", + "arguments": "{\"appointment_id\": \"apt_481\", \"slot\": \"2026-10-16T09:30\"}" + } + } + ] + } + ], + "metadata": { + "model": "gpt-realtime-mini", + "provider": "openai", + "contrib.livekit.input_scope": "realtime_session_context", + "contrib.livekit.response_id": "resp_3" + } + }, + { + "id": "tool-2", + "span_id": "tool-2", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "reschedule", + "type": "tool" + }, + "metrics": { + "start": 1791830008.9, + "end": 1791830009.3 + }, + "input": { + "appointment_id": "apt_481", + "slot": "2026-10-16T09:30" + }, + "output": { + "ok": true + }, + "metadata": { + "gen_ai.tool.call.id": "call_2", + "gen_ai.tool.name": "reschedule", + "contrib.livekit.request_span_id": "rt-3" + } + }, + { + "id": "rt-4", + "span_id": "rt-4", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "llm_response", + "type": "llm" + }, + "metrics": { + "start": 1791830009.3, + "end": 1791830012.5 + }, + "input": [ + { + "role": "system", + "content": "You are Molar, the receptionist for a dental clinic." + }, + { + "role": "user", + "content": "Hi, can I move my Thursday cleaning to Friday morning?" + }, + { + "role": "assistant", + "content": "I have nine thirty or eleven on Fri" + }, + { + "role": "user", + "content": "" + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "reschedule", + "arguments": "{\"appointment_id\": \"apt_481\", \"slot\": \"2026-10-16T09:30\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_2", + "content": "{\"ok\":true}" + } + ], + "output": [ + { + "role": "assistant", + "content": "Done — you're booked Friday at nine thirty." + } + ], + "metadata": { + "model": "gpt-realtime-mini", + "provider": "openai", + "contrib.livekit.input_scope": "realtime_session_context", + "contrib.livekit.response_id": "resp_4" + } + }, + { + "id": "timeline-0", + "span_id": "timeline-0", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "voice.timeline", + "type": "task" + }, + "metrics": { + "start": 1791830000.3, + "end": 1791830007.9 + }, + "metadata": { + "voice.timeline": { + "call_span_id": "call-span", + "chunk_index": 0, + "final": false + }, + "voice.events": [ + { + "seq": 0, + "at_ms": 300, + "basis": "provider", + "kind": "speech_start", + "speaker": "user", + "utterance_id": "item_u1" + }, + { + "seq": 1, + "at_ms": 2900, + "basis": "provider", + "kind": "speech_end", + "speaker": "user", + "utterance_id": "item_u1" + }, + { + "seq": 2, + "at_ms": 3400, + "basis": "provider", + "kind": "end_of_turn", + "utterance_ids": ["item_u1"] + }, + { + "seq": 3, + "at_ms": 3600, + "basis": "local", + "kind": "response_start", + "response_id": "resp_1", + "span_id": "rt-1", + "in_reply_to": ["item_u1"] + }, + { + "seq": 4, + "at_ms": 4100, + "basis": "local", + "kind": "tool_call", + "response_id": "resp_1", + "tool_call_id": "call_1", + "name": "find_slots", + "span_id": "tool-1" + }, + { + "seq": 5, + "at_ms": 4300, + "basis": "local", + "kind": "transcript", + "speaker": "user", + "utterance_id": "item_u1", + "final": true, + "text": "Hi, can I move my Thursday cleaning to Friday morning?", + "message_id": "item_u1", + "source": "provider" + }, + { + "seq": 6, + "at_ms": 4600, + "basis": "local", + "kind": "tool_result", + "tool_call_id": "call_1", + "span_id": "tool-1", + "outcome": "ok" + }, + { + "seq": 7, + "at_ms": 4600, + "basis": "local", + "kind": "response_start", + "response_id": "resp_2", + "span_id": "rt-2", + "in_reply_to": ["item_u1"] + }, + { + "seq": 8, + "at_ms": 5000, + "basis": "audio", + "kind": "speech_start", + "speaker": "agent", + "utterance_id": "item_a1", + "response_id": "resp_2" + }, + { + "seq": 9, + "at_ms": 7600, + "basis": "provider", + "kind": "speech_start", + "speaker": "user", + "utterance_id": "item_u2" + }, + { + "seq": 10, + "at_ms": 7600, + "basis": "provider", + "kind": "truncate", + "utterance_id": "item_a1", + "audio_end_ms": 2600 + }, + { + "seq": 11, + "at_ms": 7900, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "item_a1", + "interrupted": true, + "played_ms": 2900 + } + ] + } + }, + { + "id": "timeline-1", + "span_id": "timeline-1", + "root_span_id": "call-span", + "span_parents": ["call-span"], + "span_attributes": { + "name": "voice.timeline", + "type": "task" + }, + "metrics": { + "start": 1791830007.9, + "end": 1791830013.0 + }, + "metadata": { + "voice.timeline": { + "call_span_id": "call-span", + "chunk_index": 1, + "final": true + }, + "voice.events": [ + { + "seq": 12, + "at_ms": 7900, + "basis": "local", + "kind": "transcript", + "speaker": "agent", + "utterance_id": "item_a1", + "final": true, + "text": "I have nine thirty or eleven on Fri", + "message_id": "item_a1", + "source": "provider" + }, + { + "seq": 13, + "at_ms": 8500, + "basis": "provider", + "kind": "speech_end", + "speaker": "user", + "utterance_id": "item_u2" + }, + { + "seq": 14, + "at_ms": 8600, + "basis": "provider", + "kind": "end_of_turn", + "utterance_ids": ["item_u2"] + }, + { + "seq": 15, + "at_ms": 8600, + "basis": "local", + "kind": "response_start", + "response_id": "resp_3", + "span_id": "rt-3", + "in_reply_to": ["item_u2"] + }, + { + "seq": 16, + "at_ms": 8900, + "basis": "local", + "kind": "tool_call", + "response_id": "resp_3", + "tool_call_id": "call_2", + "name": "reschedule", + "span_id": "tool-2" + }, + { + "seq": 17, + "at_ms": 9300, + "basis": "local", + "kind": "tool_result", + "tool_call_id": "call_2", + "span_id": "tool-2", + "outcome": "ok" + }, + { + "seq": 18, + "at_ms": 9300, + "basis": "local", + "kind": "response_start", + "response_id": "resp_4", + "span_id": "rt-4", + "in_reply_to": ["item_u2"] + }, + { + "seq": 19, + "at_ms": 9700, + "basis": "audio", + "kind": "speech_start", + "speaker": "agent", + "utterance_id": "item_a2", + "response_id": "resp_4" + }, + { + "seq": 20, + "at_ms": 12500, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "item_a2", + "interrupted": false, + "played_ms": 2800 + }, + { + "seq": 21, + "at_ms": 12500, + "basis": "local", + "kind": "transcript", + "speaker": "agent", + "utterance_id": "item_a2", + "final": true, + "text": "Done — you're booked Friday at nine thirty.", + "message_id": "item_a2", + "source": "provider" + }, + { + "seq": 22, + "at_ms": 13000, + "basis": "local", + "kind": "missing", + "utterance_id": "item_u2", + "evidence": "transcript", + "reason": "input_transcription_not_received" + }, + { + "seq": 23, + "at_ms": 13000, + "basis": "local", + "kind": "call_end", + "reason": "participant_disconnected" + } + ] + } + } +] diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts new file mode 100644 index 000000000..3fb85e7d2 --- /dev/null +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts @@ -0,0 +1,43 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { timelineRowMetadataSchema, type VoiceEvent } from "./call-timeline"; + +const examples = path.resolve( + __dirname, + "../../../../../docs/voice-schema/call-timeline/examples", +); + +function events(file: string): VoiceEvent[] { + const rows: { metadata?: Record }[] = JSON.parse( + readFileSync(path.join(examples, file), "utf8"), + ); + return rows + .filter((row) => row.metadata?.["voice.timeline"] !== undefined) + .map((row) => timelineRowMetadataSchema.parse(row.metadata)) + .sort( + (a, b) => + a["voice.timeline"].chunk_index - b["voice.timeline"].chunk_index, + ) + .flatMap((chunk) => chunk["voice.events"]); +} + +describe.each(["cascaded.json", "realtime.json"])("%s", (file) => { + it("parses as a gapless, time-ordered event log", () => { + const log = events(file); + expect(log.map((event) => event.seq)).toEqual(log.map((_, i) => i)); + for (let i = 1; i < log.length; i++) + expect(log[i].at_ms).toBeGreaterThanOrEqual(log[i - 1].at_ms); + expect(log.at(-1)?.kind).toBe("call_end"); + }); + + it("pairs every speech start with an end", () => { + const log = events(file); + const open = new Set(); + for (const event of log) { + if (event.kind === "speech_start") open.add(event.utterance_id); + if (event.kind === "speech_end") open.delete(event.utterance_id); + } + expect([...open]).toEqual([]); + }); +}); diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts new file mode 100644 index 000000000..1699aae91 --- /dev/null +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts @@ -0,0 +1,106 @@ +import { z } from "zod/v3"; + +const speaker = z.enum(["user", "agent"]); + +const base = z.object({ + seq: z.number().int().nonnegative(), + at_ms: z.number().nonnegative(), + basis: z.enum(["audio", "local", "provider"]), +}); + +const speechStart = base.extend({ + kind: z.literal("speech_start"), + speaker, + utterance_id: z.string().min(1), + response_id: z.string().min(1).optional(), +}); + +const speechEnd = base.extend({ + kind: z.literal("speech_end"), + speaker, + utterance_id: z.string().min(1), + interrupted: z.boolean().optional(), + played_ms: z.number().nonnegative().optional(), +}); + +const transcript = base.extend({ + kind: z.literal("transcript"), + speaker, + utterance_id: z.string().min(1), + final: z.boolean(), + text: z.string().optional(), + message_id: z.string().min(1).optional(), + source: z.enum(["stt", "provider", "playout"]), +}); + +const endOfTurn = base.extend({ + kind: z.literal("end_of_turn"), + utterance_ids: z.array(z.string().min(1)).min(1), +}); + +const responseStart = base.extend({ + kind: z.literal("response_start"), + response_id: z.string().min(1), + span_id: z.string().min(1), + in_reply_to: z.array(z.string().min(1)).optional(), +}); + +const toolCall = base.extend({ + kind: z.literal("tool_call"), + response_id: z.string().min(1), + tool_call_id: z.string().min(1), + name: z.string().min(1), + span_id: z.string().min(1), +}); + +const toolResult = base.extend({ + kind: z.literal("tool_result"), + tool_call_id: z.string().min(1), + span_id: z.string().min(1), + outcome: z.enum(["ok", "error", "cancelled"]), +}); + +const truncate = base.extend({ + kind: z.literal("truncate"), + utterance_id: z.string().min(1), + audio_end_ms: z.number().nonnegative(), +}); + +const missing = base.extend({ + kind: z.literal("missing"), + utterance_id: z.string().min(1).optional(), + evidence: z.enum(["transcript", "speech_end", "audio", "tool_result"]), + reason: z.string().min(1), +}); + +const callEnd = base.extend({ + kind: z.literal("call_end"), + reason: z.string().min(1), +}); + +export const voiceEventSchema = z.discriminatedUnion("kind", [ + speechStart, + speechEnd, + transcript, + endOfTurn, + responseStart, + toolCall, + toolResult, + truncate, + missing, + callEnd, +]); + +export const timelineChunkSchema = z.object({ + call_span_id: z.string().min(1), + chunk_index: z.number().int().nonnegative(), + final: z.boolean(), +}); + +export const timelineRowMetadataSchema = z.object({ + "voice.timeline": timelineChunkSchema, + "voice.events": z.array(voiceEventSchema).min(1).max(256), +}); + +export type VoiceEvent = z.infer; +export type TimelineChunk = z.infer; From 495fe483a6b47922e01ef63108f98396908ec443 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:13:43 +0000 Subject: [PATCH 02/11] feat: resolve call timeline semantics gaps Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 353 ++++++++++-------- .../call-timeline/examples/cascaded.json | 112 ++++-- .../call-timeline/examples/realtime.json | 8 +- .../voice-schema/call-timeline.test.ts | 52 ++- .../livekit/voice-schema/call-timeline.ts | 22 +- 5 files changed, 328 insertions(+), 219 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 1ccb54c10..7092a8f98 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -2,9 +2,19 @@ > AI-created or modified and not human-reviewed in its current form; treat this artifact as provisional. Updated 2026-10-12. -In this option, integrations log what they observe during a call as an ordered, append-only list of timestamped events. Lingua builds speech, turns, latencies and overlaps from that list. The SDK records observations only; it does not decide what a turn or an utterance is. +In this option, integrations log what they observe during a call as an append-only event log. Each event records a speech segment boundary, a transcript, an end-of-turn decision, a generation, a tool call or a playout stop. -Implemented in this PR: the zod schema (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`), the two example traces (`examples/`), and a test that parses them and checks ordering and start/end pairing. Everything else here is a sketch: the LiveKit and Pipecat capture, and the Lingua mapping. The Lingua side is specified in the companion Lingua PR (`specs/voice-schema-call-timeline.md`). +- **The SDK** emits one caller utterance per voice-activity-detection (VAD) speech segment. It never builds turns. +- **Lingua** groups utterances into turns at `end_of_turn` events. It links speech to steps and cuts audio selections. +- **The app** computes latencies and overlaps from the log. + +**Implemented in this PR:** + +- the zod schema (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`) +- the two example traces (`examples/`) +- a test that parses them and checks versioning, `seq`, start/end pairing and utterance references + +**Sketched only:** LiveKit and Pipecat capture, and the Lingua mapping. The Lingua side is in the companion Lingua PR (`specs/voice-schema-call-timeline.md`). ## Customer problem @@ -16,163 +26,190 @@ A customer debugging a voice agent needs to answer five questions: - Who talked over whom? - Which tool call caused a pause? -Today the evidence is scattered. It sits in span names (`user_turn`, `livekit.agent_speaking`), vendor keys (`contrib.livekit.*`, `contrib.pipecat.*`), span start times on different clocks, and selections that exist only after upload. Realtime pipelines make this worse: transcripts arrive late or never, and the provider's view of an interruption differs from what the caller heard. +Today that evidence is scattered. It sits in span names (`user_turn`, `livekit.agent_speaking`), in vendor keys (`contrib.livekit.*`, `contrib.pipecat.*`), in span times on different clocks, and in selections that exist only after upload. Realtime pipelines add transcripts that arrive late or never, and an interruption point reported by the provider that differs from what the caller heard. ## Contract -### Where it lives +### Call span -The integration buffers events and flushes them as **timeline rows**. These are child spans of the call span (the span that logs `audio.recordings`). +The call span is the span that logs `audio.recordings`, or the session span when there is no recording. -| Field | Location | Type | Req. | Meaning | -| ------------------------------- | ---------------------- | --------------------- | ---- | --------------------------------------------------------------------------------- | -| span name | `span_attributes.name` | `"voice.timeline"` | yes | Stable name; one row per chunk | -| span type | `span_attributes.type` | `"task"` | yes | | -| parent | `span_parents[0]` | span id | yes | The call span | -| `voice.timeline.call_span_id` | `metadata` | string | yes | Span id of the call span (repeated so Lingua can index chunks from headers alone) | -| `voice.timeline.chunk_index` | `metadata` | int ≥ 0 | yes | 0, 1, 2… per call, no gaps | -| `voice.timeline.final` | `metadata` | bool | yes | `true` only on the last chunk, written at call end | -| `voice.events` | `metadata` | `VoiceEvent[]`, 1–256 | yes | Events with consecutive `seq` | -| `metrics.start` / `metrics.end` | row | unix s | yes | First and last event's wall time; used only for ordering spans in the trace tree | +| Metadata key | Type | Req. | Meaning | +| ------------------------------- | -------------------------------------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `voice.timeline_version` | `1` | yes | Marks the call as using this contract. Lingua builds timeline speech only when it is present; otherwise it uses the #486 turn-span path. | +| `voice.pipeline` | `"cascaded" \| "realtime" \| "half_cascade"` | yes | Pipeline shape, set at session start. `half_cascade` is a text realtime model plus TTS. | +| `voice.timeline_origin_unix_ms` | number | only without a recording | Origin of the call clock when `audio.recordings` is absent. | -**Flush rule:** +### Timeline rows -- Flush when 64 events are buffered, when the oldest buffered event is 5 s old, or at call end. -- A chunk is never rewritten. Late information becomes a new event in a later chunk. +The integration buffers events and flushes them as **timeline rows**: child spans of the call span. -**Why child rows, not the call span:** +| Field | Location | Type | Req. | Meaning | +| ---------------- | ------------------------------- | ---------------------------- | ---- | ---------------------------------------------------------------------------------- | +| name, type | `span_attributes` | `"voice.timeline"`, `"task"` | yes | One row per chunk | +| parent | `span_parents[0]` | span id | yes | The call span | +| `voice.timeline` | `metadata` | object | yes | `{ call_span_id, chunk_index, final }`, described below | +| `voice.events` | `metadata` | `VoiceEvent[]`, 1–256 | yes | Events with consecutive `seq` | +| start, end | `metrics.start` / `metrics.end` | unix s | yes | Min and max event time in the chunk. Used only to place the row in the trace tree. | -- Rewriting one growing array on the call span would make each merge O(n). -- It would also hit row-size limits on long calls. -- It would also hide the timeline until the call ends. +The `voice.timeline` object: -**Why `metadata`, not `input`/`output`:** Lingua builds `voice_calls` from span headers, without fetching payloads (`TRAJECTORY_METADATA_FIELDS`). A 64-event chunk is about 15–25 KB with transcripts and about 8 KB without. +- `call_span_id` (string): span id of the call span, repeated so Lingua can index chunks from headers. +- `chunk_index` (int ≥ 0): 0, 1, 2… per call. +- `final` (bool): `true` only on the chunk written at call end. -### `VoiceEvent` +**Flush and retry:** -Common fields: +- **Flush:** at 64 buffered events, when the oldest buffered event is 5 s old, or at call end. +- **Backlog:** if a flush is still in flight, the buffer keeps growing up to 256 events and the next chunk carries the backlog. Past 256, the SDK drops new events and logs a `missing{evidence: "timeline"}` event in the next chunk. +- **Retry:** a retried chunk reuses its `chunk_index` and `seq` values, so Lingua drops identical duplicates. +- **Immutability:** chunks are never rewritten, and late facts become new events. -| Field | Type | Req. | Meaning | -| ------- | ---------------------------------- | ---- | ------------------------------------------------------------------------------------------------------- | -| `seq` | int ≥ 0 | yes | Per-call sequence, consecutive across chunks. Defines order when `at_ms` ties; a gap means a lost chunk | -| `at_ms` | number | yes | **Call ms**: milliseconds since `audio.recordings[*].timeline.origin_unix_ms` | -| `basis` | `"audio" \| "local" \| "provider"` | yes | How `at_ms` was obtained (see Timing) | -| `kind` | enum below | yes | | +**Placement:** -Kinds: +- **Child rows, not the call span:** a growing array on the call span would cost O(n) per merge, hit row limits on long calls, and hide the log until the call ends. +- **`metadata`, not `input`:** Lingua builds `voice_calls` from headers without fetching payloads. A 64-event chunk is about 15–25 KB with transcripts and about 8 KB without. -| `kind` | Extra fields (req. unless `?`) | Emitted when | -| ---------------- | -------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | -| `speech_start` | `speaker`, `utterance_id`, `response_id?` | Caller speech detected (VAD), or agent audio starts playing out | -| `speech_end` | `speaker`, `utterance_id`, `interrupted?`, `played_ms?` | Caller speech stops, or agent playout stops (finished or cut) | -| `transcript` | `speaker`, `utterance_id`, `final`, `source: stt\|provider\|playout`, `text?`, `message_id?` | STT or provider transcription arrives; for the agent, the text actually played | -| `end_of_turn` | `utterance_ids[]` | The turn detector commits the caller's turn | -| `response_start` | `response_id`, `span_id`, `in_reply_to?[]` | An LLM or realtime generation starts | -| `tool_call` | `response_id`, `tool_call_id`, `name`, `span_id` | A tool starts running | -| `tool_result` | `tool_call_id`, `span_id`, `outcome: ok\|error\|cancelled` | The tool finishes, fails, or is cancelled | -| `truncate` | `utterance_id`, `audio_end_ms` | Realtime provider truncation of an agent item; `audio_end_ms` is the item-relative position reported to the provider | -| `missing` | `evidence: transcript\|speech_end\|audio\|tool_result`, `reason`, `utterance_id?` | The SDK knows that expected evidence will never arrive | -| `call_end` | `reason` | Session closed; must be the last event | +### `VoiceEvent` -`text` is omitted when content capture is off. Every other field is timing or identity and is always logged. +| Common field | Type | Req. | Meaning | +| ------------ | ---------------------------------- | ---- | -------------------------------------------------------------------------------------------- | +| `seq` | int ≥ 0 | yes | Emission order, consecutive per call across chunks. A gap means a lost or unflushed chunk. | +| `at_ms` | number | yes | When the thing happened, in **call ms** (see Timing). Not necessarily increasing with `seq`. | +| `basis` | `"audio" \| "local" \| "provider"` | yes | How `at_ms` was obtained | +| `kind` | enum below | yes | | + +| `kind` | Extra fields (`?` = optional) | Meaning and timing | +| ---------------- | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `speech_start` | `speaker`, `utterance_id`, `response_id?` | Start of one VAD speech segment (caller) or of playout (agent) | +| `speech_end` | `speaker`, `utterance_id`, `interrupted?`, `played_ms?` | Real end of that segment: for the caller, the VAD stop time minus the VAD silence delay; for the agent, the playout stop. `played_ms` is audio actually played. | +| `transcript` | `speaker`, `utterance_id`, `final`, `source`, `text?`, `message_id?` | Text for one utterance. `at_ms` is when the text arrived. `source` is `stt`, `provider` or `playout`. | +| `message` | `speaker`, `message_id`, `utterance_ids[]` | The framework committed a chat item made from these utterances | +| `end_of_turn` | `utterance_ids[]` | The turn detector committed the caller's turn; lists the caller utterances since the previous `end_of_turn` | +| `response_start` | `response_id`, `span_id`, `in_reply_to?[]` | An LLM or realtime generation started. `span_id` is its `llm_response` span. | +| `tool_call` | `response_id`, `tool_call_id`, `name`, `span_id` | A tool started | +| `tool_result` | `tool_call_id`, `span_id`, `outcome` | The tool finished. `outcome` is `ok`, `error` or `cancelled`. | +| `truncate` | `utterance_id`, `audio_end_ms` | Marker: the SDK told a realtime provider the agent item was cut at `audio_end_ms`, relative to the item. It does not end speech. | +| `missing` | `evidence`, `reason`, `utterance_id?` | Expected evidence will never arrive. `evidence` is `transcript`, `speech_end`, `audio`, `tool_result` or `timeline`. | +| `call_end` | `reason` | Session closed. This is the last event emitted. | + +`text` is omitted when content capture is off. All other fields carry timing or identity and are always logged. ## IDs and relationships -- **`utterance_id`:** one contiguous stretch of speech by one speaker, unique per call. It is the join key for `speech_start`, `speech_end`, `transcript`, `truncate` and `missing`. - - Caller: the provider item id when one exists (realtime), otherwise an SDK counter `u`. - - Agent: the realtime item id, or the LiveKit `speech_id` or Pipecat TTS `context_id` with a counter suffix if one handle plays several stretches. -- **`response_id`:** one model generation. - - Realtime uses the provider response id; LiveKit cascaded uses `generation_id`; Pipecat uses an SDK counter. - - `response_start.span_id` is the Braintrust span id of the `llm_response` span. This is the only link from the timeline into the step tree. -- **`tool_call_id`:** the provider tool call id. `span_id` on `tool_call` and `tool_result` is the tool span. -- **`in_reply_to`:** the caller utterances whose turn triggered this generation, when the framework knows them. LiveKit knows the committed user message for both pipelines; Pipecat knows the aggregated user turn. -- **Agent speech to a generation:** `speech_start.response_id` ties agent speech to the generation that produced it. -- **Messages:** `message_id` on `transcript` links to the framework chat item (LiveKit `ChatMessage.id`, realtime item id). Lingua can use it to match conversation messages; it is optional. -- **Turns are not emitted.** A turn is inferred by Lingua: one `end_of_turn`, the responses that reply to it, and the agent speech they produce. +**`utterance_id`: one speaker, one contiguous speech segment, unique per call.** + +- **Caller, realtime with item ids (OpenAI):** the provider input item id. +- **Caller, Gemini Live and all cascaded pipelines:** an SDK counter `u`. The provider id, if any, arrives later through `message`. +- **Agent, realtime:** the output item id. +- **Agent, LiveKit cascaded:** `/`, with `n` counting playout segments of one speech handle. +- **Agent, Pipecat:** `/`. + +**A caller turn is one or more utterances.** `end_of_turn.utterance_ids` lists them. The SDK does not merge VAD segments. + +**Transcript to utterance, when the provider gives no item id** (cascaded STT, Gemini Live): + +- Each final transcript attaches to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. +- Several finals for one utterance are joined in `seq` order. +- A final that arrives after `end_of_turn` attaches to the last utterance of that turn. + +**`message_id`:** + +- Set on `transcript` when the provider supplies the item id with the text (OpenAI realtime). +- Otherwise a `message` event links it when the chat item is committed: LiveKit `_conversationItemAdded`, or the Pipecat context aggregator push. +- Lingua uses it to match conversation messages. + +**`response_id`:** + +- LiveKit cascaded: `generation_id`. +- Realtime: the provider response id. +- Pipecat: an SDK counter. +- `response_start.span_id` is the only link from the timeline to the step tree. + +**Agent speech to its generation:** `speech_start.response_id`. + +**`in_reply_to`:** an optional hint naming the caller utterances the framework handed to this generation. Lingua's turn windows decide linking, and Lingua warns when the hint disagrees. ## Timing semantics -- **One clock.** `at_ms` is on the call clock, the same one the recording uses: `origin_unix_ms` comes from `snapshot().origin` in `@braintrust/audio`. A call ms `t` maps to segment `s` at offset `t - s.timeline.recording_start_offset_ms` when `s` is ready and covers `t`. +- **Order versus time:** + - `seq` is emission order and defines log order. + - `at_ms` is when the thing happened. For example, a VAD `speech_end` is emitted after the silence delay but dated at the real end, so it can be smaller than an earlier event's `at_ms`. + - Lingua orders speech by `at_ms` and uses `seq` only to break ties and to define turn windows. +- **Call clock:** `at_ms` is milliseconds since the recordings' `timeline.origin_unix_ms`, which is `snapshot().origin` in `@braintrust/audio`. Call ms `t` falls in a ready segment `s` at offset `t - s.timeline.recording_start_offset_ms`. - **`basis` values:** - - `audio`: derived from sample positions the recorder placed on the timeline (the caller frame anchor plus samples, or playout ranges from `playbackProgressed`). This is the best value and matches the recording by construction. - - `local`: `Date.now() - origin` when a hook fires. Callbacks are delayed by event-loop latency, typically under 20 ms. - - `provider`: a provider-reported time converted to the call clock. For example, OpenAI `audio_start_ms`/`audio_end_ms` are relative to the input audio buffer the SDK streamed; the SDK adds the call ms at which that buffer began. Use it when the provider's VAD defines the boundary. -- **Delay meanings:** - - `played_ms` on an agent `speech_end` is audio actually played out. - - `truncate.audio_end_ms` is what the provider was told. The two can differ by the output buffer: in the realtime example, 2600 vs 2900. -- **No timeline without a recording.** If no recording exists, there is no origin. Integrations still emit the timeline using `origin = session start Date.now()` and log `voice.timeline_origin_unix_ms` on the call span; without a recording, Lingua treats `audio` basis as `local`. + - `audio`: computed from sample positions the recorder placed on the timeline, such as playout ranges from `playbackProgressed`. It matches the recording by construction. + - `local`: `Date.now() - origin`, minus a known framework delay where one applies, such as the VAD silence duration. Expect event-loop jitter under 20 ms. + - `provider`: a provider time converted to the call clock. The SDK records the call ms of the first audio frame it sends after each input-buffer clear or commit as the buffer's anchor, so `at_ms = anchor + audio_start_ms` (or `audio_end_ms`). +- **Agent speech:** it ends at `speech_end`, the played end. `truncate` only records what the provider was told. In the realtime example they differ by 300 ms (2600 vs 2900). +- **No recording:** without one, `voice.timeline_origin_unix_ms` is the origin, and Lingua treats `audio` basis as `local`. ## What integrations capture, and when ### LiveKit (JS, `js/src/instrumentation/livekit`) -| Event | Cascaded (`AgentActivity._pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | -| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| user `speech_start` / `speech_end` | `EndOfTurnInfo.startedSpeakingAt` / `stoppedSpeakingAt` in `onEndOfTurn` (`local`); refined to `audio` from capture anchors when audio capture is on | `input_speech_started` / `input_speech_stopped` (`local`), or provider `audio_start_ms`/`audio_end_ms` (`provider`) when exposed | -| user `transcript` | `user_input_transcribed` (final only), `source: stt` | `input_audio_transcription_completed`, `source: provider`; may arrive after `response_start` | -| `end_of_turn` | `onEndOfTurn` hook | `input_audio_buffer.committed` / `userTurnCompleted` | -| `response_start` | `llm_response` span start (`processor.ts` `onStart`), `response_id = generation_id` | realtime `llm_response` start, `response_id = contrib.livekit.response_id` | -| `tool_call` / `tool_result` | `function_tool` span start / end; `outcome` from `function_tool.is_error`, or `cancelled` when the speech handle was interrupted before the result | same | -| agent `speech_start` / `speech_end` | `playback.ts`: first `playbackProgressed.startedAt` / `playbackFinished` (`audio`), `interrupted`, `played_ms = playbackPosition` | same | -| agent `transcript` | `_conversationItemAdded` assistant item, `source: playout` (synchronized transcript) | same, `source: provider` | -| `truncate` | — | `RealtimeSession.truncate` call (hook) | -| `missing` | at close: user utterances with no final transcript | same; common for barge-ins | -| `call_end` | `closeImpl` | same | - -Starting point: add a `Timeline` class beside `turns.ts`. It holds `seq`, a buffer and the flush rule, and is owned by the per-session `Capture`/processor state. `TurnTracker` grouping is no longer needed for emission: the timeline logs raw fragments and Lingua groups them. +| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started` / `input_speech_stopped`, using `audio_start_ms` / `audio_end_ms` when exposed (`provider`) | +| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the rule above | `input_audio_transcription_completed`, `source: provider`, with `message_id = itemId`. It may arrive after `response_start`. | +| `message` | `_conversationItemAdded` (user and assistant items) | Only needed for Gemini Live | +| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit | +| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | +| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | +| agent `speech_start` / `speech_end` | `playback.ts` `settle`: first played range start and last played range end (`audio`), plus `interrupted` and `played_ms` | same | +| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | +| `truncate` | — | the truncate request to the provider | +| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | +| `call_end` | `closeImpl` | same | + +Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, the flush and retry rules, and the transcript-attachment rule. Keep it on the per-session state in `processor.ts`. ### Pipecat (Python, sketch against `sdk-python` `delner/pipecat-voice-recording`) -| Event | Cascaded | Realtime (OpenAI/Gemini realtime services) | -| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | -| user `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`; `audio` basis from input sample clock (`_audio` alignment) | provider `input_audio_buffer.speech_started/stopped` (`provider`) | -| user `transcript` | `TranscriptionFrame` (`finalized`), `source: stt` | `TranscriptionFrame` from the service's input transcription, `source: provider` | -| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits | service commit event | -| `response_start` | `LLMFullResponseStartFrame` (`llm_response` span) | same | -| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | -| agent `speech_start` / `speech_end` | `BotStartedSpeakingFrame` / `BotStoppedSpeakingFrame` at transport output (`audio` from output write observation); `interrupted` when an `InterruptionFrame` preceded the stop | same | -| agent `transcript` | `TTSTextFrame` text accumulated per `context_id`, cut at interruption, `source: playout` | provider transcript, `source: provider` | -| `truncate` | — | service truncation call | - -## What Lingua records versus infers - -**Recorded (copied, normalized):** - -- Events sorted by `seq` into `VoiceCall.events`. -- Recordings, unchanged from #486. +The timeline observer sits at the transport output processor, so agent events reflect audio actually written. -**Inferred:** +| Event | Cascaded | Realtime (OpenAI / Gemini services) | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | +| caller `transcript` | `TranscriptionFrame` (finalized), attached by the rule above, `source: stt` | the service's input transcription, `source: provider` | +| `message` | context aggregator push of the user or assistant message | Gemini only | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | +| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | +| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | -- `Speech` entries: one per `utterance_id` with a `speech_start`. Selections come from `[start, end]` intersected with ready recordings. `interrupted` comes from `speech_end`. -- `Speech.step_ids`: - - Agent: the `response_start.span_id` of its `response_id`. - - Caller: the `span_id` of each `response_start` whose `in_reply_to` contains it. - - Without `in_reply_to`: the first `response_start` after its `end_of_turn`, flagged as inferred. -- Latencies and overlaps (see `VoiceCall.events` consumers below). - -**Not inferred:** +## What Lingua records versus infers -- Text that was never transcribed. -- A `speech_end` that never arrived. Lingua reports a warning and clips the speech at the next event or at `call_end`. +- **Records:** the events in `seq` order in `VoiceCall.events`, and recordings as in #486. +- **Infers** (algorithm in the Lingua spec): + - one `Speech` per utterance, with selections and `interrupted`; + - turns as `end_of_turn` windows; + - `step_ids` from those windows and from `response_id`; + - warnings. +- **Does not infer:** + - **Latencies and overlaps:** the app computes them from `events`; Lingua stores no derived numbers. + - **Untranscribed text:** text that was never transcribed stays missing. ## Trajectory and app mapping -`Trajectory.voice_calls[*]` keeps the #486 fields. `speech` and `recordings` are built from the timeline instead of turn spans, so #22034 and #22036 work unchanged: the voice bar, click-to-step, play-from-message and highlighting all read `speech[].step_ids` and `selections`. +`Trajectory.voice_calls[*]` keeps the #486 fields. `speech` and `recordings` are built from the timeline, so #22034 and #22036 work unchanged: the voice bar, click-to-step, play-from-message and highlighting. -The new field `VoiceCall.events` exposes the normalized log for: +`VoiceCall.events` exposes the log for a latency strip: -- a latency strip showing caller end → `end_of_turn` → `response_start` → first agent `speech_start`; -- overlap shading, from intersecting caller and agent speech; -- "no transcript" badges, from `missing` events. +- caller `speech_end` → `end_of_turn` → `response_start` → first agent `speech_start` in the window +- overlap shading +- `missing` badges +- truncate-versus-played markers -Thread messages still come from the LLM spans' inputs and outputs, as today. +Thread messages still come from LLM span inputs and outputs. ## Worked example -`examples/cascaded.json` and `examples/realtime.json` encode the same fictional call (see the scenario in the comparison note). Row tree: +`examples/cascaded.json` and `examples/realtime.json` encode the same fictional call. Row tree: ``` -livekit.agent_session audio.recordings: call-0000 (13 s) +livekit.agent_session audio.recordings: call-0000 (13 s); voice.timeline_version 1 ├── llm_response llm-1 / rt-1 → find_slots ├── find_slots tool-1 ├── llm_response llm-2 / rt-2 "I have nine thirty or eleven on Friday…" @@ -181,69 +218,57 @@ livekit.agent_session audio.recordings: call-0000 (13 s) ├── reschedule tool-2 ├── llm_response llm-4 / rt-4 "Done — you're booked…" ├── tts tts-2 (cascaded only) -├── voice.timeline timeline-0 seq 0–11 -└── voice.timeline timeline-1 seq 12–22/23, final +├── voice.timeline timeline-0 +└── voice.timeline timeline-1 final ``` Concrete differences, same audio: -| | Cascaded | Realtime | -| ------------------------------------------------ | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| Caller transcript of turn 1 | `transcript` at 3450 (`stt`), before `response_start` | `transcript` at 4300 (`provider`), **after** `tool_call` at 4100 | -| Turn-1 latency (end_of_turn → first agent audio) | 3400 → 5300 = 1900 ms: 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms: 200 ms to response, 500 ms to the tool call, 500 ms tool, 400 ms to audio | -| Barge-in | agent `speech_end` 7900, `interrupted`, `played_ms` 2600 | `truncate` at 7600 (`audio_end_ms` 2600) **and** `speech_end` at 7900 (`played_ms` 2900); a 300 ms disagreement is visible | -| Agent text actually said | `playout` transcript "…on Fri" vs planned text in `llm-2` output | provider transcript of the truncated item; `rt-2` output already holds the truncated text | -| Caller's barge-in text | `stt` transcript at 8600 | **none**: `missing{evidence: transcript}` at 13000; `rt-3` input user content is `""` | -| IDs | `u1`, `a1`, `r1` SDK ids | provider item and response ids (`item_u1`, `resp_1`) | -| Separate STT/TTS steps | `tts-1`, `tts-2` | none | +| | Cascaded | Realtime | +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Caller utterance ids | `u1`, `u2` (SDK) | `item_u1`, `item_u2` (provider) | +| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, carrying `message_id` | +| Turn-1 latency, `end_of_turn` → first agent audio | 3400 → 5300 = 1900 ms: 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms: 200 ms to response, 500 ms to the tool call, 500 ms tool, 400 ms to audio | +| Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `truncate` marker at 7600 (`audio_end_ms` 2600), then `speech_end` 7900 with `played_ms` 2900 | +| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | the `item_u1` transcript arrives mid-response | +| Caller's barge-in text | `stt` "Nine thirty works." | none: `missing{evidence: transcript}`, and the `rt-3` input user content is `""` | +| Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | +| STT/TTS steps | `tts-1`, `tts-2` | none | ## Customer usefulness -- **What was said: good.** - - Final transcripts with provenance. - - Agent text that was actually played, separate from generated text. - - Missing transcripts are explicit. -- **Playing matching audio: good.** - - Every speech interval is on the recording clock. - - Selections are computed by Lingua, so they work across segment boundaries and before upload metadata exists. -- **Diagnosing delays: strongest of the options.** - - Every gap between caller end, end-of-turn, generation, tool and first audio is a subtraction on one clock, including realtime. -- **Interruptions and overlap: strong.** - - Overlap comes from intervals, interruption from `speech_end`. - - The provider-versus-playout truncation gap is visible. -- **Following tools: adequate.** - - The timeline places tools in time and links them to spans. - - Tool arguments and results stay on tool spans. -- **Missing or uncertain evidence: strong.** - - `basis` on every timestamp. - - `missing` events. - - `seq` gaps reveal lost chunks. - - `final` reveals an in-progress call. +- **What was said: good.** Finals carry their source, the agent's played text is kept separate from generated text, and missing transcripts are explicit. +- **Playing matching audio: good.** Every interval is on the recording clock. Lingua cuts selections per segment and per channel. +- **Diagnosing delays: strongest of the options.** Each stage is a subtraction on one clock, in realtime too. +- **Interruptions and overlap: strong.** Overlap comes from intervals and interruption from `speech_end`, and the truncate-versus-played gap is visible. +- **Following tools: adequate.** Tools are placed in time and linked to spans; arguments and results stay on the spans. +- **Missing or uncertain evidence: strong.** Every time has a `basis`, plus `missing` events, `seq` gaps, and a missing `final` chunk. ## Edge cases and weaknesses -- **Barge-in truncation:** handled, as above. The planned text remains only on the LLM span. -- **Overlap:** intervals can overlap freely, and Lingua does not merge them. -- **Late or missing realtime transcripts:** a late transcript is a later event with the same `utterance_id`. Missing ones get `missing`. Weakness: if the provider never reports which item was being transcribed, the SDK must guess the `utterance_id`, as `TurnTracker.transcript` does today. -- **Failed or omitted segments:** speech still has times but no selection in that range. Lingua emits partial selections and the app shows silence. This is better than #486, where a failed segment drops the turn's selections. -- **Unknown clock offset:** `basis: local` events skew relative to `audio` events, and nothing measures that skew. A `clock_offset` event (local vs audio at the same instant) could carry it; this is not specified. -- **Call in progress:** chunks appear every 5 s or 64 events, so the latest ≤5 s are invisible. Lingua warns when no chunk is `final`. -- **Content capture off:** `text` is dropped and everything else stays, so timing analysis still works. -- **Tool cancelled by interruption:** `tool_result{outcome: cancelled}`. The cancellation's cause is not linked; it is inferred from an agent `speech_end{interrupted}` near the same time. -- **Agent handoff:** handoffs are not modeled. Events from a new agent activity continue the same log. A `speaker`-level agent name would be needed to show which agent spoke. This is a weakness. -- **Duplicate observers:** two hooks for the same moment (LiveKit native `user_turn` plus realtime events) must emit one event. The SDK owns dedup, and Lingua does not dedup. -- **Main weakness overall: indirection.** Speech, turns and the step links all depend on Lingua's derivation being right, and the raw spans in the trace tree no longer show speech on their own. Debugging the derivation means reading an event log. Row volume is low (one row per 64 events), but it is a new kind of row in the trace tree. +- **Barge-in truncation:** handled. The planned text stays only on the LLM span. +- **Overlap:** intervals overlap freely. +- **Late or missing realtime transcripts:** a late final attaches by `message_id`, or by the attachment rule when there is none. A missing one gets a `missing` event. Weakness: without item ids (Gemini Live, cascaded STT), attachment is a heuristic and can pick the wrong segment of a multi-segment turn. +- **Failed or omitted segments:** speech keeps its times but has no selection over the gap. +- **Unknown clock offset:** `local` events skew against `audio` events unmeasured. A `clock_offset` event is left as an open question. +- **Call in progress, or a crash:** the latest ≤5 s are not yet flushed. With no `final` chunk, Lingua warns and closes open speech. +- **Content capture off:** `text` is dropped, but timing analysis still works. +- **Tool cancelled by interruption:** `tool_result{outcome: cancelled}`. The cause is inferred only from a nearby interrupted agent `speech_end`. +- **Agent handoff:** not modeled. A new agent activity continues the same log, and nothing records which agent spoke. +- **Duplicate observers:** the SDK must emit one event per moment, such as LiveKit's native `user_turn` plus realtime events. Lingua deduplicates only identical retried chunks. +- **Overall weakness, indirection:** speech, turns and step links depend on Lingua's derivation, and the trace tree alone no longer shows speech. ## Implementation starting point -1. Lock the event schema from `call-timeline.ts`, including `basis` and the flush rule. -2. LiveKit: add `timeline.ts` (buffer, `seq`, flush) to the session state in `processor.ts`. Emit from the hooks already present: `onEndOfTurn`, `RealtimeObserver` listeners, `playback.ts` `settle`, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Keep existing spans and selections during the transition. -3. Add `missing` at close: utterances without a final transcript, and starts without ends. -4. Lingua: implement `specs/voice-schema-call-timeline.md`, starting with speech and selections from the timeline, then `step_ids`, then `events` on `VoiceCall`. -5. Pipecat: port `timeline.ts` to the Python `voice` package using the frame table above. +1. Lock the schema in `call-timeline.ts`, including `basis`, flush and retry, and the transcript-attachment rule. +2. LiveKit: add `timeline.ts` and the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`). Emit from `onEndOfTurn`, the `RealtimeObserver` listeners, `playback.ts` `settle`, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. +3. At close, emit `missing` events, then `call_end`, and flush the `final` chunk. +4. Lingua: implement `specs/voice-schema-call-timeline.md`. +5. Pipecat: port `timeline.ts` to the Python voice package following the table above. ## Open questions -- Should `end_of_turn` carry the detector's delay and probability (LiveKit `eou.*`), or should those stay on the native `eou` spans? -- Are 5 s and 64 events the right flush defaults for live viewing versus row count? -- Should agent `transcript` events carry word-level timings when the TTS provides alignments? +- Should `end_of_turn` carry the detector's delay and probability (LiveKit `eou.*`)? +- Are 5 s and 64 events the right flush defaults? +- Should agent `transcript` carry word timings when the TTS provides alignments? +- Is a `clock_offset` event worth adding, to measure `local` against `audio`? diff --git a/docs/voice-schema/call-timeline/examples/cascaded.json b/docs/voice-schema/call-timeline/examples/cascaded.json index 231f91e63..55868b57c 100644 --- a/docs/voice-schema/call-timeline/examples/cascaded.json +++ b/docs/voice-schema/call-timeline/examples/cascaded.json @@ -54,7 +54,7 @@ } ], "voice.timeline_version": 1, - "pipeline": "cascaded" + "voice.pipeline": "cascaded" } }, { @@ -384,7 +384,7 @@ { "seq": 0, "at_ms": 300, - "basis": "audio", + "basis": "local", "kind": "speech_start", "speaker": "user", "utterance_id": "u1" @@ -392,7 +392,7 @@ { "seq": 1, "at_ms": 2900, - "basis": "audio", + "basis": "local", "kind": "speech_end", "speaker": "user", "utterance_id": "u1" @@ -413,20 +413,28 @@ "utterance_id": "u1", "final": true, "text": "Hi, can I move my Thursday cleaning to Friday morning?", - "message_id": "item_u1", "source": "stt" }, { "seq": 4, "at_ms": 3450, "basis": "local", + "kind": "message", + "speaker": "user", + "message_id": "item_u1", + "utterance_ids": ["u1"] + }, + { + "seq": 5, + "at_ms": 3450, + "basis": "local", "kind": "response_start", "response_id": "r1", "span_id": "llm-1", "in_reply_to": ["u1"] }, { - "seq": 5, + "seq": 6, "at_ms": 4100, "basis": "local", "kind": "tool_call", @@ -436,7 +444,7 @@ "span_id": "tool-1" }, { - "seq": 6, + "seq": 7, "at_ms": 4600, "basis": "local", "kind": "tool_result", @@ -445,7 +453,7 @@ "outcome": "ok" }, { - "seq": 7, + "seq": 8, "at_ms": 4600, "basis": "local", "kind": "response_start", @@ -454,43 +462,51 @@ "in_reply_to": ["u1"] }, { - "seq": 8, + "seq": 9, "at_ms": 5300, "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "a1", + "utterance_id": "speech_1/0", "response_id": "r2" }, { - "seq": 9, + "seq": 10, "at_ms": 7600, - "basis": "audio", + "basis": "local", "kind": "speech_start", "speaker": "user", "utterance_id": "u2" }, { - "seq": 10, + "seq": 11, "at_ms": 7900, "basis": "audio", "kind": "speech_end", "speaker": "agent", - "utterance_id": "a1", + "utterance_id": "speech_1/0", "interrupted": true, "played_ms": 2600 }, { - "seq": 11, + "seq": 12, "at_ms": 7900, "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "a1", + "utterance_id": "speech_1/0", "final": true, "text": "I have nine thirty or eleven on Fri", - "message_id": "item_a1", "source": "playout" + }, + { + "seq": 13, + "at_ms": 7900, + "basis": "local", + "kind": "message", + "speaker": "agent", + "message_id": "item_a1", + "utterance_ids": ["speech_1/0"] } ] } @@ -516,34 +532,42 @@ }, "voice.events": [ { - "seq": 12, - "at_ms": 8500, - "basis": "audio", - "kind": "speech_end", - "speaker": "user", - "utterance_id": "u2" - }, - { - "seq": 13, - "at_ms": 8600, + "seq": 14, + "at_ms": 8520, "basis": "local", "kind": "transcript", "speaker": "user", "utterance_id": "u2", "final": true, "text": "Nine thirty works.", - "message_id": "item_u2", "source": "stt" }, { - "seq": 14, + "seq": 15, + "at_ms": 8500, + "basis": "local", + "kind": "speech_end", + "speaker": "user", + "utterance_id": "u2" + }, + { + "seq": 16, "at_ms": 8700, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u2"] }, { - "seq": 15, + "seq": 17, + "at_ms": 8700, + "basis": "local", + "kind": "message", + "speaker": "user", + "message_id": "item_u2", + "utterance_ids": ["u2"] + }, + { + "seq": 18, "at_ms": 8700, "basis": "local", "kind": "response_start", @@ -552,7 +576,7 @@ "in_reply_to": ["u2"] }, { - "seq": 16, + "seq": 19, "at_ms": 9000, "basis": "local", "kind": "tool_call", @@ -562,7 +586,7 @@ "span_id": "tool-2" }, { - "seq": 17, + "seq": 20, "at_ms": 9400, "basis": "local", "kind": "tool_result", @@ -571,7 +595,7 @@ "outcome": "ok" }, { - "seq": 18, + "seq": 21, "at_ms": 9400, "basis": "local", "kind": "response_start", @@ -580,38 +604,46 @@ "in_reply_to": ["u2"] }, { - "seq": 19, + "seq": 22, "at_ms": 9900, "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "a2", + "utterance_id": "speech_2/0", "response_id": "r4" }, { - "seq": 20, + "seq": 23, "at_ms": 12600, "basis": "audio", "kind": "speech_end", "speaker": "agent", - "utterance_id": "a2", + "utterance_id": "speech_2/0", "interrupted": false, "played_ms": 2700 }, { - "seq": 21, + "seq": 24, "at_ms": 12600, "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "a2", + "utterance_id": "speech_2/0", "final": true, "text": "Done — you're booked Friday at nine thirty.", - "message_id": "item_a2", "source": "playout" }, { - "seq": 22, + "seq": 25, + "at_ms": 12600, + "basis": "local", + "kind": "message", + "speaker": "agent", + "message_id": "item_a2", + "utterance_ids": ["speech_2/0"] + }, + { + "seq": 26, "at_ms": 13000, "basis": "local", "kind": "call_end", diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json index f7c21164a..ce869ad1a 100644 --- a/docs/voice-schema/call-timeline/examples/realtime.json +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -54,7 +54,7 @@ } ], "voice.timeline_version": 1, - "pipeline": "realtime" + "voice.pipeline": "realtime" } }, { @@ -437,7 +437,7 @@ { "seq": 10, "at_ms": 7600, - "basis": "provider", + "basis": "local", "kind": "truncate", "utterance_id": "item_a1", "audio_end_ms": 2600 @@ -485,7 +485,7 @@ "final": true, "text": "I have nine thirty or eleven on Fri", "message_id": "item_a1", - "source": "provider" + "source": "playout" }, { "seq": 13, @@ -568,7 +568,7 @@ "final": true, "text": "Done — you're booked Friday at nine thirty.", "message_id": "item_a2", - "source": "provider" + "source": "playout" }, { "seq": 22, diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts index 3fb85e7d2..b8b41057b 100644 --- a/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts @@ -1,18 +1,29 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { timelineRowMetadataSchema, type VoiceEvent } from "./call-timeline"; +import { + callSpanMetadataSchema, + timelineRowMetadataSchema, + type VoiceEvent, +} from "./call-timeline"; const examples = path.resolve( __dirname, "../../../../../docs/voice-schema/call-timeline/examples", ); +type Row = { + span_id: string; + span_parents: string[]; + metadata?: Record; +}; + +function rows(file: string): Row[] { + return JSON.parse(readFileSync(path.join(examples, file), "utf8")); +} + function events(file: string): VoiceEvent[] { - const rows: { metadata?: Record }[] = JSON.parse( - readFileSync(path.join(examples, file), "utf8"), - ); - return rows + return rows(file) .filter((row) => row.metadata?.["voice.timeline"] !== undefined) .map((row) => timelineRowMetadataSchema.parse(row.metadata)) .sort( @@ -23,21 +34,42 @@ function events(file: string): VoiceEvent[] { } describe.each(["cascaded.json", "realtime.json"])("%s", (file) => { - it("parses as a gapless, time-ordered event log", () => { + it("versions the call span", () => { + const call = rows(file).find((row) => row.span_parents.length === 0); + expect(callSpanMetadataSchema.parse(call?.metadata)).toMatchObject({ + "voice.timeline_version": 1, + }); + }); + + it("numbers events in emission order and ends with call_end", () => { const log = events(file); expect(log.map((event) => event.seq)).toEqual(log.map((_, i) => i)); - for (let i = 1; i < log.length; i++) - expect(log[i].at_ms).toBeGreaterThanOrEqual(log[i - 1].at_ms); expect(log.at(-1)?.kind).toBe("call_end"); }); it("pairs every speech start with an end", () => { - const log = events(file); const open = new Set(); - for (const event of log) { + for (const event of events(file)) { if (event.kind === "speech_start") open.add(event.utterance_id); if (event.kind === "speech_end") open.delete(event.utterance_id); } expect([...open]).toEqual([]); }); + + it("refers only to utterances that started", () => { + const log = events(file); + const started = new Set( + log.flatMap((event) => + event.kind === "speech_start" ? [event.utterance_id] : [], + ), + ); + const referenced = log.flatMap((event) => + event.kind === "end_of_turn" || event.kind === "message" + ? event.utterance_ids + : "utterance_id" in event && event.utterance_id !== undefined + ? [event.utterance_id] + : [], + ); + expect(referenced.filter((id) => !started.has(id))).toEqual([]); + }); }); diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts index 1699aae91..5336a0738 100644 --- a/js/src/instrumentation/livekit/voice-schema/call-timeline.ts +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts @@ -33,6 +33,13 @@ const transcript = base.extend({ source: z.enum(["stt", "provider", "playout"]), }); +const message = base.extend({ + kind: z.literal("message"), + speaker, + message_id: z.string().min(1), + utterance_ids: z.array(z.string().min(1)).min(1), +}); + const endOfTurn = base.extend({ kind: z.literal("end_of_turn"), utterance_ids: z.array(z.string().min(1)).min(1), @@ -69,7 +76,13 @@ const truncate = base.extend({ const missing = base.extend({ kind: z.literal("missing"), utterance_id: z.string().min(1).optional(), - evidence: z.enum(["transcript", "speech_end", "audio", "tool_result"]), + evidence: z.enum([ + "transcript", + "speech_end", + "audio", + "tool_result", + "timeline", + ]), reason: z.string().min(1), }); @@ -82,6 +95,7 @@ export const voiceEventSchema = z.discriminatedUnion("kind", [ speechStart, speechEnd, transcript, + message, endOfTurn, responseStart, toolCall, @@ -97,6 +111,12 @@ export const timelineChunkSchema = z.object({ final: z.boolean(), }); +export const callSpanMetadataSchema = z.object({ + "voice.timeline_version": z.literal(1), + "voice.pipeline": z.enum(["cascaded", "realtime", "half_cascade"]), + "voice.timeline_origin_unix_ms": z.number().optional(), +}); + export const timelineRowMetadataSchema = z.object({ "voice.timeline": timelineChunkSchema, "voice.events": z.array(voiceEventSchema).min(1).max(256), From cfdd497667b73f4a3b452184387185415d9b6927 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:27:39 +0000 Subject: [PATCH 03/11] chore: specify realtime caller timing for call timeline Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 89 ++++++++------- .../call-timeline/examples/realtime.json | 101 ++++++++++-------- .../livekit/voice-schema/call-timeline.ts | 1 + 3 files changed, 106 insertions(+), 85 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 7092a8f98..e408b2b1b 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -34,11 +34,12 @@ Today that evidence is scattered. It sits in span names (`user_turn`, `livekit.a The call span is the span that logs `audio.recordings`, or the session span when there is no recording. -| Metadata key | Type | Req. | Meaning | -| ------------------------------- | -------------------------------------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `voice.timeline_version` | `1` | yes | Marks the call as using this contract. Lingua builds timeline speech only when it is present; otherwise it uses the #486 turn-span path. | -| `voice.pipeline` | `"cascaded" \| "realtime" \| "half_cascade"` | yes | Pipeline shape, set at session start. `half_cascade` is a text realtime model plus TTS. | -| `voice.timeline_origin_unix_ms` | number | only without a recording | Origin of the call clock when `audio.recordings` is absent. | +| Metadata key | Type | Req. | Meaning | +| ------------------------------- | -------------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `voice.timeline_version` | `1` | yes | Marks the call as using this contract. Lingua builds timeline speech only when it is present; otherwise it uses the #486 turn-span path. | +| `voice.pipeline` | `"cascaded" \| "realtime" \| "half_cascade"` | yes | Pipeline shape, set at session start. `half_cascade` is a text-output realtime model plus TTS: caller events follow the realtime rules, and agent events follow cascaded playout. | +| `voice.timeline_origin_unix_ms` | number | only without a recording | Origin of the call clock when `audio.recordings` is absent. | +| `voice.server_vad_silence_ms` | number | no | Realtime server-VAD silence that the SDK subtracted from caller `speech_end` receipt times | ### Timeline rows @@ -99,15 +100,14 @@ The `voice.timeline` object: **`utterance_id`: one speaker, one contiguous speech segment, unique per call.** -- **Caller, realtime with item ids (OpenAI):** the provider input item id. -- **Caller, Gemini Live and all cascaded pipelines:** an SDK counter `u`. The provider id, if any, arrives later through `message`. +- **Caller, all pipelines:** an SDK counter `u`, assigned at `speech_start`. Realtime speech-start and speech-stop events carry no item id (LiveKit `input_speech_started` / `input_speech_stopped`), so the provider input item id arrives later through `message`. - **Agent, realtime:** the output item id. - **Agent, LiveKit cascaded:** `/`, with `n` counting playout segments of one speech handle. -- **Agent, Pipecat:** `/`. +- **Agent, Pipecat:** `/`, where `n` counts playout stretches within the TTS context: it increments when output resumes after an interruption or a gap. **A caller turn is one or more utterances.** `end_of_turn.utterance_ids` lists them. The SDK does not merge VAD segments. -**Transcript to utterance, when the provider gives no item id** (cascaded STT, Gemini Live): +**Transcript to utterance**, for every pipeline, because no caller speech event carries an item id: - Each final transcript attaches to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. - Several finals for one utterance are joined in `seq` order. @@ -115,8 +115,8 @@ The `voice.timeline` object: **`message_id`:** -- Set on `transcript` when the provider supplies the item id with the text (OpenAI realtime). -- Otherwise a `message` event links it when the chat item is committed: LiveKit `_conversationItemAdded`, or the Pipecat context aggregator push. +- **Caller:** a `message` event maps the utterance to the chat item. In cascaded pipelines it fires when the item commits (LiveKit `_conversationItemAdded`, Pipecat context aggregator push). In realtime it fires when `input_audio_transcription_completed` arrives with `itemId`. +- **Agent, realtime:** set on `transcript`, since the output item id is the `utterance_id`. - Lingua uses it to match conversation messages. **`response_id`:** @@ -128,7 +128,10 @@ The `voice.timeline` object: **Agent speech to its generation:** `speech_start.response_id`. -**`in_reply_to`:** an optional hint naming the caller utterances the framework handed to this generation. Lingua's turn windows decide linking, and Lingua warns when the hint disagrees. +**`in_reply_to`:** the caller utterances the framework handed to this generation. + +- **Required on preemptive generations,** meaning any `response_start` emitted before its turn's `end_of_turn` (LiveKit preemptive generation knows the user turn it answers). Otherwise it is recommended. +- **Linking:** Lingua uses `in_reply_to` whenever it is present, and falls back to the `end_of_turn` window only when it is absent. ## Timing semantics @@ -140,7 +143,15 @@ The `voice.timeline` object: - **`basis` values:** - `audio`: computed from sample positions the recorder placed on the timeline, such as playout ranges from `playbackProgressed`. It matches the recording by construction. - `local`: `Date.now() - origin`, minus a known framework delay where one applies, such as the VAD silence duration. Expect event-loop jitter under 20 ms. - - `provider`: a provider time converted to the call clock. The SDK records the call ms of the first audio frame it sends after each input-buffer clear or commit as the buffer's anchor, so `at_ms = anchor + audio_start_ms` (or `audio_end_ms`). + - `provider`: a provider time converted to the call clock. Use it only when the anchor is known; otherwise use `local`. + + | Provider | Provider time | Anchor | Result | + | --------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------- | ------------------------------------------ | + | OpenAI Realtime | `audio_start_ms` / `audio_end_ms`, cumulative from the start of all audio written to the input buffer this session | call ms of the first audio frame sent in the session | `at_ms = anchor + audio_*_ms` (`provider`) | + | Gemini Live | none | — | receipt time (`local`) | + + LiveKit's realtime events do not expose these fields today, so LiveKit realtime uses `local`. + - **Agent speech:** it ends at `speech_end`, the played end. `truncate` only records what the provider was told. In the realtime example they differ by 300 ms (2600 vs 2900). - **No recording:** without one, `voice.timeline_origin_unix_ms` is the origin, and Lingua treats `audio` basis as `local`. @@ -148,19 +159,19 @@ The `voice.timeline` object: ### LiveKit (JS, `js/src/instrumentation/livekit`) -| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started` / `input_speech_stopped`, using `audio_start_ms` / `audio_end_ms` when exposed (`provider`) | -| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the rule above | `input_audio_transcription_completed`, `source: provider`, with `message_id = itemId`. It may arrive after `response_start`. | -| `message` | `_conversationItemAdded` (user and assistant items) | Only needed for Gemini Live | -| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit | -| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | -| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | -| agent `speech_start` / `speech_end` | `playback.ts` `settle`: first played range start and last played range end (`audio`), plus `interrupted` and `played_ms` | same | -| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | -| `truncate` | — | the truncate request to the provider | -| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | -| `call_end` | `closeImpl` | same | +| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started`: `u`, at receipt time (`local`). `input_speech_stopped`: receipt time minus the configured server-VAD silence (`turn_detection.silence_duration_ms`, logged as `voice.server_vad_silence_ms`) when known; otherwise receipt time (`local`), which then includes the provider's silence padding. | +| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the rule above | `input_audio_transcription_completed`, `source: provider`, attached by the rule above and followed by `message`. It may arrive after `response_start`. | +| `message` | `_conversationItemAdded` (user and assistant items) | caller: when `input_audio_transcription_completed` arrives (`message_id = itemId`) | +| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit | +| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | +| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | +| agent `speech_start` / `speech_end` | Start: emitted at playout start, the first `playbackStarted` / `playbackProgressed` (`audio`). End: at `playbackFinished`, with the last played range end (`audio`), `interrupted` and `played_ms`. | same | +| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | +| `truncate` | — | the truncate request to the provider | +| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | +| `call_end` | `closeImpl` | same | Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, the flush and retry rules, and the transcript-attachment rule. Keep it on the per-session state in `processor.ts`. @@ -168,16 +179,16 @@ Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, The timeline observer sits at the transport output processor, so agent events reflect audio actually written. -| Event | Cascaded | Realtime (OpenAI / Gemini services) | -| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | -| caller `transcript` | `TranscriptionFrame` (finalized), attached by the rule above, `source: stt` | the service's input transcription, `source: provider` | -| `message` | context aggregator push of the user or assistant message | Gemini only | -| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | -| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | -| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | -| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | -| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | +| Event | Cascaded | Realtime (OpenAI / Gemini services) | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | +| caller `transcript` | `TranscriptionFrame` (finalized), attached by the rule above, `source: stt` | the service's input transcription, `source: provider` | +| `message` | context aggregator push of the user or assistant message | Gemini only | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | +| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | +| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | ## What Lingua records versus infers @@ -226,11 +237,11 @@ Concrete differences, same audio: | | Cascaded | Realtime | | ------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| Caller utterance ids | `u1`, `u2` (SDK) | `item_u1`, `item_u2` (provider) | -| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, carrying `message_id` | +| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 4300 | +| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, attached to `u1` by the rule | | Turn-1 latency, `end_of_turn` → first agent audio | 3400 → 5300 = 1900 ms: 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms: 200 ms to response, 500 ms to the tool call, 500 ms tool, 400 ms to audio | | Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `truncate` marker at 7600 (`audio_end_ms` 2600), then `speech_end` 7900 with `played_ms` 2900 | -| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | the `item_u1` transcript arrives mid-response | +| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is dated receipt 3400 − 500 ms server-VAD silence = 2900 | | Caller's barge-in text | `stt` "Nine thirty works." | none: `missing{evidence: transcript}`, and the `rt-3` input user content is `""` | | Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | | STT/TTS steps | `tts-1`, `tts-2` | none | diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json index ce869ad1a..75941a728 100644 --- a/docs/voice-schema/call-timeline/examples/realtime.json +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -54,7 +54,8 @@ } ], "voice.timeline_version": 1, - "voice.pipeline": "realtime" + "voice.pipeline": "realtime", + "voice.server_vad_silence_ms": 500 } }, { @@ -336,7 +337,7 @@ }, "metrics": { "start": 1791830000.3, - "end": 1791830007.9 + "end": 1791830007.6 }, "metadata": { "voice.timeline": { @@ -348,25 +349,25 @@ { "seq": 0, "at_ms": 300, - "basis": "provider", + "basis": "local", "kind": "speech_start", "speaker": "user", - "utterance_id": "item_u1" + "utterance_id": "u1" }, { "seq": 1, "at_ms": 2900, - "basis": "provider", + "basis": "local", "kind": "speech_end", "speaker": "user", - "utterance_id": "item_u1" + "utterance_id": "u1" }, { "seq": 2, "at_ms": 3400, - "basis": "provider", + "basis": "local", "kind": "end_of_turn", - "utterance_ids": ["item_u1"] + "utterance_ids": ["u1"] }, { "seq": 3, @@ -375,7 +376,7 @@ "kind": "response_start", "response_id": "resp_1", "span_id": "rt-1", - "in_reply_to": ["item_u1"] + "in_reply_to": ["u1"] }, { "seq": 4, @@ -393,14 +394,22 @@ "basis": "local", "kind": "transcript", "speaker": "user", - "utterance_id": "item_u1", + "utterance_id": "u1", "final": true, "text": "Hi, can I move my Thursday cleaning to Friday morning?", - "message_id": "item_u1", "source": "provider" }, { "seq": 6, + "at_ms": 4300, + "basis": "local", + "kind": "message", + "speaker": "user", + "message_id": "item_u1", + "utterance_ids": ["u1"] + }, + { + "seq": 7, "at_ms": 4600, "basis": "local", "kind": "tool_result", @@ -409,16 +418,16 @@ "outcome": "ok" }, { - "seq": 7, + "seq": 8, "at_ms": 4600, "basis": "local", "kind": "response_start", "response_id": "resp_2", "span_id": "rt-2", - "in_reply_to": ["item_u1"] + "in_reply_to": ["u1"] }, { - "seq": 8, + "seq": 9, "at_ms": 5000, "basis": "audio", "kind": "speech_start", @@ -427,30 +436,20 @@ "response_id": "resp_2" }, { - "seq": 9, + "seq": 10, "at_ms": 7600, - "basis": "provider", + "basis": "local", "kind": "speech_start", "speaker": "user", - "utterance_id": "item_u2" + "utterance_id": "u2" }, { - "seq": 10, + "seq": 11, "at_ms": 7600, "basis": "local", "kind": "truncate", "utterance_id": "item_a1", "audio_end_ms": 2600 - }, - { - "seq": 11, - "at_ms": 7900, - "basis": "audio", - "kind": "speech_end", - "speaker": "agent", - "utterance_id": "item_a1", - "interrupted": true, - "played_ms": 2900 } ] } @@ -478,6 +477,16 @@ { "seq": 12, "at_ms": 7900, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "item_a1", + "interrupted": true, + "played_ms": 2900 + }, + { + "seq": 13, + "at_ms": 7900, "basis": "local", "kind": "transcript", "speaker": "agent", @@ -488,31 +497,31 @@ "source": "playout" }, { - "seq": 13, + "seq": 14, "at_ms": 8500, - "basis": "provider", + "basis": "local", "kind": "speech_end", "speaker": "user", - "utterance_id": "item_u2" + "utterance_id": "u2" }, { - "seq": 14, + "seq": 15, "at_ms": 8600, - "basis": "provider", + "basis": "local", "kind": "end_of_turn", - "utterance_ids": ["item_u2"] + "utterance_ids": ["u2"] }, { - "seq": 15, + "seq": 16, "at_ms": 8600, "basis": "local", "kind": "response_start", "response_id": "resp_3", "span_id": "rt-3", - "in_reply_to": ["item_u2"] + "in_reply_to": ["u2"] }, { - "seq": 16, + "seq": 17, "at_ms": 8900, "basis": "local", "kind": "tool_call", @@ -522,7 +531,7 @@ "span_id": "tool-2" }, { - "seq": 17, + "seq": 18, "at_ms": 9300, "basis": "local", "kind": "tool_result", @@ -531,16 +540,16 @@ "outcome": "ok" }, { - "seq": 18, + "seq": 19, "at_ms": 9300, "basis": "local", "kind": "response_start", "response_id": "resp_4", "span_id": "rt-4", - "in_reply_to": ["item_u2"] + "in_reply_to": ["u2"] }, { - "seq": 19, + "seq": 20, "at_ms": 9700, "basis": "audio", "kind": "speech_start", @@ -549,7 +558,7 @@ "response_id": "resp_4" }, { - "seq": 20, + "seq": 21, "at_ms": 12500, "basis": "audio", "kind": "speech_end", @@ -559,7 +568,7 @@ "played_ms": 2800 }, { - "seq": 21, + "seq": 22, "at_ms": 12500, "basis": "local", "kind": "transcript", @@ -571,16 +580,16 @@ "source": "playout" }, { - "seq": 22, + "seq": 23, "at_ms": 13000, "basis": "local", "kind": "missing", - "utterance_id": "item_u2", + "utterance_id": "u2", "evidence": "transcript", "reason": "input_transcription_not_received" }, { - "seq": 23, + "seq": 24, "at_ms": 13000, "basis": "local", "kind": "call_end", diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts index 5336a0738..91e170a5e 100644 --- a/js/src/instrumentation/livekit/voice-schema/call-timeline.ts +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.ts @@ -115,6 +115,7 @@ export const callSpanMetadataSchema = z.object({ "voice.timeline_version": z.literal(1), "voice.pipeline": z.enum(["cascaded", "realtime", "half_cascade"]), "voice.timeline_origin_unix_ms": z.number().optional(), + "voice.server_vad_silence_ms": z.number().nonnegative().optional(), }); export const timelineRowMetadataSchema = z.object({ From 47062f7869d7e3d7f52d15b9f5edfaf190d38666 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:31:56 +0000 Subject: [PATCH 04/11] chore: specify late realtime transcript attachment Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 35 ++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index e408b2b1b..e25e41f1c 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -102,16 +102,19 @@ The `voice.timeline` object: - **Caller, all pipelines:** an SDK counter `u`, assigned at `speech_start`. Realtime speech-start and speech-stop events carry no item id (LiveKit `input_speech_started` / `input_speech_stopped`), so the provider input item id arrives later through `message`. - **Agent, realtime:** the output item id. -- **Agent, LiveKit cascaded:** `/`, with `n` counting playout segments of one speech handle. +- **Agent, LiveKit cascaded and `half_cascade`:** `/`, with `n` counting playout segments of one speech handle. `half_cascade` uses the cascaded agent scheme because playout comes from TTS. - **Agent, Pipecat:** `/`, where `n` counts playout stretches within the TTS context: it increments when output resumes after an interruption or a gap. **A caller turn is one or more utterances.** `end_of_turn.utterance_ids` lists them. The SDK does not merge VAD segments. -**Transcript to utterance**, for every pipeline, because no caller speech event carries an item id: +**Transcript to utterance.** No caller speech event carries an item id, so the SDK attaches each caller final by one of two rules. -- Each final transcript attaches to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. -- Several finals for one utterance are joined in `seq` order. -- A final that arrives after `end_of_turn` attaches to the last utterance of that turn. +- **STT finals (`source: stt`, cascaded and half-cascade STT):** attach to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. Several finals for one utterance are joined in `seq` order. A final that arrives after `end_of_turn` attaches to the last utterance of that turn. +- **Provider finals (`source: provider`, realtime):** attach FIFO to the oldest committed caller utterance (listed in an `end_of_turn`) that has no final yet. A provider transcribes committed audio in commit order, often after the next utterance has started, so "open at arrival" would pick the wrong one. + - In the realtime example, `u1`'s final arrives at 4300 while no utterance is open, and FIFO picks `u1`. + - If the caller had already barged in, `u2` would be open, yet FIFO would still give the final to `u1`, because `u2` is not committed. +- **`message` event:** follows the same attachment as the final it accompanies. +- **No final:** an utterance that never gets a final never gets a `message`. It gets `missing{evidence: transcript}` at close instead. **`message_id`:** @@ -162,7 +165,7 @@ The `voice.timeline` object: | Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | | ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started`: `u`, at receipt time (`local`). `input_speech_stopped`: receipt time minus the configured server-VAD silence (`turn_detection.silence_duration_ms`, logged as `voice.server_vad_silence_ms`) when known; otherwise receipt time (`local`), which then includes the provider's silence padding. | -| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the rule above | `input_audio_transcription_completed`, `source: provider`, attached by the rule above and followed by `message`. It may arrive after `response_start`. | +| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the STT rule | `input_audio_transcription_completed`, `source: provider`, attached FIFO by the provider rule and followed by `message{message_id: itemId}`. It may arrive after `response_start`. | | `message` | `_conversationItemAdded` (user and assistant items) | caller: when `input_audio_transcription_completed` arrives (`message_id = itemId`) | | `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit | | `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | @@ -179,16 +182,16 @@ Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, The timeline observer sits at the transport output processor, so agent events reflect audio actually written. -| Event | Cascaded | Realtime (OpenAI / Gemini services) | -| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | -| caller `transcript` | `TranscriptionFrame` (finalized), attached by the rule above, `source: stt` | the service's input transcription, `source: provider` | -| `message` | context aggregator push of the user or assistant message | Gemini only | -| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | -| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | -| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | -| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | -| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | +| Event | Cascaded | Realtime (OpenAI / Gemini services) | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | +| caller `transcript` | `TranscriptionFrame` (finalized), attached by the STT rule, `source: stt` | the service's input transcription, attached FIFO by the provider rule, `source: provider` | +| `message` | context aggregator push of the user or assistant message | Gemini only | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | +| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | +| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | ## What Lingua records versus infers From f36c0eeaca7ac097d4b730e3fb7149052db4be02 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:33:22 +0000 Subject: [PATCH 05/11] chore: clarify realtime transcript grouping Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 49 ++++++++++++----------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index e25e41f1c..c1c53efb7 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -111,10 +111,11 @@ The `voice.timeline` object: - **STT finals (`source: stt`, cascaded and half-cascade STT):** attach to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. Several finals for one utterance are joined in `seq` order. A final that arrives after `end_of_turn` attaches to the last utterance of that turn. - **Provider finals (`source: provider`, realtime):** attach FIFO to the oldest committed caller utterance (listed in an `end_of_turn`) that has no final yet. A provider transcribes committed audio in commit order, often after the next utterance has started, so "open at arrival" would pick the wrong one. + - **Assumption:** each committed `end_of_turn` group becomes one provider input item, so one provider final arrives per group. If a group has several utterances, the final attaches to the whole oldest group that lacks a final. Its first utterance gets the text, and the group's other utterances are marked covered, not missing. That way a multi-utterance group never shifts later turns' finals. - In the realtime example, `u1`'s final arrives at 4300 while no utterance is open, and FIFO picks `u1`. - If the caller had already barged in, `u2` would be open, yet FIFO would still give the final to `u1`, because `u2` is not committed. - **`message` event:** follows the same attachment as the final it accompanies. -- **No final:** an utterance that never gets a final never gets a `message`. It gets `missing{evidence: transcript}` at close instead. +- **No final:** an utterance that never gets a final, and is not covered by its group's final, never gets a `message`. It gets `missing{evidence: transcript}` at close instead. **`message_id`:** @@ -162,19 +163,19 @@ The `voice.timeline` object: ### LiveKit (JS, `js/src/instrumentation/livekit`) -| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started`: `u`, at receipt time (`local`). `input_speech_stopped`: receipt time minus the configured server-VAD silence (`turn_detection.silence_duration_ms`, logged as `voice.server_vad_silence_ms`) when known; otherwise receipt time (`local`), which then includes the provider's silence padding. | -| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the STT rule | `input_audio_transcription_completed`, `source: provider`, attached FIFO by the provider rule and followed by `message{message_id: itemId}`. It may arrive after `response_start`. | -| `message` | `_conversationItemAdded` (user and assistant items) | caller: when `input_audio_transcription_completed` arrives (`message_id = itemId`) | -| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit | -| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | -| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | -| agent `speech_start` / `speech_end` | Start: emitted at playout start, the first `playbackStarted` / `playbackProgressed` (`audio`). End: at `playbackFinished`, with the last played range end (`audio`), `interrupted` and `played_ms`. | same | -| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | -| `truncate` | — | the truncate request to the provider | -| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | -| `call_end` | `closeImpl` | same | +| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started`: `u`, at receipt time (`local`). `input_speech_stopped`: receipt time minus the configured server-VAD silence (`turn_detection.silence_duration_ms`, logged as `voice.server_vad_silence_ms`) when known; otherwise receipt time (`local`), which then includes the provider's silence padding. With server turn detection off, caller speech comes from the local VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`, `local`), as in cascaded. | +| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the STT rule | `input_audio_transcription_completed`, `source: provider`, attached FIFO by the provider rule and followed by `message{message_id: itemId}`. It may arrive after `response_start`. | +| `message` | `_conversationItemAdded` (user and assistant items) | caller: when `input_audio_transcription_completed` arrives (`message_id = itemId`) | +| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit; with server turn detection off, LiveKit's own turn commit (`onEndOfTurn`) | +| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | +| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | +| agent `speech_start` / `speech_end` | Start: emitted at playout start, the first `playbackStarted` / `playbackProgressed` (`audio`). End: at `playbackFinished`, with the last played range end (`audio`), `interrupted` and `played_ms`. | same | +| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | +| `truncate` | — | the truncate request to the provider | +| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | +| `call_end` | `closeImpl` | same | Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, the flush and retry rules, and the transcript-attachment rule. Keep it on the per-session state in `processor.ts`. @@ -182,16 +183,16 @@ Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, The timeline observer sits at the transport output processor, so agent events reflect audio actually written. -| Event | Cascaded | Realtime (OpenAI / Gemini services) | -| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | -| caller `transcript` | `TranscriptionFrame` (finalized), attached by the STT rule, `source: stt` | the service's input transcription, attached FIFO by the provider rule, `source: provider` | -| `message` | context aggregator push of the user or assistant message | Gemini only | -| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | -| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | -| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | -| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | -| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | +| Event | Cascaded | Realtime (OpenAI / Gemini services) | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | +| caller `transcript` | `TranscriptionFrame` (finalized), attached by the STT rule, `source: stt` | the service's input transcription, attached FIFO by the provider rule, `source: provider` | +| `message` | context aggregator push of the user or assistant message | realtime: only for providers with input item ids (OpenAI), at transcription completion; none for Gemini Live, which has no item ids | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | +| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | +| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | ## What Lingua records versus infers From 5c42f0a47ff3c071fa9a672e4929a6b20e38bcae Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:50:12 +0000 Subject: [PATCH 06/11] chore: note the implemented lingua mapping Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index c1c53efb7..cb8da3a47 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -14,7 +14,7 @@ In this option, integrations log what they observe during a call as an append-on - the two example traces (`examples/`) - a test that parses them and checks versioning, `seq`, start/end pairing and utterance references -**Sketched only:** LiveKit and Pipecat capture, and the Lingua mapping. The Lingua side is in the companion Lingua PR (`specs/voice-schema-call-timeline.md`). +**Implemented in the companion Lingua PR:** the Lingua mapping, built from `specs/voice-schema-call-timeline.md` with fixtures from these examples. **Sketched only:** LiveKit and Pipecat capture. ## Customer problem @@ -278,7 +278,7 @@ Concrete differences, same audio: 1. Lock the schema in `call-timeline.ts`, including `basis`, flush and retry, and the transcript-attachment rule. 2. LiveKit: add `timeline.ts` and the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`). Emit from `onEndOfTurn`, the `RealtimeObserver` listeners, `playback.ts` `settle`, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. 3. At close, emit `missing` events, then `call_end`, and flush the `final` chunk. -4. Lingua: implement `specs/voice-schema-call-timeline.md`. +4. Lingua: the builder exists in the companion PR; extend it with the app allow-list keys after #486 is rebased. 5. Pipecat: port `timeline.ts` to the Python voice package following the table above. ## Open questions From 4faa8cf9c8f09f249c6ad32a760c227278041bf6 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:57:41 +0000 Subject: [PATCH 07/11] chore: align call timeline capture with livekit 1.9 sources Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 73 ++++++++------- .../call-timeline/examples/cascaded.json | 32 +++---- .../call-timeline/examples/realtime.json | 92 +++++++++++-------- 3 files changed, 108 insertions(+), 89 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index cb8da3a47..6d3a94b5f 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -101,8 +101,7 @@ The `voice.timeline` object: **`utterance_id`: one speaker, one contiguous speech segment, unique per call.** - **Caller, all pipelines:** an SDK counter `u`, assigned at `speech_start`. Realtime speech-start and speech-stop events carry no item id (LiveKit `input_speech_started` / `input_speech_stopped`), so the provider input item id arrives later through `message`. -- **Agent, realtime:** the output item id. -- **Agent, LiveKit cascaded and `half_cascade`:** `/`, with `n` counting playout segments of one speech handle. `half_cascade` uses the cascaded agent scheme because playout comes from TTS. +- **Agent, LiveKit (all pipelines):** `/`, where `n` counts playout stretches of one speech handle. Each tool continuation runs its own reply task with its own handle (for example `speech_2/0` after the `speech_1` tool call). Do not use `lk.speech_id`: it is turn-level and is shared across tool continuations. A realtime output item id is linked later through a `message` event, when LiveKit adds the item to the conversation. - **Agent, Pipecat:** `/`, where `n` counts playout stretches within the TTS context: it increments when output resumes after an interruption or a gap. **A caller turn is one or more utterances.** `end_of_turn.utterance_ids` lists them. The SDK does not merge VAD segments. @@ -120,21 +119,20 @@ The `voice.timeline` object: **`message_id`:** - **Caller:** a `message` event maps the utterance to the chat item. In cascaded pipelines it fires when the item commits (LiveKit `_conversationItemAdded`, Pipecat context aggregator push). In realtime it fires when `input_audio_transcription_completed` arrives with `itemId`. -- **Agent, realtime:** set on `transcript`, since the output item id is the `utterance_id`. +- **Agent:** a `message` event maps `/` to the assistant chat item, which is the realtime output item id in realtime pipelines, when LiveKit adds it (`_conversationItemAdded`). - Lingua uses it to match conversation messages. **`response_id`:** -- LiveKit cascaded: `generation_id`. -- Realtime: the provider response id. +- LiveKit (both pipelines): the `SpeechHandle.id` captured at `_pipelineReplyTaskImpl` / `_realtimeGenerationTaskImpl` entry. The provider response id stays on the `llm_response` span (`contrib.livekit.response_id`). - Pipecat: an SDK counter. -- `response_start.span_id` is the only link from the timeline to the step tree. +- `response_start.span_id` is the `llm_response` span started inside that handle's reply task. If a reply task starts several, it is the latest one started before playback starts. This is the only link from the timeline to the step tree. **Agent speech to its generation:** `speech_start.response_id`. **`in_reply_to`:** the caller utterances the framework handed to this generation. -- **Required on preemptive generations,** meaning any `response_start` emitted before its turn's `end_of_turn` (LiveKit preemptive generation knows the user turn it answers). Otherwise it is recommended. +- **Required on preemptive generations,** meaning any `response_start` emitted before its turn's `end_of_turn`. LiveKit: snapshot the uncommitted caller utterance ids in `onPreemptiveGeneration`, then attach them to the `response_start` of that preemptive speech handle's `llm_response` span, including preemptive responses that are later discarded. Otherwise it is recommended. - **Linking:** Lingua uses `in_reply_to` whenever it is present, and falls back to the `end_of_turn` window only when it is absent. ## Timing semantics @@ -156,26 +154,30 @@ The `voice.timeline` object: LiveKit's realtime events do not expose these fields today, so LiveKit realtime uses `local`. -- **Agent speech:** it ends at `speech_end`, the played end. `truncate` only records what the provider was told. In the realtime example they differ by 300 ms (2600 vs 2900). +- **Agent speech:** it ends at `speech_end`, the played end. `truncate` only records what was sent to the provider. + - **LiveKit:** it computes `audioEndMs` from the same local `playbackPosition` (`agent_activity.ts:4789-4802`), so `truncate.audio_end_ms` always equals `played_ms`. In the realtime example both are 2900. + - **Pipecat:** only integrations that observe the provider independently, such as Pipecat reading raw provider events, can show a provider-versus-playout disagreement. - **No recording:** without one, `voice.timeline_origin_unix_ms` is the origin, and Lingua treats `audio` basis as `local`. ## What integrations capture, and when ### LiveKit (JS, `js/src/instrumentation/livekit`) -| Event | Cascaded (`_pipelineReplyTaskImpl`) | Realtime (`_realtimeGenerationTaskImpl`, `RealtimeObserver`) | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | Per VAD segment: new hooks on `AgentActivity.onStartOfSpeech` / `onEndOfSpeech`. Start = now − `speechDuration`, end = now − `silenceDuration` (`local`). These match the native `livekit.user_speaking` spans, not the whole-turn `EndOfTurnInfo` bounds. | `input_speech_started`: `u`, at receipt time (`local`). `input_speech_stopped`: receipt time minus the configured server-VAD silence (`turn_detection.silence_duration_ms`, logged as `voice.server_vad_silence_ms`) when known; otherwise receipt time (`local`), which then includes the provider's silence padding. With server turn detection off, caller speech comes from the local VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`, `local`), as in cascaded. | -| caller `transcript` | `user_input_transcribed` finals, `source: stt`, attached by the STT rule | `input_audio_transcription_completed`, `source: provider`, attached FIFO by the provider rule and followed by `message{message_id: itemId}`. It may arrive after `response_start`. | -| `message` | `_conversationItemAdded` (user and assistant items) | caller: when `input_audio_transcription_completed` arrives (`message_id = itemId`) | -| `end_of_turn` | `onEndOfTurn`, with the utterances since the previous commit | `userTurnCompleted` / buffer commit; with server turn detection off, LiveKit's own turn commit (`onEndOfTurn`) | -| `response_start` | `llm_response` span `onStart`, `response_id = generation_id` | realtime `llm_response` `onStart`, `response_id = contrib.livekit.response_id` | -| `tool_call` / `tool_result` | `function_tool` span start and end. `cancelled` when the speech handle was interrupted first. | same | -| agent `speech_start` / `speech_end` | Start: emitted at playout start, the first `playbackStarted` / `playbackProgressed` (`audio`). End: at `playbackFinished`, with the last played range end (`audio`), `interrupted` and `played_ms`. | same | -| agent `transcript` | `playbackFinished.synchronizedTranscript` (`source: playout`), else assistant item text (`provider`) | same | -| `truncate` | — | the truncate request to the provider | -| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | -| `call_end` | `closeImpl` | same | +Citations are `@livekit/agents` 1.9.2 `src/` paths; `aa.ts` is `voice/agent_activity.ts`. + +| Event | Cascaded (`_pipelineReplyTaskImpl`, `aa.ts:3747`) | Realtime (`_realtimeGenerationTaskImpl`, `aa.ts:4496`) | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | Per VAD segment: hook `onStartOfSpeech` (`aa.ts:2108`) / `onEndOfSpeech` (`aa.ts:2152`). Start = now − `speechDuration` − `inferenceDuration`; end = now − `silenceDuration` − `inferenceDuration`, as LiveKit computes for native `user_speaking` (`aa.ts:2111-2112`, `2156-2157`). `local`. | **Server turn detection on:** `onInputSpeechStarted` / `onInputSpeechStopped` (`aa.ts:1967`, `2000`); the events carry no provider ms (`llm/realtime.ts:16-18`). `u` at receipt (`local`); end = receipt − `voice.server_vad_silence_ms` only when that option is passed to the integration, else receipt, which includes the silence padding. Local VAD hooks are not used. **Server turn detection off:** local VAD hooks as in cascaded. | +| caller `transcript` | `user_input_transcribed` finals (`onFinalTranscript`, `aa.ts:2369`), `source: stt`, STT rule | `input_audio_transcription_completed`, `source: provider`, FIFO provider rule, then `message`; may arrive after `response_start` | +| `message` | `_conversationItemAdded` for user and assistant items (`aa.ts:3272`, `4273`) | assistant: `_conversationItemAdded` (`aa.ts:4827`); user: at transcription completion | +| `end_of_turn` | Only when `onEndOfTurn` (`aa.ts:2690`) resolves `true` and the turn was committed, not when it was skipped because scheduling is paused (`aa.ts:2700-2711`). `at_ms` = resolve time; lists caller utterances since the previous `end_of_turn`. | **Server turn detection on:** there is no commit event (`userTurnCompleted` returns early, `aa.ts:3301-3302`), so emit at `_realtimeGenerationTaskImpl` entry for a generation that answers caller input, listing caller utterances since the previous `end_of_turn`. Tool-continuation generations have no new utterances and emit none. **Server turn detection off:** as in cascaded. | +| `response_start` | `llm_response` span start inside the handle's reply task; `response_id` = `SpeechHandle.id`; `in_reply_to` from the `onPreemptiveGeneration` snapshot (`aa.ts:2433`) for preemptive handles | same, `response_id` = `SpeechHandle.id`. Preemptive generation is cascaded only (`aa.ts:2441`). | +| `tool_call` / `tool_result` | `function_tool` span start and end; `cancelled` when the handle was interrupted first | same | +| agent `speech_start` / `speech_end` | Playback hook on `AudioOutput`, installed independently of audio capture so audio-off calls still get bounds. Start = first `playbackStarted` (`voice/io.ts:118`, `210`) / `captureFrame`, attributed to `activity.currentSpeech` (`aa.ts:1168`). End = `playbackFinished` with `playbackPosition` (`interrupted`, `played_ms`). `audio` when capture places frames, else `local`. | same | +| agent `transcript` | `playbackFinished.synchronizedTranscript`, `source: playout`; else assistant item text (`provider`) | same | +| `truncate` | — | Only when `capabilities.messageTruncation` and the output was cut (`aa.ts:4789`). `audio_end_ms` = `playbackPosition` (`aa.ts:4797-4802`), so it equals `played_ms`. | +| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | +| `call_end` | `closeImpl` | same | Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, the flush and retry rules, and the transcript-attachment rule. Keep it on the per-session state in `processor.ts`. @@ -215,7 +217,7 @@ The timeline observer sits at the transport output processor, so agent events re - caller `speech_end` → `end_of_turn` → `response_start` → first agent `speech_start` in the window - overlap shading - `missing` badges -- truncate-versus-played markers +- truncate markers Thread messages still come from LLM span inputs and outputs. @@ -239,23 +241,24 @@ livekit.agent_session audio.recordings: call-0000 (13 s); voice.timel Concrete differences, same audio: -| | Cascaded | Realtime | -| ------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 4300 | -| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, attached to `u1` by the rule | -| Turn-1 latency, `end_of_turn` → first agent audio | 3400 → 5300 = 1900 ms: 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms: 200 ms to response, 500 ms to the tool call, 500 ms tool, 400 ms to audio | -| Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `truncate` marker at 7600 (`audio_end_ms` 2600), then `speech_end` 7900 with `played_ms` 2900 | -| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is dated receipt 3400 − 500 ms server-VAD silence = 2900 | -| Caller's barge-in text | `stt` "Nine thirty works." | none: `missing{evidence: transcript}`, and the `rt-3` input user content is `""` | -| Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | -| STT/TTS steps | `tts-1`, `tts-2` | none | +| | Cascaded | Realtime | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 4300 | +| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, attached to `u1` by the rule | +| Turn-1 latency, caller end → first agent audio | 2900 → 5300 = 2400 ms: 500 ms end-of-turn wait, 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 2900 → 5000 = 2100 ms: 700 ms to generation entry (`end_of_turn` at 3600; server-VAD silence and commit are not separable in LiveKit), 500 ms to the tool call, 500 ms tool, 400 ms to audio | +| Agent utterance ids | `speech_2/0`, `speech_4/0`; `message` links `item_a1`, `item_a2` | same handle-based ids; `message` links the realtime output items `item_a1`, `item_a2` | +| Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `speech_end` 7900 with `played_ms` 2900, and a `truncate` marker at 7900 with the same 2900 | +| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is dated receipt 3400 − 500 ms server-VAD silence = 2900 | +| Caller's barge-in text | `stt` "Nine thirty works." | none: `missing{evidence: transcript}`, and the `rt-3` input user content is `""` | +| Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | +| STT/TTS steps | `tts-1`, `tts-2` | none | ## Customer usefulness - **What was said: good.** Finals carry their source, the agent's played text is kept separate from generated text, and missing transcripts are explicit. - **Playing matching audio: good.** Every interval is on the recording clock. Lingua cuts selections per segment and per channel. - **Diagnosing delays: strongest of the options.** Each stage is a subtraction on one clock, in realtime too. -- **Interruptions and overlap: strong.** Overlap comes from intervals and interruption from `speech_end`, and the truncate-versus-played gap is visible. +- **Interruptions and overlap: strong.** Overlap comes from intervals and interruption from `speech_end`. A provider-versus-playout gap is visible only where the integration observes the provider independently (Pipecat); with LiveKit the two values are equal. - **Following tools: adequate.** Tools are placed in time and linked to spans; arguments and results stay on the spans. - **Missing or uncertain evidence: strong.** Every time has a `basis`, plus `missing` events, `seq` gaps, and a missing `final` chunk. @@ -269,14 +272,14 @@ Concrete differences, same audio: - **Call in progress, or a crash:** the latest ≤5 s are not yet flushed. With no `final` chunk, Lingua warns and closes open speech. - **Content capture off:** `text` is dropped, but timing analysis still works. - **Tool cancelled by interruption:** `tool_result{outcome: cancelled}`. The cause is inferred only from a nearby interrupted agent `speech_end`. -- **Agent handoff:** not modeled. A new agent activity continues the same log, and nothing records which agent spoke. +- **Agent handoff:** not modeled. A new agent activity continues the same log, and nothing records which agent spoke. `voice.pipeline` is the pipeline at call start; a handoff that changes the LLM type (for example cascaded to realtime) is out of MVP scope. - **Duplicate observers:** the SDK must emit one event per moment, such as LiveKit's native `user_turn` plus realtime events. Lingua deduplicates only identical retried chunks. - **Overall weakness, indirection:** speech, turns and step links depend on Lingua's derivation, and the trace tree alone no longer shows speech. ## Implementation starting point 1. Lock the schema in `call-timeline.ts`, including `basis`, flush and retry, and the transcript-attachment rule. -2. LiveKit: add `timeline.ts` and the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`). Emit from `onEndOfTurn`, the `RealtimeObserver` listeners, `playback.ts` `settle`, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. +2. LiveKit: add `timeline.ts`, the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`), and a playback hook installed independently of audio capture. Emit from `onEndOfTurn` (when it resolves `true` and commits), `onPreemptiveGeneration` (the `in_reply_to` snapshot), `_pipelineReplyTaskImpl` / `_realtimeGenerationTaskImpl` entry (the speech handle, and realtime `end_of_turn`), the `RealtimeObserver` listeners, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. 3. At close, emit `missing` events, then `call_end`, and flush the `final` chunk. 4. Lingua: the builder exists in the companion PR; extend it with the app allow-list keys after #486 is rebased. 5. Pipecat: port `timeline.ts` to the Python voice package following the table above. diff --git a/docs/voice-schema/call-timeline/examples/cascaded.json b/docs/voice-schema/call-timeline/examples/cascaded.json index 55868b57c..eee612420 100644 --- a/docs/voice-schema/call-timeline/examples/cascaded.json +++ b/docs/voice-schema/call-timeline/examples/cascaded.json @@ -429,7 +429,7 @@ "at_ms": 3450, "basis": "local", "kind": "response_start", - "response_id": "r1", + "response_id": "speech_1", "span_id": "llm-1", "in_reply_to": ["u1"] }, @@ -438,7 +438,7 @@ "at_ms": 4100, "basis": "local", "kind": "tool_call", - "response_id": "r1", + "response_id": "speech_1", "tool_call_id": "call_1", "name": "find_slots", "span_id": "tool-1" @@ -457,7 +457,7 @@ "at_ms": 4600, "basis": "local", "kind": "response_start", - "response_id": "r2", + "response_id": "speech_2", "span_id": "llm-2", "in_reply_to": ["u1"] }, @@ -467,8 +467,8 @@ "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "speech_1/0", - "response_id": "r2" + "utterance_id": "speech_2/0", + "response_id": "speech_2" }, { "seq": 10, @@ -484,7 +484,7 @@ "basis": "audio", "kind": "speech_end", "speaker": "agent", - "utterance_id": "speech_1/0", + "utterance_id": "speech_2/0", "interrupted": true, "played_ms": 2600 }, @@ -494,7 +494,7 @@ "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "speech_1/0", + "utterance_id": "speech_2/0", "final": true, "text": "I have nine thirty or eleven on Fri", "source": "playout" @@ -506,7 +506,7 @@ "kind": "message", "speaker": "agent", "message_id": "item_a1", - "utterance_ids": ["speech_1/0"] + "utterance_ids": ["speech_2/0"] } ] } @@ -571,7 +571,7 @@ "at_ms": 8700, "basis": "local", "kind": "response_start", - "response_id": "r3", + "response_id": "speech_3", "span_id": "llm-3", "in_reply_to": ["u2"] }, @@ -580,7 +580,7 @@ "at_ms": 9000, "basis": "local", "kind": "tool_call", - "response_id": "r3", + "response_id": "speech_3", "tool_call_id": "call_2", "name": "reschedule", "span_id": "tool-2" @@ -599,7 +599,7 @@ "at_ms": 9400, "basis": "local", "kind": "response_start", - "response_id": "r4", + "response_id": "speech_4", "span_id": "llm-4", "in_reply_to": ["u2"] }, @@ -609,8 +609,8 @@ "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "speech_2/0", - "response_id": "r4" + "utterance_id": "speech_4/0", + "response_id": "speech_4" }, { "seq": 23, @@ -618,7 +618,7 @@ "basis": "audio", "kind": "speech_end", "speaker": "agent", - "utterance_id": "speech_2/0", + "utterance_id": "speech_4/0", "interrupted": false, "played_ms": 2700 }, @@ -628,7 +628,7 @@ "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "speech_2/0", + "utterance_id": "speech_4/0", "final": true, "text": "Done — you're booked Friday at nine thirty.", "source": "playout" @@ -640,7 +640,7 @@ "kind": "message", "speaker": "agent", "message_id": "item_a2", - "utterance_ids": ["speech_2/0"] + "utterance_ids": ["speech_4/0"] }, { "seq": 26, diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json index 75941a728..51e086f4e 100644 --- a/docs/voice-schema/call-timeline/examples/realtime.json +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -337,7 +337,7 @@ }, "metrics": { "start": 1791830000.3, - "end": 1791830007.6 + "end": 1791830007.9 }, "metadata": { "voice.timeline": { @@ -364,7 +364,7 @@ }, { "seq": 2, - "at_ms": 3400, + "at_ms": 3600, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u1"] @@ -374,7 +374,7 @@ "at_ms": 3600, "basis": "local", "kind": "response_start", - "response_id": "resp_1", + "response_id": "speech_1", "span_id": "rt-1", "in_reply_to": ["u1"] }, @@ -383,7 +383,7 @@ "at_ms": 4100, "basis": "local", "kind": "tool_call", - "response_id": "resp_1", + "response_id": "speech_1", "tool_call_id": "call_1", "name": "find_slots", "span_id": "tool-1" @@ -422,7 +422,7 @@ "at_ms": 4600, "basis": "local", "kind": "response_start", - "response_id": "resp_2", + "response_id": "speech_2", "span_id": "rt-2", "in_reply_to": ["u1"] }, @@ -432,8 +432,8 @@ "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "item_a1", - "response_id": "resp_2" + "utterance_id": "speech_2/0", + "response_id": "speech_2" }, { "seq": 10, @@ -445,11 +445,13 @@ }, { "seq": 11, - "at_ms": 7600, - "basis": "local", - "kind": "truncate", - "utterance_id": "item_a1", - "audio_end_ms": 2600 + "at_ms": 7900, + "basis": "audio", + "kind": "speech_end", + "speaker": "agent", + "utterance_id": "speech_2/0", + "interrupted": true, + "played_ms": 2900 } ] } @@ -477,12 +479,10 @@ { "seq": 12, "at_ms": 7900, - "basis": "audio", - "kind": "speech_end", - "speaker": "agent", - "utterance_id": "item_a1", - "interrupted": true, - "played_ms": 2900 + "basis": "local", + "kind": "truncate", + "utterance_id": "speech_2/0", + "audio_end_ms": 2900 }, { "seq": 13, @@ -490,14 +490,22 @@ "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "item_a1", + "utterance_id": "speech_2/0", "final": true, "text": "I have nine thirty or eleven on Fri", - "message_id": "item_a1", "source": "playout" }, { "seq": 14, + "at_ms": 7900, + "basis": "local", + "kind": "message", + "speaker": "agent", + "message_id": "item_a1", + "utterance_ids": ["speech_2/0"] + }, + { + "seq": 15, "at_ms": 8500, "basis": "local", "kind": "speech_end", @@ -505,33 +513,33 @@ "utterance_id": "u2" }, { - "seq": 15, + "seq": 16, "at_ms": 8600, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u2"] }, { - "seq": 16, + "seq": 17, "at_ms": 8600, "basis": "local", "kind": "response_start", - "response_id": "resp_3", + "response_id": "speech_3", "span_id": "rt-3", "in_reply_to": ["u2"] }, { - "seq": 17, + "seq": 18, "at_ms": 8900, "basis": "local", "kind": "tool_call", - "response_id": "resp_3", + "response_id": "speech_3", "tool_call_id": "call_2", "name": "reschedule", "span_id": "tool-2" }, { - "seq": 18, + "seq": 19, "at_ms": 9300, "basis": "local", "kind": "tool_result", @@ -540,47 +548,55 @@ "outcome": "ok" }, { - "seq": 19, + "seq": 20, "at_ms": 9300, "basis": "local", "kind": "response_start", - "response_id": "resp_4", + "response_id": "speech_4", "span_id": "rt-4", "in_reply_to": ["u2"] }, { - "seq": 20, + "seq": 21, "at_ms": 9700, "basis": "audio", "kind": "speech_start", "speaker": "agent", - "utterance_id": "item_a2", - "response_id": "resp_4" + "utterance_id": "speech_4/0", + "response_id": "speech_4" }, { - "seq": 21, + "seq": 22, "at_ms": 12500, "basis": "audio", "kind": "speech_end", "speaker": "agent", - "utterance_id": "item_a2", + "utterance_id": "speech_4/0", "interrupted": false, "played_ms": 2800 }, { - "seq": 22, + "seq": 23, "at_ms": 12500, "basis": "local", "kind": "transcript", "speaker": "agent", - "utterance_id": "item_a2", + "utterance_id": "speech_4/0", "final": true, "text": "Done — you're booked Friday at nine thirty.", - "message_id": "item_a2", "source": "playout" }, { - "seq": 23, + "seq": 24, + "at_ms": 12500, + "basis": "local", + "kind": "message", + "speaker": "agent", + "message_id": "item_a2", + "utterance_ids": ["speech_4/0"] + }, + { + "seq": 25, "at_ms": 13000, "basis": "local", "kind": "missing", @@ -589,7 +605,7 @@ "reason": "input_transcription_not_received" }, { - "seq": 24, + "seq": 26, "at_ms": 13000, "basis": "local", "kind": "call_end", From 005fc49c48427be19c45507d6d4ed7d8d2af2d8b Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:09:26 +0000 Subject: [PATCH 08/11] chore: tighten call timeline capture rules Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 26 +++++++++---------- .../call-timeline/examples/cascaded.json | 14 +++++----- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 6d3a94b5f..23d2203b7 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -165,19 +165,19 @@ The `voice.timeline` object: Citations are `@livekit/agents` 1.9.2 `src/` paths; `aa.ts` is `voice/agent_activity.ts`. -| Event | Cascaded (`_pipelineReplyTaskImpl`, `aa.ts:3747`) | Realtime (`_realtimeGenerationTaskImpl`, `aa.ts:4496`) | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | Per VAD segment: hook `onStartOfSpeech` (`aa.ts:2108`) / `onEndOfSpeech` (`aa.ts:2152`). Start = now − `speechDuration` − `inferenceDuration`; end = now − `silenceDuration` − `inferenceDuration`, as LiveKit computes for native `user_speaking` (`aa.ts:2111-2112`, `2156-2157`). `local`. | **Server turn detection on:** `onInputSpeechStarted` / `onInputSpeechStopped` (`aa.ts:1967`, `2000`); the events carry no provider ms (`llm/realtime.ts:16-18`). `u` at receipt (`local`); end = receipt − `voice.server_vad_silence_ms` only when that option is passed to the integration, else receipt, which includes the silence padding. Local VAD hooks are not used. **Server turn detection off:** local VAD hooks as in cascaded. | -| caller `transcript` | `user_input_transcribed` finals (`onFinalTranscript`, `aa.ts:2369`), `source: stt`, STT rule | `input_audio_transcription_completed`, `source: provider`, FIFO provider rule, then `message`; may arrive after `response_start` | -| `message` | `_conversationItemAdded` for user and assistant items (`aa.ts:3272`, `4273`) | assistant: `_conversationItemAdded` (`aa.ts:4827`); user: at transcription completion | -| `end_of_turn` | Only when `onEndOfTurn` (`aa.ts:2690`) resolves `true` and the turn was committed, not when it was skipped because scheduling is paused (`aa.ts:2700-2711`). `at_ms` = resolve time; lists caller utterances since the previous `end_of_turn`. | **Server turn detection on:** there is no commit event (`userTurnCompleted` returns early, `aa.ts:3301-3302`), so emit at `_realtimeGenerationTaskImpl` entry for a generation that answers caller input, listing caller utterances since the previous `end_of_turn`. Tool-continuation generations have no new utterances and emit none. **Server turn detection off:** as in cascaded. | -| `response_start` | `llm_response` span start inside the handle's reply task; `response_id` = `SpeechHandle.id`; `in_reply_to` from the `onPreemptiveGeneration` snapshot (`aa.ts:2433`) for preemptive handles | same, `response_id` = `SpeechHandle.id`. Preemptive generation is cascaded only (`aa.ts:2441`). | -| `tool_call` / `tool_result` | `function_tool` span start and end; `cancelled` when the handle was interrupted first | same | -| agent `speech_start` / `speech_end` | Playback hook on `AudioOutput`, installed independently of audio capture so audio-off calls still get bounds. Start = first `playbackStarted` (`voice/io.ts:118`, `210`) / `captureFrame`, attributed to `activity.currentSpeech` (`aa.ts:1168`). End = `playbackFinished` with `playbackPosition` (`interrupted`, `played_ms`). `audio` when capture places frames, else `local`. | same | -| agent `transcript` | `playbackFinished.synchronizedTranscript`, `source: playout`; else assistant item text (`provider`) | same | -| `truncate` | — | Only when `capabilities.messageTruncation` and the output was cut (`aa.ts:4789`). `audio_end_ms` = `playbackPosition` (`aa.ts:4797-4802`), so it equals `played_ms`. | -| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | -| `call_end` | `closeImpl` | same | +| Event | Cascaded (`_pipelineReplyTaskImpl`, `aa.ts:3747`) | Realtime (`_realtimeGenerationTaskImpl`, `aa.ts:4496`) | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| caller `speech_start` / `speech_end` | Per VAD segment: hook `onStartOfSpeech` (`aa.ts:2108`) / `onEndOfSpeech` (`aa.ts:2152`). Start = now − `speechDuration` − `inferenceDuration`; end = now − `silenceDuration` − `inferenceDuration`, as LiveKit computes for native `user_speaking` (`aa.ts:2111-2112`, `2156-2157`). `local`. | **Server turn detection on:** `onInputSpeechStarted` / `onInputSpeechStopped` (`aa.ts:1967`, `2000`); the events carry no provider ms (`llm/realtime.ts:16-18`). `u` at receipt (`local`); end = receipt − `voice.server_vad_silence_ms` only when that option is passed to the integration, else receipt, which includes the silence padding. Local VAD hooks are not used. **Server turn detection off:** local VAD hooks as in cascaded. | +| caller `transcript` | `user_input_transcribed` finals (`onFinalTranscript`, `aa.ts:2369`), `source: stt`, STT rule | `input_audio_transcription_completed`, `source: provider`, FIFO provider rule, then `message`; may arrive after `response_start` | +| `message` | Emitted only from the `AgentSession._conversationItemAdded` hook. The user item comes from the normal commit inside `_pipelineReplyTaskImpl` (`aa.ts:3975`, only when `speechHandle.scheduled`); a skipped turn (`commitSkippedUserTurn`, `aa.ts:3272`) emits no `message`. Assistant items also come through this hook (`aa.ts:4273`). | Same single hook: the user item when its transcript is inserted (`aa.ts:2054`), and the assistant item (`aa.ts:4827`). The transcription event emits only `transcript`, never `message`. | +| `end_of_turn` | Committed turns only. At `onEndOfTurn` (`aa.ts:2690`), record the resolve time if the call returned `true` and neither `schedulingPaused` (`aa.ts:1230`) nor `newTurnsBlocked` (`aa.ts:550`) was set before it (the paused branch is `aa.ts:2700-2711`). Emit the event, with `at_ms` = that resolve time, when the turn's reply is scheduled (the user commit at `aa.ts:3975`). Turns that `userTurnCompleted` later drops emit **no** `end_of_turn`: the AMD `skipReply` (`aa.ts:2697`, committed via `aa.ts:3316-3319`), the non-interruptible drop (`aa.ts:3328-3333`), and `StopResponse` (`aa.ts:3388-3390`). Their utterances leave the pending list and appear in no `end_of_turn`, so Lingua attaches them to the turn open at their start. Lists the caller utterances since the previous `end_of_turn`. | **Server turn detection on:** there is no commit event (`userTurnCompleted` returns early, `aa.ts:3301-3302`). Emit at `_realtimeGenerationTaskImpl` entry for a generation that answers caller input, listing caller utterances since the previous `end_of_turn` that are already closed. Utterances still open at task entry are excluded and go to the next `end_of_turn`. Tool-continuation generations have no new utterances and emit none. **Server turn detection off:** as in cascaded. | +| `response_start` | `llm_response` span start inside the handle's reply task. LiveKit's `speechHandleStorage` (`agent.ts:70`) is not exported, so the SDK sets its own async-local context with the `SpeechHandle` at `_pipelineReplyTaskImpl` entry and reads it in the span processor's `onStart`. `response_id` = `SpeechHandle.id`. For preemptive handles, `in_reply_to` comes from the `onPreemptiveGeneration` snapshot (`aa.ts:2433`, guard at `aa.ts:2440`). | The `_realtimeGenerationTaskImpl` hook reads its `inferenceSpan` argument (the `realtime_inference` span started at `aa.ts:4478`, exported as `llm_response`) and its handle; `response_id` = `SpeechHandle.id`. Preemptive generation is cascaded only (`aa.ts:2441`). | +| `tool_call` / `tool_result` | `function_tool` span start and end; `cancelled` when the handle was interrupted first | same | +| agent `speech_start` / `speech_end` | Playback hook on `AudioOutput`, installed independently of audio capture so audio-off calls still get bounds. Start = first `playbackStarted` (`voice/io.ts:118`, `210`) / `captureFrame`, attributed to `activity.currentSpeech` (`aa.ts:1168`). End = `playbackFinished` with `playbackPosition` (`interrupted`, `played_ms`). `audio` when capture places frames, else `local`. | same | +| agent `transcript` | `playbackFinished.synchronizedTranscript`, `source: playout`; else assistant item text (`provider`) | same | +| `truncate` | — | Only when `capabilities.messageTruncation` and the output was cut (`aa.ts:4789`). `audio_end_ms` = `playbackPosition` (`aa.ts:4797-4802`), so it equals `played_ms`. | +| `missing` | At close: caller utterances in a committed turn with no final transcript, and starts without ends | same | +| `call_end` | `closeImpl` | same | Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, the flush and retry rules, and the transcript-attachment rule. Keep it on the per-session state in `processor.ts`. diff --git a/docs/voice-schema/call-timeline/examples/cascaded.json b/docs/voice-schema/call-timeline/examples/cascaded.json index eee612420..d4de7b5f3 100644 --- a/docs/voice-schema/call-timeline/examples/cascaded.json +++ b/docs/voice-schema/call-timeline/examples/cascaded.json @@ -399,13 +399,6 @@ }, { "seq": 2, - "at_ms": 3400, - "basis": "local", - "kind": "end_of_turn", - "utterance_ids": ["u1"] - }, - { - "seq": 3, "at_ms": 3450, "basis": "local", "kind": "transcript", @@ -415,6 +408,13 @@ "text": "Hi, can I move my Thursday cleaning to Friday morning?", "source": "stt" }, + { + "seq": 3, + "at_ms": 3400, + "basis": "local", + "kind": "end_of_turn", + "utterance_ids": ["u1"] + }, { "seq": 4, "at_ms": 3450, From 3d5118e43e8bc520077b2ffc9961fbbc4333e92c Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:58:48 +0000 Subject: [PATCH 09/11] feat: add schema at a glance for call-timeline Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 294 +++++++++++++----- .../call-timeline/examples/realtime.json | 52 ++-- .../voice-schema/call-timeline.test.ts | 15 + 3 files changed, 264 insertions(+), 97 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 23d2203b7..64a81502d 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -2,19 +2,7 @@ > AI-created or modified and not human-reviewed in its current form; treat this artifact as provisional. Updated 2026-10-12. -In this option, integrations log what they observe during a call as an append-only event log. Each event records a speech segment boundary, a transcript, an end-of-turn decision, a generation, a tool call or a playout stop. - -- **The SDK** emits one caller utterance per voice-activity-detection (VAD) speech segment. It never builds turns. -- **Lingua** groups utterances into turns at `end_of_turn` events. It links speech to steps and cuts audio selections. -- **The app** computes latencies and overlaps from the log. - -**Implemented in this PR:** - -- the zod schema (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`) -- the two example traces (`examples/`) -- a test that parses them and checks versioning, `seq`, start/end pairing and utterance references - -**Implemented in the companion Lingua PR:** the Lingua mapping, built from `specs/voice-schema-call-timeline.md` with fixtures from these examples. **Sketched only:** LiveKit and Pipecat capture. +Integrations log what they observe during a call as an append-only event log on child rows of the call span. The SDK never builds speech spans or turns: Lingua groups caller utterances into turns at `end_of_turn` events, links speech to steps and cuts audio selections, and the app computes latencies and overlaps from the log. ## Customer problem @@ -26,32 +14,167 @@ A customer debugging a voice agent needs to answer five questions: - Who talked over whom? - Which tool call caused a pause? -Today that evidence is scattered. It sits in span names (`user_turn`, `livekit.agent_speaking`), in vendor keys (`contrib.livekit.*`, `contrib.pipecat.*`), in span times on different clocks, and in selections that exist only after upload. Realtime pipelines add transcripts that arrive late or never, and an interruption point reported by the provider that differs from what the caller heard. +Today that evidence is scattered. It sits in span names (`user_turn`, `livekit.agent_speaking`), in vendor keys (`contrib.livekit.*`, `contrib.pipecat.*`), in span times on different clocks, and in selections that exist only after upload. Realtime pipelines add transcripts that arrive late or never, and an interruption point reported by the provider that may differ from what the caller heard. + +## Schema at a glance + +Evidence lives in `metadata["voice.events"]` on `voice.timeline` child rows; the call span carries only the recordings and two required keys. Step spans (`llm_response`, tools, `tts`) are unchanged. + + +```ts +// Call span (root). span_attributes = { name: "livekit.agent_session", type: "task" } +type CallRow = { + input: { audio: Record } // shared @braintrust/audio, ready segments only + metadata: { + "audio.recordings"?: AudioRecording[] // shared; absent when audio capture is off + "voice.timeline_version": 1 // required; selects the timeline path in Lingua + "voice.pipeline": "cascaded" | "realtime" | "half_cascade" // required; pipeline at call start + "voice.timeline_origin_unix_ms"?: number // unix ms; required iff "audio.recordings" is absent + "voice.server_vad_silence_ms"?: number // ms already subtracted from realtime caller speech_end + } +} +type AudioRecording = { // shared layer, unchanged + id: RecordingId; state: "ready" | "pending" | "omitted"; duration_ms?: number // duration_ms: ready only + attachment?: { span_id: string; ref: string } // ready only; ref = "/input/audio/" + sources?: { channel_index: number; boundary: string }[] // "session_input" = ch 0 caller, "local_audio_output" = ch 1 agent + timeline?: { origin_unix_ms: number; recording_start_offset_ms: number } // unix ms; call ms +} +// "audio.selections" is not logged by this option: Lingua cuts selections from speech intervals. + +// Timeline row: child of the call span, one per flushed chunk. span_attributes = { name: "voice.timeline", type: "task" } +type TimelineRow = { + span_parents: [CallSpanId] + metrics: { start: number; end: number } // unix s = (origin_unix_ms + min/max at_ms) / 1000 + metadata: { + "voice.timeline": { call_span_id: string; chunk_index: number; final: boolean } // final: only the call-end chunk + "voice.events": VoiceEvent[] // 1..256, consecutive seq across chunks + } +} +type VoiceEvent = { seq: number; at_ms: number; basis: "audio" | "local" | "provider" } & ( // seq: int >= 0; at_ms: call ms >= 0 + | { kind: "speech_start"; speaker: Speaker; utterance_id: UtteranceId; response_id?: ResponseId } // response_id: agent only + | { kind: "speech_end"; speaker: Speaker; utterance_id: UtteranceId; interrupted?: boolean; played_ms?: number } // agent only; played_ms: ms + | { kind: "transcript"; speaker: Speaker; utterance_id: UtteranceId; final: boolean + source: "stt" | "provider" | "playout"; text?: string; message_id?: MessageId } // at_ms = arrival; text absent if capture off + | { kind: "message"; speaker: Speaker; message_id: MessageId; utterance_ids: UtteranceId[] } // chat item <- utterances, >= 1 + | { kind: "end_of_turn"; utterance_ids: UtteranceId[] } // caller utterances committed since the previous end_of_turn, >= 1 + | { kind: "response_start"; response_id: ResponseId; span_id: string; in_reply_to?: UtteranceId[] } // span_id of the llm_response row + | { kind: "tool_call"; response_id: ResponseId; tool_call_id: string; name: string; span_id: string } // span_id of the tool row + | { kind: "tool_result"; tool_call_id: string; span_id: string; outcome: "ok" | "error" | "cancelled" } + | { kind: "truncate"; utterance_id: UtteranceId; audio_end_ms: number } // ms after that agent speech_start; marker only + | { kind: "missing"; evidence: "transcript" | "speech_end" | "audio" | "tool_result" | "timeline"; reason: string + utterance_id?: UtteranceId } // required for transcript / speech_end + | { kind: "call_end"; reason: string } // last event +) +type Speaker = "user" | "agent" +type UtteranceId = string // caller "u"; agent "/" (Pipecat "/") +type ResponseId = string // LiveKit SpeechHandle.id of the reply task; Pipecat SDK counter +type MessageId = string // framework chat item id (realtime: provider item id) +// Step rows (unchanged): tool row metadata["gen_ai.tool.call.id"] = tool_call_id, +// metadata["contrib.livekit.request_span_id"] = span_id of the llm_response that requested it. +``` + +**Time.** + +- `at_ms` is call ms: milliseconds since `audio.recordings[].timeline.origin_unix_ms` (or `voice.timeline_origin_unix_ms` without a recording). Call ms `t` is offset `t - recording_start_offset_ms` inside a segment. +- `basis` says how each `at_ms` was obtained: `audio` (recorder sample positions: agent playout), `local` (hook wall clock minus a known delay: caller VAD, transcripts, end of turn, responses, tools), `provider` (provider ms on a known anchor: Pipecat realtime only). `played_ms` and `truncate.audio_end_ms` are durations from the agent `speech_start`. +- `metrics.start` / `metrics.end` are unix seconds. On timeline rows they only place the row; on step rows they are span-clock times and are never mixed with `at_ms`. + +**Relationships.** + +``` +transcript.utterance_id, speech_end.utterance_id, truncate.utterance_id -> speech_start.utterance_id +end_of_turn.utterance_ids[], response_start.in_reply_to[] -> caller utterance ids +message.utterance_ids[] -> utterance ids; message.message_id / transcript.message_id -> chat item id +speech_start.response_id (agent) -> response_start.response_id -> response_start.span_id = llm_response row +tool_call.response_id -> response_start.response_id; tool_call.tool_call_id -> tool_result.tool_call_id +tool_call.span_id / tool_result.span_id -> tool row; tool row "gen_ai.tool.call.id" = tool_call_id +``` + +**Scenario excerpt.** Selected `voice.events` from [`examples/cascaded.json`](examples/cascaded.json) and [`examples/realtime.json`](examples/realtime.json), unchanged except for omitted events. + + +```jsonc +// cascaded: call span { "voice.timeline_version": 1, "voice.pipeline": "cascaded" } +{ "seq": 1, "at_ms": 2900, "basis": "local", "kind": "speech_end", "speaker": "user", "utterance_id": "u1" }, +{ "seq": 2, "at_ms": 3450, "basis": "local", "kind": "transcript", "speaker": "user", "utterance_id": "u1", "final": true, + "text": "Hi, can I move my Thursday cleaning to Friday morning?", "source": "stt" }, +{ "seq": 3, "at_ms": 3400, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u1"] }, +{ "seq": 4, "at_ms": 3450, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u1", "utterance_ids": ["u1"] }, +{ "seq": 5, "at_ms": 3450, "basis": "local", "kind": "response_start", "response_id": "speech_1", "span_id": "llm-1", "in_reply_to": ["u1"] }, +{ "seq": 6, "at_ms": 4100, "basis": "local", "kind": "tool_call", "response_id": "speech_1", "tool_call_id": "call_1", + "name": "find_slots", "span_id": "tool-1" }, +{ "seq": 9, "at_ms": 5300, "basis": "audio", "kind": "speech_start", "speaker": "agent", "utterance_id": "speech_2/0", "response_id": "speech_2" }, +{ "seq": 10, "at_ms": 7600, "basis": "local", "kind": "speech_start", "speaker": "user", "utterance_id": "u2" }, +{ "seq": 11, "at_ms": 7900, "basis": "audio", "kind": "speech_end", "speaker": "agent", "utterance_id": "speech_2/0", + "interrupted": true, "played_ms": 2600 }, +{ "seq": 12, "at_ms": 7900, "basis": "local", "kind": "transcript", "speaker": "agent", "utterance_id": "speech_2/0", "final": true, + "text": "I have nine thirty or eleven on Fri", "source": "playout" }, +{ "seq": 14, "at_ms": 8520, "basis": "local", "kind": "transcript", "speaker": "user", "utterance_id": "u2", "final": true, + "text": "Nine thirty works.", "source": "stt" } +``` + + +```jsonc +// realtime: call span adds "voice.server_vad_silence_ms": 500 +{ "seq": 1, "at_ms": 2900, "basis": "local", "kind": "speech_end", "speaker": "user", "utterance_id": "u1" }, // receipt 3400 - 500 +{ "seq": 2, "at_ms": 3600, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u1"] }, // generation entry, not a commit +{ "seq": 3, "at_ms": 3600, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u1", "utterance_ids": ["u1"] }, +{ "seq": 4, "at_ms": 3600, "basis": "local", "kind": "response_start", "response_id": "speech_1", "span_id": "rt-1", "in_reply_to": ["u1"] }, +{ "seq": 5, "at_ms": 4100, "basis": "local", "kind": "tool_call", "response_id": "speech_1", "tool_call_id": "call_1", + "name": "find_slots", "span_id": "tool-1" }, +{ "seq": 6, "at_ms": 4300, "basis": "local", "kind": "transcript", "speaker": "user", "utterance_id": "u1", "final": true, + "text": "Hi, can I move my Thursday cleaning to Friday morning?", "source": "provider", "message_id": "item_u1" }, // after the tool call +{ "seq": 9, "at_ms": 5000, "basis": "audio", "kind": "speech_start", "speaker": "agent", "utterance_id": "speech_2/0", "response_id": "speech_2" }, +{ "seq": 10, "at_ms": 7600, "basis": "local", "kind": "speech_start", "speaker": "user", "utterance_id": "u2" }, +{ "seq": 11, "at_ms": 7900, "basis": "audio", "kind": "speech_end", "speaker": "agent", "utterance_id": "speech_2/0", + "interrupted": true, "played_ms": 2900 }, +{ "seq": 12, "at_ms": 7900, "basis": "local", "kind": "truncate", "utterance_id": "speech_2/0", "audio_end_ms": 2900 }, // realtime only; = played_ms +{ "seq": 13, "at_ms": 7900, "basis": "local", "kind": "transcript", "speaker": "agent", "utterance_id": "speech_2/0", "final": true, + "text": "I have nine thirty or eleven on Fri", "source": "playout" }, +{ "seq": 17, "at_ms": 8600, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u2", "utterance_ids": ["u2"] }, // no transcript +{ "seq": 26, "at_ms": 13000, "basis": "local", "kind": "missing", "utterance_id": "u2", "evidence": "transcript", + "reason": "input_transcription_not_received" } // replaces the cascaded seq 14 transcript +``` + +Cascaded vs realtime: what changes in the payload + +| | Cascaded | Realtime | +| ------------------------ | ---------------------------------------- | ------------------------------------------------------------------------------------------ | +| Call span | `voice.pipeline: "cascaded"` | `"realtime"`, plus `voice.server_vad_silence_ms` when the integration is given it | +| Caller transcript | `source: "stt"`, before `response_start` | `source: "provider"` with `message_id`; may arrive after `tool_call`, or never (`missing`) | +| `end_of_turn.at_ms` | turn detector resolve time (3400) | entry of the answering generation (3600) | +| Caller `message` | at the user commit | at generation entry, even without a transcript (`item_u2`) | +| Interrupted agent speech | `speech_end` with `played_ms` only | same, plus a `truncate` marker equal to `played_ms` (LiveKit) | +| Step rows | `llm_response` + `tts` rows | `llm_response` rows only (`contrib.livekit.response_id` = provider response id) | + +Implemented: zod types (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`), examples, test. Sketch: LiveKit/Pipecat emission. Lingua mapping: implemented in the companion Lingua PR (`specs/voice-schema-call-timeline.md`, branch `kevchoi/voice-schema-call-timeline`). ## Contract +**Implemented** as zod in `call-timeline.ts`, exercised by both examples and the test. Field-level rules the zod cannot express are stated here. + ### Call span The call span is the span that logs `audio.recordings`, or the session span when there is no recording. -| Metadata key | Type | Req. | Meaning | -| ------------------------------- | -------------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `voice.timeline_version` | `1` | yes | Marks the call as using this contract. Lingua builds timeline speech only when it is present; otherwise it uses the #486 turn-span path. | -| `voice.pipeline` | `"cascaded" \| "realtime" \| "half_cascade"` | yes | Pipeline shape, set at session start. `half_cascade` is a text-output realtime model plus TTS: caller events follow the realtime rules, and agent events follow cascaded playout. | -| `voice.timeline_origin_unix_ms` | number | only without a recording | Origin of the call clock when `audio.recordings` is absent. | -| `voice.server_vad_silence_ms` | number | no | Realtime server-VAD silence that the SDK subtracted from caller `speech_end` receipt times | +| Metadata key | Type | Req. | Meaning | +| ------------------------------- | -------------------------------------------- | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `voice.timeline_version` | `1` | yes | Marks the call as using this contract. Lingua builds timeline speech only when it is present; otherwise it uses the #486 turn-span path. | +| `voice.pipeline` | `"cascaded" \| "realtime" \| "half_cascade"` | yes | Pipeline shape, set at session start. `half_cascade` is a text-output realtime model plus TTS: caller events follow the realtime rules, and agent events follow cascaded playout. | +| `voice.timeline_origin_unix_ms` | number, unix ms | yes when `audio.recordings` is absent | Origin of the call clock when there is no recording. Omit it when there is a recording. | +| `voice.server_vad_silence_ms` | number, ms | no | Realtime server-VAD silence that the SDK subtracted from caller `speech_end` receipt times. Set only when the integration was given that value. | ### Timeline rows The integration buffers events and flushes them as **timeline rows**: child spans of the call span. -| Field | Location | Type | Req. | Meaning | -| ---------------- | ------------------------------- | ---------------------------- | ---- | ---------------------------------------------------------------------------------- | -| name, type | `span_attributes` | `"voice.timeline"`, `"task"` | yes | One row per chunk | -| parent | `span_parents[0]` | span id | yes | The call span | -| `voice.timeline` | `metadata` | object | yes | `{ call_span_id, chunk_index, final }`, described below | -| `voice.events` | `metadata` | `VoiceEvent[]`, 1–256 | yes | Events with consecutive `seq` | -| start, end | `metrics.start` / `metrics.end` | unix s | yes | Min and max event time in the chunk. Used only to place the row in the trace tree. | +| Field | Location | Type | Req. | Meaning | +| ---------------- | ------------------------------- | ---------------------------- | ---- | ---------------------------------------------------------------------------------------------------------- | +| name, type | `span_attributes` | `"voice.timeline"`, `"task"` | yes | One row per chunk | +| parent | `span_parents[0]` | span id | yes | The call span | +| `voice.timeline` | `metadata` | object | yes | `{ call_span_id, chunk_index, final }`, described below | +| `voice.events` | `metadata` | `VoiceEvent[]`, 1–256 | yes | Events with consecutive `seq` | +| start, end | `metrics.start` / `metrics.end` | unix s | yes | `(origin_unix_ms + min/max at_ms) / 1000` over the chunk's events. Used only to place the row in the tree. | The `voice.timeline` object: @@ -73,26 +196,33 @@ The `voice.timeline` object: ### `VoiceEvent` -| Common field | Type | Req. | Meaning | -| ------------ | ---------------------------------- | ---- | -------------------------------------------------------------------------------------------- | -| `seq` | int ≥ 0 | yes | Emission order, consecutive per call across chunks. A gap means a lost or unflushed chunk. | -| `at_ms` | number | yes | When the thing happened, in **call ms** (see Timing). Not necessarily increasing with `seq`. | -| `basis` | `"audio" \| "local" \| "provider"` | yes | How `at_ms` was obtained | -| `kind` | enum below | yes | | - -| `kind` | Extra fields (`?` = optional) | Meaning and timing | -| ---------------- | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `speech_start` | `speaker`, `utterance_id`, `response_id?` | Start of one VAD speech segment (caller) or of playout (agent) | -| `speech_end` | `speaker`, `utterance_id`, `interrupted?`, `played_ms?` | Real end of that segment: for the caller, the VAD stop time minus the VAD silence delay; for the agent, the playout stop. `played_ms` is audio actually played. | -| `transcript` | `speaker`, `utterance_id`, `final`, `source`, `text?`, `message_id?` | Text for one utterance. `at_ms` is when the text arrived. `source` is `stt`, `provider` or `playout`. | -| `message` | `speaker`, `message_id`, `utterance_ids[]` | The framework committed a chat item made from these utterances | -| `end_of_turn` | `utterance_ids[]` | The turn detector committed the caller's turn; lists the caller utterances since the previous `end_of_turn` | -| `response_start` | `response_id`, `span_id`, `in_reply_to?[]` | An LLM or realtime generation started. `span_id` is its `llm_response` span. | -| `tool_call` | `response_id`, `tool_call_id`, `name`, `span_id` | A tool started | -| `tool_result` | `tool_call_id`, `span_id`, `outcome` | The tool finished. `outcome` is `ok`, `error` or `cancelled`. | -| `truncate` | `utterance_id`, `audio_end_ms` | Marker: the SDK told a realtime provider the agent item was cut at `audio_end_ms`, relative to the item. It does not end speech. | -| `missing` | `evidence`, `reason`, `utterance_id?` | Expected evidence will never arrive. `evidence` is `transcript`, `speech_end`, `audio`, `tool_result` or `timeline`. | -| `call_end` | `reason` | Session closed. This is the last event emitted. | +| Common field | Type | Req. | Meaning | +| ------------ | ---------------------------------- | ---- | ------------------------------------------------------------------------------------------------------ | +| `seq` | int ≥ 0 | yes | Emission order, consecutive per call across chunks. A gap means a lost or unflushed chunk. | +| `at_ms` | number ≥ 0 | yes | When the thing happened, in **call ms** (see Timing semantics). Not necessarily increasing with `seq`. | +| `basis` | `"audio" \| "local" \| "provider"` | yes | How `at_ms` was obtained | +| `kind` | enum below | yes | | + +| `kind` | Extra fields (`?` = optional) | Meaning and timing | +| ---------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `speech_start` | `speaker`, `utterance_id`, `response_id?` | Start of one VAD speech segment (caller) or of playout (agent). `response_id` is set on agent speech when playback was attributed to a reply task, and never on caller speech. | +| `speech_end` | `speaker`, `utterance_id`, `interrupted?`, `played_ms?` | Real end of that segment. Caller: the VAD stop time minus the known silence delay; LiveKit realtime without `voice.server_vad_silence_ms` uses the receipt time, which includes the padding. Agent: the playout stop. `interrupted` and `played_ms` (ms of audio played since `speech_start`) are always set on agent speech, never on caller. | +| `transcript` | `speaker`, `utterance_id`, `final`, `source`, `text?`, `message_id?` | Text for one utterance. `at_ms` is when the text arrived. `source` is `stt`, `provider` or `playout`. MVP integrations emit finals only (`final: true`); `final: false` is reserved for interim text, which Lingua copies but never attaches. `message_id` is set when the transcript event itself carries the chat item id (realtime provider finals). | +| `message` | `speaker`, `message_id`, `utterance_ids[]` | The framework committed a chat item made from these utterances. One per chat item. | +| `end_of_turn` | `utterance_ids[]` | The turn detector committed the caller's turn; lists the caller utterances since the previous `end_of_turn`. | +| `response_start` | `response_id`, `span_id`, `in_reply_to?[]` | An LLM or realtime generation started. `span_id` is the `span_id` of its `llm_response` row. | +| `tool_call` | `response_id`, `tool_call_id`, `name`, `span_id` | A tool started. `tool_call_id` equals the LLM output `tool_calls[].id` and the tool row's `gen_ai.tool.call.id`; `span_id` is the tool row. | +| `tool_result` | `tool_call_id`, `span_id`, `outcome` | The tool finished. `outcome` is `ok`, `error` or `cancelled`. | +| `truncate` | `utterance_id`, `audio_end_ms` | Marker: the SDK told a realtime provider the agent item was cut at `audio_end_ms`, in ms after that utterance's `speech_start`. It does not end speech. | +| `missing` | `evidence`, `reason`, `utterance_id?` | Expected evidence will never arrive. `at_ms` is when that was decided (call end for close-time checks). `reason` is a free snake_case string. See the emitter list below. | +| `call_end` | `reason` | Session closed. This is the last event emitted. `reason` is a free snake_case string such as `participant_disconnected`. | + +`missing` emitters in the MVP: + +- `transcript` (with `utterance_id`): at close, a committed caller utterance with no final and not covered by its group's final. +- `speech_end` (with `utterance_id`): at close, a `speech_start` with no end. +- `timeline` (no `utterance_id`): buffer overflow, as above. +- `audio` and `tool_result`: reserved, with no MVP emitter. Recording gaps live on `audio.recordings[].state`, and a `tool_call` with no `tool_result` is itself the evidence. `text` is omitted when content capture is off. All other fields carry timing or identity and are always logged. @@ -106,33 +236,35 @@ The `voice.timeline` object: **A caller turn is one or more utterances.** `end_of_turn.utterance_ids` lists them. The SDK does not merge VAD segments. -**Transcript to utterance.** No caller speech event carries an item id, so the SDK attaches each caller final by one of two rules. +**Transcript to utterance.** No caller speech event carries an item id, so the SDK attaches each caller final by one of these rules. - **STT finals (`source: stt`, cascaded and half-cascade STT):** attach to the caller utterance that is open at arrival, or else to the most recently ended one, within the current uncommitted turn. Several finals for one utterance are joined in `seq` order. A final that arrives after `end_of_turn` attaches to the last utterance of that turn. -- **Provider finals (`source: provider`, realtime):** attach FIFO to the oldest committed caller utterance (listed in an `end_of_turn`) that has no final yet. A provider transcribes committed audio in commit order, often after the next utterance has started, so "open at arrival" would pick the wrong one. - - **Assumption:** each committed `end_of_turn` group becomes one provider input item, so one provider final arrives per group. If a group has several utterances, the final attaches to the whole oldest group that lacks a final. Its first utterance gets the text, and the group's other utterances are marked covered, not missing. That way a multi-utterance group never shifts later turns' finals. - - In the realtime example, `u1`'s final arrives at 4300 while no utterance is open, and FIFO picks `u1`. - - If the caller had already barged in, `u2` would be open, yet FIFO would still give the final to `u1`, because `u2` is not committed. -- **`message` event:** follows the same attachment as the final it accompanies. -- **No final:** an utterance that never gets a final, and is not covered by its group's final, never gets a `message`. It gets `missing{evidence: transcript}` at close instead. +- **Provider finals with an item id (`source: provider`, LiveKit realtime, Pipecat with OpenAI):** set `message_id` to the event's item id and attach to the first utterance of the `message` that already mapped that id (see `message_id` below). +- **Provider finals without an item id (Gemini Live), or whose id was never mapped:** attach FIFO to the oldest committed caller utterance group (an `end_of_turn` list) that has no final yet. A provider transcribes committed audio in commit order, often after the next utterance has started, so "open at arrival" would pick the wrong one. + - **Assumption:** each committed `end_of_turn` group becomes one provider input item, so one provider final arrives per group. The final's text goes on the group's first utterance, and the group's other utterances are marked covered, not missing. That way a multi-utterance group never shifts later turns' finals. + - If the caller had already barged in, the barge-in utterance would be open, yet the final still goes to the older committed group. +- **No final:** an utterance that never gets a final, and is not covered by its group's final, gets `missing{evidence: transcript}` at close. It still has its `message` when the framework committed an item for it. **`message_id`:** -- **Caller:** a `message` event maps the utterance to the chat item. In cascaded pipelines it fires when the item commits (LiveKit `_conversationItemAdded`, Pipecat context aggregator push). In realtime it fires when `input_audio_transcription_completed` arrives with `itemId`. +- **Caller, cascaded:** a `message` event fires when the user item commits (LiveKit `_conversationItemAdded`, Pipecat context aggregator push). +- **Caller, realtime (decision E):** LiveKit adds the realtime user item to the conversation only when a final transcript arrives, so the SDK does not wait for it. At `_realtimeGenerationTaskImpl` entry it diffs `realtimeSession.chatCtx.items` for new user items and pairs the oldest new item with the oldest committed caller utterance group that has no `message` yet. It emits `message` there, after `end_of_turn`, even when no transcript ever arrives (`item_u2` in the realtime example). The later `_conversationItemAdded` for the same item id emits nothing. - **Agent:** a `message` event maps `/` to the assistant chat item, which is the realtime output item id in realtime pipelines, when LiveKit adds it (`_conversationItemAdded`). -- Lingua uses it to match conversation messages. +- **Use:** Lingua and the app use it to match conversation messages; Lingua copies it and derives nothing from it. **`response_id`:** - LiveKit (both pipelines): the `SpeechHandle.id` captured at `_pipelineReplyTaskImpl` / `_realtimeGenerationTaskImpl` entry. The provider response id stays on the `llm_response` span (`contrib.livekit.response_id`). - Pipecat: an SDK counter. -- `response_start.span_id` is the `llm_response` span started inside that handle's reply task. If a reply task starts several, it is the latest one started before playback starts. This is the only link from the timeline to the step tree. +- `response_start.span_id` is the `llm_response` span started inside that handle's reply task. If a reply task starts several, it is the latest one started before playback starts. This is the only link from the timeline to the LLM steps. **Agent speech to its generation:** `speech_start.response_id`. +**Tools:** `tool_call.response_id` is the generation that requested the tool. `tool_call_id` joins `tool_call`, `tool_result`, the tool row (`gen_ai.tool.call.id`) and the LLM `tool_calls[].id`. + **`in_reply_to`:** the caller utterances the framework handed to this generation. -- **Required on preemptive generations,** meaning any `response_start` emitted before its turn's `end_of_turn`. LiveKit: snapshot the uncommitted caller utterance ids in `onPreemptiveGeneration`, then attach them to the `response_start` of that preemptive speech handle's `llm_response` span, including preemptive responses that are later discarded. Otherwise it is recommended. +- **Required on preemptive generations,** meaning any `response_start` emitted before its turn's `end_of_turn`. LiveKit: snapshot the uncommitted caller utterance ids in `onPreemptiveGeneration`, then attach them to the `response_start` of that preemptive speech handle's `llm_response` span, including preemptive responses that are later discarded. Otherwise it is recommended, and both examples set it. - **Linking:** Lingua uses `in_reply_to` whenever it is present, and falls back to the `end_of_turn` window only when it is absent. ## Timing semantics @@ -142,6 +274,7 @@ The `voice.timeline` object: - `at_ms` is when the thing happened. For example, a VAD `speech_end` is emitted after the silence delay but dated at the real end, so it can be smaller than an earlier event's `at_ms`. - Lingua orders speech by `at_ms` and uses `seq` only to break ties and to define turn windows. - **Call clock:** `at_ms` is milliseconds since the recordings' `timeline.origin_unix_ms`, which is `snapshot().origin` in `@braintrust/audio`. Call ms `t` falls in a ready segment `s` at offset `t - s.timeline.recording_start_offset_ms`. +- **Span clock:** `metrics.start` / `metrics.end` on every row are unix seconds. On timeline rows they are derived from `at_ms`. On step rows they are the span's own wall-clock times; `metrics.start * 1000 - origin_unix_ms` approximates call ms, but Lingua never mixes the two. - **`basis` values:** - `audio`: computed from sample positions the recorder placed on the timeline, such as playout ranges from `playbackProgressed`. It matches the recording by construction. - `local`: `Date.now() - origin`, minus a known framework delay where one applies, such as the VAD silence duration. Expect event-loop jitter under 20 ms. @@ -161,6 +294,8 @@ The `voice.timeline` object: ## What integrations capture, and when +**Sketch.** No emission code exists yet; these tables are the plan, checked against `@livekit/agents` 1.9.2 sources. + ### LiveKit (JS, `js/src/instrumentation/livekit`) Citations are `@livekit/agents` 1.9.2 `src/` paths; `aa.ts` is `voice/agent_activity.ts`. @@ -168,8 +303,8 @@ Citations are `@livekit/agents` 1.9.2 `src/` paths; `aa.ts` is `voice/agent_acti | Event | Cascaded (`_pipelineReplyTaskImpl`, `aa.ts:3747`) | Realtime (`_realtimeGenerationTaskImpl`, `aa.ts:4496`) | | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | caller `speech_start` / `speech_end` | Per VAD segment: hook `onStartOfSpeech` (`aa.ts:2108`) / `onEndOfSpeech` (`aa.ts:2152`). Start = now − `speechDuration` − `inferenceDuration`; end = now − `silenceDuration` − `inferenceDuration`, as LiveKit computes for native `user_speaking` (`aa.ts:2111-2112`, `2156-2157`). `local`. | **Server turn detection on:** `onInputSpeechStarted` / `onInputSpeechStopped` (`aa.ts:1967`, `2000`); the events carry no provider ms (`llm/realtime.ts:16-18`). `u` at receipt (`local`); end = receipt − `voice.server_vad_silence_ms` only when that option is passed to the integration, else receipt, which includes the silence padding. Local VAD hooks are not used. **Server turn detection off:** local VAD hooks as in cascaded. | -| caller `transcript` | `user_input_transcribed` finals (`onFinalTranscript`, `aa.ts:2369`), `source: stt`, STT rule | `input_audio_transcription_completed`, `source: provider`, FIFO provider rule, then `message`; may arrive after `response_start` | -| `message` | Emitted only from the `AgentSession._conversationItemAdded` hook. The user item comes from the normal commit inside `_pipelineReplyTaskImpl` (`aa.ts:3975`, only when `speechHandle.scheduled`); a skipped turn (`commitSkippedUserTurn`, `aa.ts:3272`) emits no `message`. Assistant items also come through this hook (`aa.ts:4273`). | Same single hook: the user item when its transcript is inserted (`aa.ts:2054`), and the assistant item (`aa.ts:4827`). The transcription event emits only `transcript`, never `message`. | +| caller `transcript` | `user_input_transcribed` finals (`onFinalTranscript`, `aa.ts:2369`), `source: stt`, STT rule | `input_audio_transcription_completed`, `source: provider`, `message_id` = its `itemId`, attached through the earlier `message` (FIFO rule as fallback); may arrive after `response_start` | +| `message` | Emitted only from the `AgentSession._conversationItemAdded` hook. The user item comes from the normal commit inside `_pipelineReplyTaskImpl` (`aa.ts:3975`, only when `speechHandle.scheduled`); a skipped turn (`commitSkippedUserTurn`, `aa.ts:3272`) emits no `message`. Assistant items also come through this hook (`aa.ts:4273`). | Caller: at `_realtimeGenerationTaskImpl` entry, after `end_of_turn`, from the diff of `realtimeSession.chatCtx.items` (decision E), so items without a transcript still get one; the later `_conversationItemAdded` for that id (`aa.ts:2054`) is deduplicated. Agent: `_conversationItemAdded` (`aa.ts:4827`). The transcription event emits only `transcript`. | | `end_of_turn` | Committed turns only. At `onEndOfTurn` (`aa.ts:2690`), record the resolve time if the call returned `true` and neither `schedulingPaused` (`aa.ts:1230`) nor `newTurnsBlocked` (`aa.ts:550`) was set before it (the paused branch is `aa.ts:2700-2711`). Emit the event, with `at_ms` = that resolve time, when the turn's reply is scheduled (the user commit at `aa.ts:3975`). Turns that `userTurnCompleted` later drops emit **no** `end_of_turn`: the AMD `skipReply` (`aa.ts:2697`, committed via `aa.ts:3316-3319`), the non-interruptible drop (`aa.ts:3328-3333`), and `StopResponse` (`aa.ts:3388-3390`). Their utterances leave the pending list and appear in no `end_of_turn`, so Lingua attaches them to the turn open at their start. Lists the caller utterances since the previous `end_of_turn`. | **Server turn detection on:** there is no commit event (`userTurnCompleted` returns early, `aa.ts:3301-3302`). Emit at `_realtimeGenerationTaskImpl` entry for a generation that answers caller input, listing caller utterances since the previous `end_of_turn` that are already closed. Utterances still open at task entry are excluded and go to the next `end_of_turn`. Tool-continuation generations have no new utterances and emit none. **Server turn detection off:** as in cascaded. | | `response_start` | `llm_response` span start inside the handle's reply task. LiveKit's `speechHandleStorage` (`agent.ts:70`) is not exported, so the SDK sets its own async-local context with the `SpeechHandle` at `_pipelineReplyTaskImpl` entry and reads it in the span processor's `onStart`. `response_id` = `SpeechHandle.id`. For preemptive handles, `in_reply_to` comes from the `onPreemptiveGeneration` snapshot (`aa.ts:2433`, guard at `aa.ts:2440`). | The `_realtimeGenerationTaskImpl` hook reads its `inferenceSpan` argument (the `realtime_inference` span started at `aa.ts:4478`, exported as `llm_response`) and its handle; `response_id` = `SpeechHandle.id`. Preemptive generation is cascaded only (`aa.ts:2441`). | | `tool_call` / `tool_result` | `function_tool` span start and end; `cancelled` when the handle was interrupted first | same | @@ -185,19 +320,21 @@ Starting point: add `timeline.ts` beside `turns.ts`. It owns `seq`, the buffer, The timeline observer sits at the transport output processor, so agent events reflect audio actually written. -| Event | Cascaded | Realtime (OpenAI / Gemini services) | -| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | -| caller `transcript` | `TranscriptionFrame` (finalized), attached by the STT rule, `source: stt` | the service's input transcription, attached FIFO by the provider rule, `source: provider` | -| `message` | context aggregator push of the user or assistant message | realtime: only for providers with input item ids (OpenAI), at transcription completion; none for Gemini Live, which has no item ids | -| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | -| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | -| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | -| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | -| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | +| Event | Cascaded | Realtime (OpenAI / Gemini services) | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| caller `speech_start` / `speech_end` | `VADUserStartedSpeakingFrame` / `VADUserStoppedSpeakingFrame`, end dated stop time − VAD `stop_secs` (`local`); `audio` when the input sample clock is aligned | provider speech start/stop (`provider`, anchored as above) | +| caller `transcript` | `TranscriptionFrame` (finalized), attached by the STT rule, `source: stt` | the service's input transcription, attached FIFO by the provider rule, `source: provider` | +| `message` | context aggregator push of the user or assistant message | caller: at the provider's commit event with its item id (OpenAI `input_audio_buffer.committed`), with or without a transcript; none for Gemini Live, which has no item ids | +| `end_of_turn` | `UserStoppedSpeakingFrame` after the turn strategy commits; `utterance_ids` = VAD utterances since the previous `end_of_turn` | the service's commit event | +| `response_start` | `LLMFullResponseStartFrame`; `span_id` is the SDK's own `llm_response` span id, known when the span is created; `in_reply_to` = the same VAD utterances | same | +| `tool_call` / `tool_result` | `FunctionCallInProgressFrame` / `FunctionCallResultFrame` / `FunctionCallCancelFrame` | same | +| agent `speech_start` / `speech_end` | `TTSAudioRawFrame` bytes written by the output transport: start = call ms of the first write of a `context_id`; end = start + bytes written / (sample rate × channels × 2) × 1000, which also gives `played_ms`, both with `basis: audio`; `interrupted` when an `InterruptionFrame` stops it | same, per output item | +| agent `transcript` | `TTSTextFrame` words written before the stop, `source: playout` | same; provider transcript only as a fallback (`source: provider`) | ## What Lingua records versus infers +**Implemented** in the companion Lingua PR (builder, fixtures from these examples). + - **Records:** the events in `seq` order in `VoiceCall.events`, and recordings as in #486. - **Infers** (algorithm in the Lingua spec): - one `Speech` per utterance, with selections and `interrupted`; @@ -210,6 +347,8 @@ The timeline observer sits at the transport output processor, so agent events re ## Trajectory and app mapping +**Implemented in Lingua** (`VoiceCall.events`, `Speech` from the timeline); the app strip is a sketch. + `Trajectory.voice_calls[*]` keeps the #486 fields. `speech` and `recordings` are built from the timeline, so #22034 and #22036 work unchanged: the voice bar, click-to-step, play-from-message and highlighting. `VoiceCall.events` exposes the log for a latency strip: @@ -223,7 +362,7 @@ Thread messages still come from LLM span inputs and outputs. ## Worked example -`examples/cascaded.json` and `examples/realtime.json` encode the same fictional call. Row tree: +**Implemented:** `examples/cascaded.json` and `examples/realtime.json` encode the same fictional call and are parsed by the test. Row tree: ``` livekit.agent_session audio.recordings: call-0000 (13 s); voice.timeline_version 1 @@ -243,13 +382,13 @@ Concrete differences, same audio: | | Cascaded | Realtime | | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 4300 | -| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300, after `tool_call` at 4100, attached to `u1` by the rule | +| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 3600 (generation entry) | +| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300 with `message_id: item_u1`, after `tool_call` at 4100 | | Turn-1 latency, caller end → first agent audio | 2900 → 5300 = 2400 ms: 500 ms end-of-turn wait, 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 2900 → 5000 = 2100 ms: 700 ms to generation entry (`end_of_turn` at 3600; server-VAD silence and commit are not separable in LiveKit), 500 ms to the tool call, 500 ms tool, 400 ms to audio | | Agent utterance ids | `speech_2/0`, `speech_4/0`; `message` links `item_a1`, `item_a2` | same handle-based ids; `message` links the realtime output items `item_a1`, `item_a2` | | Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `speech_end` 7900 with `played_ms` 2900, and a `truncate` marker at 7900 with the same 2900 | | Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is dated receipt 3400 − 500 ms server-VAD silence = 2900 | -| Caller's barge-in text | `stt` "Nine thirty works." | none: `missing{evidence: transcript}`, and the `rt-3` input user content is `""` | +| Caller's barge-in text | `stt` "Nine thirty works." | none: `message` `item_u2` at 8600 but `missing{evidence: transcript}`; the `rt-3` input user content is `""` | | Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | | STT/TTS steps | `tts-1`, `tts-2` | none | @@ -266,7 +405,7 @@ Concrete differences, same audio: - **Barge-in truncation:** handled. The planned text stays only on the LLM span. - **Overlap:** intervals overlap freely. -- **Late or missing realtime transcripts:** a late final attaches by `message_id`, or by the attachment rule when there is none. A missing one gets a `missing` event. Weakness: without item ids (Gemini Live, cascaded STT), attachment is a heuristic and can pick the wrong segment of a multi-segment turn. +- **Late or missing realtime transcripts:** a late final attaches by `message_id`, or by the FIFO rule when there is none. A missing one gets a `missing` event, and the utterance still has its `message` from generation entry. Weakness: without item ids (Gemini Live, cascaded STT), attachment is a heuristic and can pick the wrong segment of a multi-segment turn. - **Failed or omitted segments:** speech keeps its times but has no selection over the gap. - **Unknown clock offset:** `local` events skew against `audio` events unmeasured. A `clock_offset` event is left as an open question. - **Call in progress, or a crash:** the latest ≤5 s are not yet flushed. With no `final` chunk, Lingua warns and closes open speech. @@ -278,8 +417,10 @@ Concrete differences, same audio: ## Implementation starting point +Step 1 is implemented (zod, examples, test); steps 2, 3 and 5 are sketches; step 4 is implemented in the companion Lingua PR. + 1. Lock the schema in `call-timeline.ts`, including `basis`, flush and retry, and the transcript-attachment rule. -2. LiveKit: add `timeline.ts`, the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`), and a playback hook installed independently of audio capture. Emit from `onEndOfTurn` (when it resolves `true` and commits), `onPreemptiveGeneration` (the `in_reply_to` snapshot), `_pipelineReplyTaskImpl` / `_realtimeGenerationTaskImpl` entry (the speech handle, and realtime `end_of_turn`), the `RealtimeObserver` listeners, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. +2. LiveKit: add `timeline.ts`, the VAD hooks (`onStartOfSpeech` / `onEndOfSpeech`), and a playback hook installed independently of audio capture. Emit from `onEndOfTurn` (when it resolves `true` and commits), `onPreemptiveGeneration` (the `in_reply_to` snapshot), `_pipelineReplyTaskImpl` / `_realtimeGenerationTaskImpl` entry (the speech handle, and realtime `end_of_turn` and caller `message`), the `RealtimeObserver` listeners, `_conversationItemAdded`, and `onStart`/`onEnd` for `llm_response` and `function_tool`. Set `voice.timeline_version` and `voice.pipeline` at `_startImpl`. Keep existing spans and selections during the transition. 3. At close, emit `missing` events, then `call_end`, and flush the `final` chunk. 4. Lingua: the builder exists in the companion PR; extend it with the app allow-list keys after #486 is rebased. 5. Pipecat: port `timeline.ts` to the Python voice package following the table above. @@ -290,3 +431,4 @@ Concrete differences, same audio: - Are 5 s and 64 events the right flush defaults? - Should agent `transcript` carry word timings when the TTS provides alignments? - Is a `clock_offset` event worth adding, to measure `local` against `audio`? +- **Recommended extension: message ids on LLM spans.** Thread messages come from LLM span inputs, which carry no chat item ids, so the app can join a thread message to its audio only through `Speech.step_ids` today. Logging `voice.context_message_ids` (one id per input message, in order) and `voice.output_message_id` on `llm_response` spans would let click-message-to-audio join on `message.message_id` directly. Not added here: it touches the LLM span contract and the realtime input merging, and step links already cover the #22036 flows. diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json index 51e086f4e..81ded4f30 100644 --- a/docs/voice-schema/call-timeline/examples/realtime.json +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -373,13 +373,22 @@ "seq": 3, "at_ms": 3600, "basis": "local", + "kind": "message", + "speaker": "user", + "message_id": "item_u1", + "utterance_ids": ["u1"] + }, + { + "seq": 4, + "at_ms": 3600, + "basis": "local", "kind": "response_start", "response_id": "speech_1", "span_id": "rt-1", "in_reply_to": ["u1"] }, { - "seq": 4, + "seq": 5, "at_ms": 4100, "basis": "local", "kind": "tool_call", @@ -389,7 +398,7 @@ "span_id": "tool-1" }, { - "seq": 5, + "seq": 6, "at_ms": 4300, "basis": "local", "kind": "transcript", @@ -397,16 +406,8 @@ "utterance_id": "u1", "final": true, "text": "Hi, can I move my Thursday cleaning to Friday morning?", - "source": "provider" - }, - { - "seq": 6, - "at_ms": 4300, - "basis": "local", - "kind": "message", - "speaker": "user", - "message_id": "item_u1", - "utterance_ids": ["u1"] + "source": "provider", + "message_id": "item_u1" }, { "seq": 7, @@ -523,13 +524,22 @@ "seq": 17, "at_ms": 8600, "basis": "local", + "kind": "message", + "speaker": "user", + "message_id": "item_u2", + "utterance_ids": ["u2"] + }, + { + "seq": 18, + "at_ms": 8600, + "basis": "local", "kind": "response_start", "response_id": "speech_3", "span_id": "rt-3", "in_reply_to": ["u2"] }, { - "seq": 18, + "seq": 19, "at_ms": 8900, "basis": "local", "kind": "tool_call", @@ -539,7 +549,7 @@ "span_id": "tool-2" }, { - "seq": 19, + "seq": 20, "at_ms": 9300, "basis": "local", "kind": "tool_result", @@ -548,7 +558,7 @@ "outcome": "ok" }, { - "seq": 20, + "seq": 21, "at_ms": 9300, "basis": "local", "kind": "response_start", @@ -557,7 +567,7 @@ "in_reply_to": ["u2"] }, { - "seq": 21, + "seq": 22, "at_ms": 9700, "basis": "audio", "kind": "speech_start", @@ -566,7 +576,7 @@ "response_id": "speech_4" }, { - "seq": 22, + "seq": 23, "at_ms": 12500, "basis": "audio", "kind": "speech_end", @@ -576,7 +586,7 @@ "played_ms": 2800 }, { - "seq": 23, + "seq": 24, "at_ms": 12500, "basis": "local", "kind": "transcript", @@ -587,7 +597,7 @@ "source": "playout" }, { - "seq": 24, + "seq": 25, "at_ms": 12500, "basis": "local", "kind": "message", @@ -596,7 +606,7 @@ "utterance_ids": ["speech_4/0"] }, { - "seq": 25, + "seq": 26, "at_ms": 13000, "basis": "local", "kind": "missing", @@ -605,7 +615,7 @@ "reason": "input_transcription_not_received" }, { - "seq": 26, + "seq": 27, "at_ms": 13000, "basis": "local", "kind": "call_end", diff --git a/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts index b8b41057b..f774c93a9 100644 --- a/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts +++ b/js/src/instrumentation/livekit/voice-schema/call-timeline.test.ts @@ -72,4 +72,19 @@ describe.each(["cascaded.json", "realtime.json"])("%s", (file) => { ); expect(referenced.filter((id) => !started.has(id))).toEqual([]); }); + + it("maps every committed caller utterance to a message", () => { + const log = events(file); + const mapped = new Set( + log.flatMap((event) => + event.kind === "message" && event.speaker === "user" + ? event.utterance_ids + : [], + ), + ); + const committed = log.flatMap((event) => + event.kind === "end_of_turn" ? event.utterance_ids : [], + ); + expect(committed.filter((id) => !mapped.has(id))).toEqual([]); + }); }); From fb273cc0ce976849752c96933eee455eb9640bd6 Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:01:33 +0000 Subject: [PATCH 10/11] chore: retime realtime call timeline turn 2 to the padding rule Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 44 +++++++++---------- .../call-timeline/examples/realtime.json | 29 ++++++------ 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 64a81502d..79411e972 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -115,8 +115,8 @@ tool_call.span_id / tool_result.span_id -> tool row; tool row "gen_ai.tool.call ```jsonc -// realtime: call span adds "voice.server_vad_silence_ms": 500 -{ "seq": 1, "at_ms": 2900, "basis": "local", "kind": "speech_end", "speaker": "user", "utterance_id": "u1" }, // receipt 3400 - 500 +// realtime: call span { "voice.timeline_version": 1, "voice.pipeline": "realtime" }; no silence value was given +{ "seq": 1, "at_ms": 3400, "basis": "local", "kind": "speech_end", "speaker": "user", "utterance_id": "u1" }, // receipt, incl. 500 ms padding { "seq": 2, "at_ms": 3600, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u1"] }, // generation entry, not a commit { "seq": 3, "at_ms": 3600, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u1", "utterance_ids": ["u1"] }, { "seq": 4, "at_ms": 3600, "basis": "local", "kind": "response_start", "response_id": "speech_1", "span_id": "rt-1", "in_reply_to": ["u1"] }, @@ -131,21 +131,21 @@ tool_call.span_id / tool_result.span_id -> tool row; tool row "gen_ai.tool.call { "seq": 12, "at_ms": 7900, "basis": "local", "kind": "truncate", "utterance_id": "speech_2/0", "audio_end_ms": 2900 }, // realtime only; = played_ms { "seq": 13, "at_ms": 7900, "basis": "local", "kind": "transcript", "speaker": "agent", "utterance_id": "speech_2/0", "final": true, "text": "I have nine thirty or eleven on Fri", "source": "playout" }, -{ "seq": 17, "at_ms": 8600, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u2", "utterance_ids": ["u2"] }, // no transcript +{ "seq": 17, "at_ms": 9000, "basis": "local", "kind": "message", "speaker": "user", "message_id": "item_u2", "utterance_ids": ["u2"] }, // no transcript { "seq": 26, "at_ms": 13000, "basis": "local", "kind": "missing", "utterance_id": "u2", "evidence": "transcript", "reason": "input_transcription_not_received" } // replaces the cascaded seq 14 transcript ``` Cascaded vs realtime: what changes in the payload -| | Cascaded | Realtime | -| ------------------------ | ---------------------------------------- | ------------------------------------------------------------------------------------------ | -| Call span | `voice.pipeline: "cascaded"` | `"realtime"`, plus `voice.server_vad_silence_ms` when the integration is given it | -| Caller transcript | `source: "stt"`, before `response_start` | `source: "provider"` with `message_id`; may arrive after `tool_call`, or never (`missing`) | -| `end_of_turn.at_ms` | turn detector resolve time (3400) | entry of the answering generation (3600) | -| Caller `message` | at the user commit | at generation entry, even without a transcript (`item_u2`) | -| Interrupted agent speech | `speech_end` with `played_ms` only | same, plus a `truncate` marker equal to `played_ms` (LiveKit) | -| Step rows | `llm_response` + `tts` rows | `llm_response` rows only (`contrib.livekit.response_id` = provider response id) | +| | Cascaded | Realtime | +| ------------------------ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Call span | `voice.pipeline: "cascaded"` | `"realtime"`; `voice.server_vad_silence_ms` only when the integration is given it (not in the example) | +| Caller transcript | `source: "stt"`, before `response_start` | `source: "provider"` with `message_id`; may arrive after `tool_call`, or never (`missing`) | +| `end_of_turn.at_ms` | turn detector resolve time (3400) | entry of the answering generation (3600) | +| Caller `message` | at the user commit | at generation entry, even without a transcript (`item_u2`) | +| Interrupted agent speech | `speech_end` with `played_ms` only | same, plus a `truncate` marker equal to `played_ms` (LiveKit) | +| Step rows | `llm_response` + `tts` rows | `llm_response` rows only (`contrib.livekit.response_id` = provider response id) | Implemented: zod types (`js/src/instrumentation/livekit/voice-schema/call-timeline.ts`), examples, test. Sketch: LiveKit/Pipecat emission. Lingua mapping: implemented in the companion Lingua PR (`specs/voice-schema-call-timeline.md`, branch `kevchoi/voice-schema-call-timeline`). @@ -380,17 +380,17 @@ livekit.agent_session audio.recordings: call-0000 (13 s); voice.timel Concrete differences, same audio: -| | Cascaded | Realtime | -| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 3600 (generation entry) | -| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300 with `message_id: item_u1`, after `tool_call` at 4100 | -| Turn-1 latency, caller end → first agent audio | 2900 → 5300 = 2400 ms: 500 ms end-of-turn wait, 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 2900 → 5000 = 2100 ms: 700 ms to generation entry (`end_of_turn` at 3600; server-VAD silence and commit are not separable in LiveKit), 500 ms to the tool call, 500 ms tool, 400 ms to audio | -| Agent utterance ids | `speech_2/0`, `speech_4/0`; `message` links `item_a1`, `item_a2` | same handle-based ids; `message` links the realtime output items `item_a1`, `item_a2` | -| Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `speech_end` 7900 with `played_ms` 2900, and a `truncate` marker at 7900 with the same 2900 | -| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is dated receipt 3400 − 500 ms server-VAD silence = 2900 | -| Caller's barge-in text | `stt` "Nine thirty works." | none: `message` `item_u2` at 8600 but `missing{evidence: transcript}`; the `rt-3` input user content is `""` | -| Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | -| STT/TTS steps | `tts-1`, `tts-2` | none | +| | Cascaded | Realtime | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Caller utterance ids | `u1`, `u2` | `u1`, `u2`; `message` maps `u1` → `item_u1` at 3600 (generation entry) | +| Caller transcript of turn 1 | `stt` at 3450; `message` links `item_u1` at commit | `provider` at 4300 with `message_id: item_u1`, after `tool_call` at 4100 | +| Turn-1 latency, caller end → first agent audio | 2900 → 5300 = 2400 ms: 500 ms end-of-turn wait, 50 ms STT, 650 ms LLM, 500 ms tool, 500 ms LLM, 200 ms TTS | 3400 → 5000 = 1600 ms measured from the padded end (2100 ms from the audible end at 2900): 200 ms to generation entry (`end_of_turn` at 3600), 500 ms to the tool call, 500 ms tool, 400 ms to audio | +| Agent utterance ids | `speech_2/0`, `speech_4/0`; `message` links `item_a1`, `item_a2` | same handle-based ids; `message` links the realtime output items `item_a1`, `item_a2` | +| Barge-in | agent `speech_end` 7900, `played_ms` 2600 | `speech_end` 7900 with `played_ms` 2900, and a `truncate` marker at 7900 with the same 2900 | +| Out-of-order evidence | `u2` STT final arrives at 8520, before the VAD stop is emitted; `speech_end` (seq 15) is dated 8500 | caller `speech_end` is the receipt time (3400, 9000) and includes the 500 ms server-VAD padding, because no silence value was given | +| Caller's barge-in text | `stt` "Nine thirty works." | none: `message` `item_u2` at 9000 but `missing{evidence: transcript}`; the `rt-3` input user content is `""` | +| Agent text actually said | `playout` "…on Fri"; `llm-2` output holds the planned sentence | `playout` "…on Fri"; `rt-2` output already truncated | +| STT/TTS steps | `tts-1`, `tts-2` | none | ## Customer usefulness diff --git a/docs/voice-schema/call-timeline/examples/realtime.json b/docs/voice-schema/call-timeline/examples/realtime.json index 81ded4f30..d672e4d4c 100644 --- a/docs/voice-schema/call-timeline/examples/realtime.json +++ b/docs/voice-schema/call-timeline/examples/realtime.json @@ -54,8 +54,7 @@ } ], "voice.timeline_version": 1, - "voice.pipeline": "realtime", - "voice.server_vad_silence_ms": 500 + "voice.pipeline": "realtime" } }, { @@ -193,8 +192,8 @@ "type": "llm" }, "metrics": { - "start": 1791830008.6, - "end": 1791830008.9 + "start": 1791830009.0, + "end": 1791830009.3 }, "input": [ { @@ -247,8 +246,8 @@ "type": "tool" }, "metrics": { - "start": 1791830008.9, - "end": 1791830009.3 + "start": 1791830009.3, + "end": 1791830009.7 }, "input": { "appointment_id": "apt_481", @@ -273,7 +272,7 @@ "type": "llm" }, "metrics": { - "start": 1791830009.3, + "start": 1791830009.7, "end": 1791830012.5 }, "input": [ @@ -356,7 +355,7 @@ }, { "seq": 1, - "at_ms": 2900, + "at_ms": 3400, "basis": "local", "kind": "speech_end", "speaker": "user", @@ -507,7 +506,7 @@ }, { "seq": 15, - "at_ms": 8500, + "at_ms": 9000, "basis": "local", "kind": "speech_end", "speaker": "user", @@ -515,14 +514,14 @@ }, { "seq": 16, - "at_ms": 8600, + "at_ms": 9000, "basis": "local", "kind": "end_of_turn", "utterance_ids": ["u2"] }, { "seq": 17, - "at_ms": 8600, + "at_ms": 9000, "basis": "local", "kind": "message", "speaker": "user", @@ -531,7 +530,7 @@ }, { "seq": 18, - "at_ms": 8600, + "at_ms": 9000, "basis": "local", "kind": "response_start", "response_id": "speech_3", @@ -540,7 +539,7 @@ }, { "seq": 19, - "at_ms": 8900, + "at_ms": 9300, "basis": "local", "kind": "tool_call", "response_id": "speech_3", @@ -550,7 +549,7 @@ }, { "seq": 20, - "at_ms": 9300, + "at_ms": 9700, "basis": "local", "kind": "tool_result", "tool_call_id": "call_2", @@ -559,7 +558,7 @@ }, { "seq": 21, - "at_ms": 9300, + "at_ms": 9700, "basis": "local", "kind": "response_start", "response_id": "speech_4", From a10e64cb75ed0282eed9278ee61af0a2af753c9d Mon Sep 17 00:00:00 2001 From: kevchoi <16812845+kevchoi@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:11:46 +0000 Subject: [PATCH 11/11] chore: mark call span audio input optional in schema glance Co-Authored-By: Claude Opus 5.5 --- docs/voice-schema/call-timeline/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/voice-schema/call-timeline/README.md b/docs/voice-schema/call-timeline/README.md index 79411e972..8b927906a 100644 --- a/docs/voice-schema/call-timeline/README.md +++ b/docs/voice-schema/call-timeline/README.md @@ -24,7 +24,7 @@ Evidence lives in `metadata["voice.events"]` on `voice.timeline` child rows; the ```ts // Call span (root). span_attributes = { name: "livekit.agent_session", type: "task" } type CallRow = { - input: { audio: Record } // shared @braintrust/audio, ready segments only + input?: { audio: Record } // shared @braintrust/audio, ready segments; absent when capture is off metadata: { "audio.recordings"?: AudioRecording[] // shared; absent when audio capture is off "voice.timeline_version": 1 // required; selects the timeline path in Lingua