From 71580439bbf2a422e00cc2863931fd8d4693bd53 Mon Sep 17 00:00:00 2001 From: klopez4212 Date: Sat, 4 Jul 2026 16:47:38 +0100 Subject: [PATCH] fix(dictation): use client_secrets API, track transcripts by item_id - Switch relay from /v1/realtime/sessions to /v1/realtime/client_secrets with the wrapped { session: { ... } } request shape per OpenAI's current WebRTC guide. The old endpoint returns non-2xx, breaking dictation. - Redesign TranscriptSegmentState to track per-item segments keyed by item_id. Completed events for different turns can arrive out of order; reconciling by item_id preserves utterance ordering and prevents text reordering or partial-turn drops during fast consecutive speech. Signed-off-by: klopez4212 --- crates/buzz-relay/src/api/transcribe.rs | 21 ++-- .../dictation/hooks/useRealtimeDictation.ts | 4 +- .../dictation/lib/realtimeAudio.test.mjs | 112 ++++++++++++++---- .../features/dictation/lib/realtimeAudio.ts | 94 +++++++++++---- 4 files changed, 174 insertions(+), 57 deletions(-) diff --git a/crates/buzz-relay/src/api/transcribe.rs b/crates/buzz-relay/src/api/transcribe.rs index 9210938cc..ff8af8a99 100644 --- a/crates/buzz-relay/src/api/transcribe.rs +++ b/crates/buzz-relay/src/api/transcribe.rs @@ -20,7 +20,8 @@ use crate::state::AppState; use super::api_error; -const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions"; +const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = + "https://api.openai.com/v1/realtime/client_secrets"; const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1"; /// Response for `GET /transcribe/status`. @@ -76,17 +77,19 @@ pub async fn create_transcribe_session( let client = reqwest::Client::new(); let response = client - .post(OPENAI_REALTIME_SESSIONS_URL) + .post(OPENAI_REALTIME_CLIENT_SECRETS_URL) .header("Authorization", format!("Bearer {api_key}")) .header("Content-Type", "application/json") .json(&serde_json::json!({ - "model": "gpt-4o-mini-realtime-preview", - "modalities": ["text"], - "input_audio_transcription": { - "model": model, - }, - "turn_detection": { - "type": "server_vad", + "session": { + "model": "gpt-4o-mini-realtime-preview", + "modalities": ["text"], + "input_audio_transcription": { + "model": model, + }, + "turn_detection": { + "type": "server_vad", + } } })) .timeout(std::time::Duration::from_secs(10)) diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts index ce289e279..a65048d06 100644 --- a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -15,6 +15,7 @@ import { createPeerConnection, createTranscriptSegmentState, flushAudioBuffer, + getTranscriptText, mergeTranscriptEvent, } from "../lib/realtimeAudio"; @@ -116,8 +117,7 @@ export function useRealtimeDictation({ return; } - const prevText = - segmentStateRef.current.committed + segmentStateRef.current.pendingDelta; + const prevText = getTranscriptText(segmentStateRef.current); const merged = mergeTranscriptEvent(segmentStateRef.current, event); if (merged === prevText) return; diff --git a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs index e2fe52cb5..abfb26ecd 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs +++ b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs @@ -1,11 +1,6 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; -// We test the pure logic via dynamic import of the TS source compiled by the -// test runner (vitest/tsx). Since this is an .mjs file run by the Node test -// runner through the desktop vitest config, import the built output or use -// a direct TS import if the runner supports it. - // Inline the logic to keep the test self-contained and avoid bundler issues. const TRANSCRIPT_DELTA_EVENT = "conversation.item.input_audio_transcription.delta"; @@ -13,36 +8,55 @@ const TRANSCRIPT_COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed"; function createTranscriptSegmentState() { - return { committed: "", pendingDelta: "" }; + return { itemOrder: [], items: new Map() }; +} + +function getOrCreateItem(state, itemId) { + let seg = state.items.get(itemId); + if (!seg) { + seg = { pending: "", finalized: null }; + state.items.set(itemId, seg); + state.itemOrder.push(itemId); + } + return seg; } function mergeTranscriptEvent(state, event) { + const itemId = event.item_id ?? "__default__"; + if (event.type === TRANSCRIPT_DELTA_EVENT) { + const seg = getOrCreateItem(state, itemId); const delta = event.delta ?? ""; if (delta) { - state.pendingDelta += delta; + seg.pending += delta; } } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { - const finalText = event.transcript ?? ""; - const separator = state.committed && finalText ? "" : ""; - state.committed = state.committed + separator + finalText; - state.pendingDelta = ""; + const seg = getOrCreateItem(state, itemId); + seg.finalized = event.transcript ?? ""; } - return state.committed + state.pendingDelta; + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + result += seg.finalized ?? seg.pending; + } + return result; } describe("mergeTranscriptEvent", () => { - it("accumulates delta events", () => { + it("accumulates delta events for a single item", () => { const state = createTranscriptSegmentState(); const r1 = mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", delta: "hello ", }); assert.equal(r1, "hello "); const r2 = mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", delta: "world", }); assert.equal(r2, "hello world"); @@ -52,59 +66,115 @@ describe("mergeTranscriptEvent", () => { const state = createTranscriptSegmentState(); mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", delta: "hello world", }); - // Completed event carries corrected/punctuated version const result = mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", transcript: "Hello, world.", }); assert.equal(result, "Hello, world."); - assert.equal(state.committed, "Hello, world."); - assert.equal(state.pendingDelta, ""); }); - it("handles multiple segments sequentially", () => { + it("handles multiple items in order", () => { const state = createTranscriptSegmentState(); - // First segment + // First item mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", delta: "first ", }); mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", transcript: "First. ", }); - // Second segment + // Second item mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_2", delta: "second", }); - assert.equal(state.committed + state.pendingDelta, "First. second"); + assert.equal(state.items.get("item_2").pending, "second"); const result = mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", transcript: "Second.", }); assert.equal(result, "First. Second."); }); + it("handles out-of-order completed events by item id", () => { + const state = createTranscriptSegmentState(); + + // Both items start with deltas + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: "first", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_2", + delta: "second", + }); + + // item_2 completes before item_1 + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Second. ", + }); + + // item_1 still shows pending + let result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", + delta: " more", + }); + assert.equal(result, "first moreSecond. "); + + // item_1 finally completes + result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "First more. ", + }); + assert.equal(result, "First more. Second. "); + }); + it("does not duplicate text on completed event", () => { const state = createTranscriptSegmentState(); mergeTranscriptEvent(state, { type: TRANSCRIPT_DELTA_EVENT, + item_id: "item_1", delta: "hello world", }); - // Without the fix, this would append: "hello worldHello, world." const result = mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", transcript: "Hello, world.", }); assert.equal(result, "Hello, world."); }); + + it("falls back to __default__ when item_id is missing", () => { + const state = createTranscriptSegmentState(); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "no id", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "No id.", + }); + assert.equal(result, "No id."); + }); }); diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts index bed28a727..4aaaeb148 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.ts +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -56,49 +56,93 @@ export async function connectPeerConnection(options: { } /** - * State for tracking the current transcription segment. Completed events - * carry the final text for the same segment that prior deltas built up, - * potentially with corrections/punctuation. We track the delta accumulation - * so we can replace it with the finalized text on completion. + * Per-item tracking for a single transcription turn. OpenAI's Realtime API + * sends delta and completed events tagged with an `item_id`; completed events + * for different turns can arrive out of order, so we reconcile by item id. */ -export interface TranscriptSegmentState { - /** Text committed from previous (completed) segments. */ - committed: string; - /** Accumulated delta text for the in-progress segment. */ - pendingDelta: string; -} - -export function createTranscriptSegmentState(): TranscriptSegmentState { - return { committed: "", pendingDelta: "" }; +interface ItemSegment { + /** Accumulated delta text (replaced by finalized on completion). */ + pending: string; + /** Finalized text (set once the completed event arrives). */ + finalized: string | null; } /** - * Merge a transcript event into the segment state. + * State for tracking transcription segments keyed by item id. + * Maintains insertion order so the full transcript is reconstructed + * in the order items were first seen. + */ +export interface TranscriptSegmentState { + /** Ordered item ids (insertion order = utterance order). */ + itemOrder: string[]; + /** Per-item segment data. */ + items: Map; +} + +export function createTranscriptSegmentState(): TranscriptSegmentState { + return { itemOrder: [], items: new Map() }; +} + +/** Get the current full transcript text from segment state. */ +export function getTranscriptText(state: TranscriptSegmentState): string { + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + result += seg.finalized ?? seg.pending; + } + return result; +} + +/** Internal: get or create the segment for an item. */ +function getOrCreateItem( + state: TranscriptSegmentState, + itemId: string, +): ItemSegment { + let seg = state.items.get(itemId); + if (!seg) { + seg = { pending: "", finalized: null }; + state.items.set(itemId, seg); + state.itemOrder.push(itemId); + } + return seg; +} + +/** + * Merge a transcript event into the segment state, keyed by `item_id`. * - * - Delta events: append to `pendingDelta`. - * - Completed events: replace `pendingDelta` with the finalized transcript, - * then commit it (move to `committed` and reset `pendingDelta`). + * - Delta events: append to the item's `pending` text. + * - Completed events: store `finalized` text, replacing accumulated deltas. * - * Returns the full merged text (committed + pending). + * Returns the full merged text across all items in order. */ export function mergeTranscriptEvent( state: TranscriptSegmentState, event: TranscriptEvent, ): string { + // Use item_id from the event; fall back to a synthetic key for events + // that lack one (shouldn't happen in practice, but be defensive). + const itemId = event.item_id ?? "__default__"; + if (event.type === TRANSCRIPT_DELTA_EVENT) { + const seg = getOrCreateItem(state, itemId); const delta = event.delta ?? ""; if (delta) { - state.pendingDelta += delta; + seg.pending += delta; } } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { - const finalText = event.transcript ?? ""; - // Replace the accumulated deltas with the finalized text, then commit. - const separator = state.committed && finalText ? "" : ""; - state.committed = state.committed + separator + finalText; - state.pendingDelta = ""; + const seg = getOrCreateItem(state, itemId); + seg.finalized = event.transcript ?? ""; } - return state.committed + state.pendingDelta; + // Reconstruct full text from all items in order. + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + result += seg.finalized ?? seg.pending; + } + return result; } // ── Audio buffer capture ──────────────────────────────────────────────────