fix(dictation): use client_secrets API, track transcripts by item_id

- Switch relay from /v1/realtime/sessions to /v1/realtime/client_secrets
  with the wrapped { session: { ... } } request shape per OpenAI's current
  WebRTC guide. The old endpoint returns non-2xx, breaking dictation.

- Redesign TranscriptSegmentState to track per-item segments keyed by
  item_id. Completed events for different turns can arrive out of order;
  reconciling by item_id preserves utterance ordering and prevents text
  reordering or partial-turn drops during fast consecutive speech.

Signed-off-by: klopez4212 <klopez4212@gmail.com>

crates/buzz-relay/src/api/transcribe.rs

@@ -20,7 +20,8 @@ use crate::state::AppState;
 
 use super::api_error;
 
-const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions";
+const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str =
+    "https://api.openai.com/v1/realtime/client_secrets";
 const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1";
 
 /// Response for `GET /transcribe/status`.
@@ -76,17 +77,19 @@ pub async fn create_transcribe_session(
 
     let client = reqwest::Client::new();
     let response = client
-        .post(OPENAI_REALTIME_SESSIONS_URL)
+        .post(OPENAI_REALTIME_CLIENT_SECRETS_URL)
         .header("Authorization", format!("Bearer {api_key}"))
         .header("Content-Type", "application/json")
         .json(&serde_json::json!({
-            "model": "gpt-4o-mini-realtime-preview",
-            "modalities": ["text"],
-            "input_audio_transcription": {
-                "model": model,
-            },
-            "turn_detection": {
-                "type": "server_vad",
+            "session": {
+                "model": "gpt-4o-mini-realtime-preview",
+                "modalities": ["text"],
+                "input_audio_transcription": {
+                    "model": model,
+                },
+                "turn_detection": {
+                    "type": "server_vad",
+                }
             }
         }))
         .timeout(std::time::Duration::from_secs(10))

desktop/src/features/dictation/hooks/useRealtimeDictation.ts

@@ -15,6 +15,7 @@ import {
   createPeerConnection,
   createTranscriptSegmentState,
   flushAudioBuffer,
+  getTranscriptText,
   mergeTranscriptEvent,
 } from "../lib/realtimeAudio";
 
@@ -116,8 +117,7 @@ export function useRealtimeDictation({
       return;
     }
 
-    const prevText =
-      segmentStateRef.current.committed + segmentStateRef.current.pendingDelta;
+    const prevText = getTranscriptText(segmentStateRef.current);
     const merged = mergeTranscriptEvent(segmentStateRef.current, event);
 
     if (merged === prevText) return;

desktop/src/features/dictation/lib/realtimeAudio.test.mjs

@@ -1,11 +1,6 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 
-// We test the pure logic via dynamic import of the TS source compiled by the
-// test runner (vitest/tsx). Since this is an .mjs file run by the Node test
-// runner through the desktop vitest config, import the built output or use
-// a direct TS import if the runner supports it.
-
 // Inline the logic to keep the test self-contained and avoid bundler issues.
 const TRANSCRIPT_DELTA_EVENT =
   "conversation.item.input_audio_transcription.delta";
@@ -13,36 +8,55 @@ const TRANSCRIPT_COMPLETED_EVENT =
   "conversation.item.input_audio_transcription.completed";
 
 function createTranscriptSegmentState() {
-  return { committed: "", pendingDelta: "" };
+  return { itemOrder: [], items: new Map() };
+}
+
+function getOrCreateItem(state, itemId) {
+  let seg = state.items.get(itemId);
+  if (!seg) {
+    seg = { pending: "", finalized: null };
+    state.items.set(itemId, seg);
+    state.itemOrder.push(itemId);
+  }
+  return seg;
 }
 
 function mergeTranscriptEvent(state, event) {
+  const itemId = event.item_id ?? "__default__";
+
   if (event.type === TRANSCRIPT_DELTA_EVENT) {
+    const seg = getOrCreateItem(state, itemId);
     const delta = event.delta ?? "";
     if (delta) {
-      state.pendingDelta += delta;
+      seg.pending += delta;
     }
   } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
-    const finalText = event.transcript ?? "";
-    const separator = state.committed && finalText ? "" : "";
-    state.committed = state.committed + separator + finalText;
-    state.pendingDelta = "";
+    const seg = getOrCreateItem(state, itemId);
+    seg.finalized = event.transcript ?? "";
   }
 
-  return state.committed + state.pendingDelta;
+  let result = "";
+  for (const id of state.itemOrder) {
+    const seg = state.items.get(id);
+    if (!seg) continue;
+    result += seg.finalized ?? seg.pending;
+  }
+  return result;
 }
 
 describe("mergeTranscriptEvent", () => {
-  it("accumulates delta events", () => {
+  it("accumulates delta events for a single item", () => {
     const state = createTranscriptSegmentState();
     const r1 = mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
       delta: "hello ",
     });
     assert.equal(r1, "hello ");
 
     const r2 = mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
       delta: "world",
     });
     assert.equal(r2, "hello world");
@@ -52,59 +66,115 @@ describe("mergeTranscriptEvent", () => {
     const state = createTranscriptSegmentState();
     mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
       delta: "hello world",
     });
 
-    // Completed event carries corrected/punctuated version
     const result = mergeTranscriptEvent(state, {
       type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_1",
       transcript: "Hello, world.",
     });
     assert.equal(result, "Hello, world.");
-    assert.equal(state.committed, "Hello, world.");
-    assert.equal(state.pendingDelta, "");
   });
 
-  it("handles multiple segments sequentially", () => {
+  it("handles multiple items in order", () => {
     const state = createTranscriptSegmentState();
 
-    // First segment
+    // First item
     mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
       delta: "first ",
     });
     mergeTranscriptEvent(state, {
       type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_1",
       transcript: "First. ",
     });
 
-    // Second segment
+    // Second item
     mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_2",
       delta: "second",
     });
-    assert.equal(state.committed + state.pendingDelta, "First. second");
+    assert.equal(state.items.get("item_2").pending, "second");
 
     const result = mergeTranscriptEvent(state, {
       type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_2",
       transcript: "Second.",
     });
     assert.equal(result, "First. Second.");
   });
 
+  it("handles out-of-order completed events by item id", () => {
+    const state = createTranscriptSegmentState();
+
+    // Both items start with deltas
+    mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
+      delta: "first",
+    });
+    mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_2",
+      delta: "second",
+    });
+
+    // item_2 completes before item_1
+    mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_2",
+      transcript: "Second. ",
+    });
+
+    // item_1 still shows pending
+    let result = mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
+      delta: " more",
+    });
+    assert.equal(result, "first moreSecond. ");
+
+    // item_1 finally completes
+    result = mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_1",
+      transcript: "First more. ",
+    });
+    assert.equal(result, "First more. Second. ");
+  });
+
   it("does not duplicate text on completed event", () => {
     const state = createTranscriptSegmentState();
 
     mergeTranscriptEvent(state, {
       type: TRANSCRIPT_DELTA_EVENT,
+      item_id: "item_1",
       delta: "hello world",
     });
 
-    // Without the fix, this would append: "hello worldHello, world."
     const result = mergeTranscriptEvent(state, {
       type: TRANSCRIPT_COMPLETED_EVENT,
+      item_id: "item_1",
       transcript: "Hello, world.",
     });
     assert.equal(result, "Hello, world.");
   });
+
+  it("falls back to __default__ when item_id is missing", () => {
+    const state = createTranscriptSegmentState();
+    mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_DELTA_EVENT,
+      delta: "no id",
+    });
+    const result = mergeTranscriptEvent(state, {
+      type: TRANSCRIPT_COMPLETED_EVENT,
+      transcript: "No id.",
+    });
+    assert.equal(result, "No id.");
+  });
 });

desktop/src/features/dictation/lib/realtimeAudio.ts

@@ -56,49 +56,93 @@ export async function connectPeerConnection(options: {
 }
 
 /**
- * State for tracking the current transcription segment. Completed events
- * carry the final text for the same segment that prior deltas built up,
- * potentially with corrections/punctuation. We track the delta accumulation
- * so we can replace it with the finalized text on completion.
+ * Per-item tracking for a single transcription turn. OpenAI's Realtime API
+ * sends delta and completed events tagged with an `item_id`; completed events
+ * for different turns can arrive out of order, so we reconcile by item id.
  */
-export interface TranscriptSegmentState {
-  /** Text committed from previous (completed) segments. */
-  committed: string;
-  /** Accumulated delta text for the in-progress segment. */
-  pendingDelta: string;
-}
-
-export function createTranscriptSegmentState(): TranscriptSegmentState {
-  return { committed: "", pendingDelta: "" };
+interface ItemSegment {
+  /** Accumulated delta text (replaced by finalized on completion). */
+  pending: string;
+  /** Finalized text (set once the completed event arrives). */
+  finalized: string | null;
 }
 
 /**
- * Merge a transcript event into the segment state.
+ * State for tracking transcription segments keyed by item id.
+ * Maintains insertion order so the full transcript is reconstructed
+ * in the order items were first seen.
+ */
+export interface TranscriptSegmentState {
+  /** Ordered item ids (insertion order = utterance order). */
+  itemOrder: string[];
+  /** Per-item segment data. */
+  items: Map<string, ItemSegment>;
+}
+
+export function createTranscriptSegmentState(): TranscriptSegmentState {
+  return { itemOrder: [], items: new Map() };
+}
+
+/** Get the current full transcript text from segment state. */
+export function getTranscriptText(state: TranscriptSegmentState): string {
+  let result = "";
+  for (const id of state.itemOrder) {
+    const seg = state.items.get(id);
+    if (!seg) continue;
+    result += seg.finalized ?? seg.pending;
+  }
+  return result;
+}
+
+/** Internal: get or create the segment for an item. */
+function getOrCreateItem(
+  state: TranscriptSegmentState,
+  itemId: string,
+): ItemSegment {
+  let seg = state.items.get(itemId);
+  if (!seg) {
+    seg = { pending: "", finalized: null };
+    state.items.set(itemId, seg);
+    state.itemOrder.push(itemId);
+  }
+  return seg;
+}
+
+/**
+ * Merge a transcript event into the segment state, keyed by `item_id`.
  *
- * - Delta events: append to `pendingDelta`.
- * - Completed events: replace `pendingDelta` with the finalized transcript,
- *   then commit it (move to `committed` and reset `pendingDelta`).
+ * - Delta events: append to the item's `pending` text.
+ * - Completed events: store `finalized` text, replacing accumulated deltas.
  *
- * Returns the full merged text (committed + pending).
+ * Returns the full merged text across all items in order.
  */
 export function mergeTranscriptEvent(
   state: TranscriptSegmentState,
   event: TranscriptEvent,
 ): string {
+  // Use item_id from the event; fall back to a synthetic key for events
+  // that lack one (shouldn't happen in practice, but be defensive).
+  const itemId = event.item_id ?? "__default__";
+
   if (event.type === TRANSCRIPT_DELTA_EVENT) {
+    const seg = getOrCreateItem(state, itemId);
     const delta = event.delta ?? "";
     if (delta) {
-      state.pendingDelta += delta;
+      seg.pending += delta;
     }
   } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
-    const finalText = event.transcript ?? "";
-    // Replace the accumulated deltas with the finalized text, then commit.
-    const separator = state.committed && finalText ? "" : "";
-    state.committed = state.committed + separator + finalText;
-    state.pendingDelta = "";
+    const seg = getOrCreateItem(state, itemId);
+    seg.finalized = event.transcript ?? "";
   }
 
-  return state.committed + state.pendingDelta;
+  // Reconstruct full text from all items in order.
+  let result = "";
+  for (const id of state.itemOrder) {
+    const seg = state.items.get(id);
+    if (!seg) continue;
+    result += seg.finalized ?? seg.pending;
+  }
+  return result;
 }
 
 // ── Audio buffer capture ──────────────────────────────────────────────────