diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts index a65048d06..a607f1515 100644 --- a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -8,6 +8,7 @@ import { type AudioBufferCapture, type TranscriptEvent, type TranscriptSegmentState, + BUFFER_COMMITTED_EVENT, TRANSCRIPT_COMPLETED_EVENT, TRANSCRIPT_DELTA_EVENT, connectPeerConnection, @@ -112,7 +113,8 @@ export function useRealtimeDictation({ if ( event.type !== TRANSCRIPT_DELTA_EVENT && - event.type !== TRANSCRIPT_COMPLETED_EVENT + event.type !== TRANSCRIPT_COMPLETED_EVENT && + event.type !== BUFFER_COMMITTED_EVENT ) { return; } diff --git a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs index abfb26ecd..adf0ca663 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs +++ b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs @@ -6,25 +6,52 @@ const TRANSCRIPT_DELTA_EVENT = "conversation.item.input_audio_transcription.delta"; const TRANSCRIPT_COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed"; +const BUFFER_COMMITTED_EVENT = "input_audio_buffer.committed"; function createTranscriptSegmentState() { return { itemOrder: [], items: new Map() }; } -function getOrCreateItem(state, itemId) { +function getOrCreateItem(state, itemId, previousItemId) { let seg = state.items.get(itemId); if (!seg) { seg = { pending: "", finalized: null }; state.items.set(itemId, seg); - state.itemOrder.push(itemId); + if (previousItemId) { + const prevIndex = state.itemOrder.indexOf(previousItemId); + if (prevIndex !== -1) { + state.itemOrder.splice(prevIndex + 1, 0, itemId); + } else { + state.itemOrder.push(itemId); + } + } else { + state.itemOrder.push(itemId); + } } return seg; } +function getTranscriptText(state) { + let result = ""; + for (const id of state.itemOrder) { + const seg = state.items.get(id); + if (!seg) continue; + const text = seg.finalized ?? seg.pending; + if (!text) continue; + if (result && !result.endsWith(" ") && !text.startsWith(" ")) { + result += " "; + } + result += text; + } + return result; +} + function mergeTranscriptEvent(state, event) { const itemId = event.item_id ?? "__default__"; - if (event.type === TRANSCRIPT_DELTA_EVENT) { + if (event.type === BUFFER_COMMITTED_EVENT) { + getOrCreateItem(state, itemId, event.previous_item_id ?? undefined); + } else if (event.type === TRANSCRIPT_DELTA_EVENT) { const seg = getOrCreateItem(state, itemId); const delta = event.delta ?? ""; if (delta) { @@ -35,13 +62,7 @@ function mergeTranscriptEvent(state, event) { seg.finalized = event.transcript ?? ""; } - let result = ""; - for (const id of state.itemOrder) { - const seg = state.items.get(id); - if (!seg) continue; - result += seg.finalized ?? seg.pending; - } - return result; + return getTranscriptText(state); } describe("mergeTranscriptEvent", () => { @@ -90,7 +111,7 @@ describe("mergeTranscriptEvent", () => { mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, item_id: "item_1", - transcript: "First. ", + transcript: "First.", }); // Second item @@ -128,7 +149,7 @@ describe("mergeTranscriptEvent", () => { mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, item_id: "item_2", - transcript: "Second. ", + transcript: "Second.", }); // item_1 still shows pending @@ -137,15 +158,15 @@ describe("mergeTranscriptEvent", () => { item_id: "item_1", delta: " more", }); - assert.equal(result, "first moreSecond. "); + assert.equal(result, "first more Second."); // item_1 finally completes result = mergeTranscriptEvent(state, { type: TRANSCRIPT_COMPLETED_EVENT, item_id: "item_1", - transcript: "First more. ", + transcript: "First more.", }); - assert.equal(result, "First more. Second. "); + assert.equal(result, "First more. Second."); }); it("does not duplicate text on completed event", () => { @@ -177,4 +198,111 @@ describe("mergeTranscriptEvent", () => { }); assert.equal(result, "No id."); }); + + it("preserves committed item order when completions arrive out of order", () => { + const state = createTranscriptSegmentState(); + + // Server commits items in order: item_1 then item_2 + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_1", + previous_item_id: null, + }); + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_2", + previous_item_id: "item_1", + }); + + // But item_2's completion arrives first + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Second.", + }); + + // Then item_1's completion arrives + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "First.", + }); + + // Order must follow committed order, not arrival order + assert.equal(result, "First. Second."); + }); + + it("inserts item after previous_item_id even if later items already exist", () => { + const state = createTranscriptSegmentState(); + + // Commit item_1 + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_1", + }); + + // Commit item_3 (item_2 not yet committed) + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_3", + previous_item_id: "item_1", + }); + + // Now commit item_2 between item_1 and item_3 + // (previous_item_id = item_1, so it goes after item_1) + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_2", + previous_item_id: "item_1", + }); + + // Add transcripts + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_1", + transcript: "One.", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_2", + transcript: "Two.", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_3", + transcript: "Three.", + }); + + // item_2 was inserted after item_1 (before item_3) + assert.equal(result, "One. Two. Three."); + }); + + it("handles completion-only flow with committed order", () => { + const state = createTranscriptSegmentState(); + + // Server commits both items + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_A", + }); + mergeTranscriptEvent(state, { + type: BUFFER_COMMITTED_EVENT, + item_id: "item_B", + previous_item_id: "item_A", + }); + + // Only completed events arrive (no deltas) — in reverse order + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_B", + transcript: "Bravo.", + }); + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + item_id: "item_A", + transcript: "Alpha.", + }); + + assert.equal(result, "Alpha. Bravo."); + }); }); diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts index 2ad6e010d..afac2ad33 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.ts +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -9,12 +9,14 @@ export const TRANSCRIPT_DELTA_EVENT = "conversation.item.input_audio_transcription.delta"; export const TRANSCRIPT_COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed"; +export const BUFFER_COMMITTED_EVENT = "input_audio_buffer.committed"; const MAX_BUFFER_CHUNKS = 500; // ~10s at 20ms per chunk export type TranscriptEvent = { type?: string; item_id?: string; + previous_item_id?: string; content_index?: number; delta?: string; transcript?: string; @@ -99,16 +101,32 @@ export function getTranscriptText(state: TranscriptSegmentState): string { return result; } -/** Internal: get or create the segment for an item. */ +/** + * Internal: get or create the segment for an item. + * When `previousItemId` is provided (from committed events), the new item is + * inserted after that item in `itemOrder` to preserve the server's utterance + * order — even if transcript events for a later item arrive first. + */ function getOrCreateItem( state: TranscriptSegmentState, itemId: string, + previousItemId?: string, ): ItemSegment { let seg = state.items.get(itemId); if (!seg) { seg = { pending: "", finalized: null }; state.items.set(itemId, seg); - state.itemOrder.push(itemId); + if (previousItemId) { + const prevIndex = state.itemOrder.indexOf(previousItemId); + if (prevIndex !== -1) { + state.itemOrder.splice(prevIndex + 1, 0, itemId); + } else { + // Previous item not yet seen — append (best effort). + state.itemOrder.push(itemId); + } + } else { + state.itemOrder.push(itemId); + } } return seg; } @@ -116,6 +134,8 @@ function getOrCreateItem( /** * Merge a transcript event into the segment state, keyed by `item_id`. * + * - Committed events: register the item in the correct order using + * `previous_item_id` from the server, before any transcript arrives. * - Delta events: append to the item's `pending` text. * - Completed events: store `finalized` text, replacing accumulated deltas. * @@ -129,7 +149,10 @@ export function mergeTranscriptEvent( // that lack one (shouldn't happen in practice, but be defensive). const itemId = event.item_id ?? "__default__"; - if (event.type === TRANSCRIPT_DELTA_EVENT) { + if (event.type === BUFFER_COMMITTED_EVENT) { + // Register the item in the correct position before transcripts arrive. + getOrCreateItem(state, itemId, event.previous_item_id ?? undefined); + } else if (event.type === TRANSCRIPT_DELTA_EVENT) { const seg = getOrCreateItem(state, itemId); const delta = event.delta ?? ""; if (delta) {