diff --git a/desktop/src/features/dictation/api/transcribeSession.ts b/desktop/src/features/dictation/api/transcribeSession.ts index dddfe89c3..276ca495a 100644 --- a/desktop/src/features/dictation/api/transcribeSession.ts +++ b/desktop/src/features/dictation/api/transcribeSession.ts @@ -1,4 +1,4 @@ -import { getRelayHttpUrl } from "@/shared/api/tauri"; +import { getRelayHttpUrl, signRelayEvent } from "@/shared/api/tauri"; export interface TranscribeStatus { configured: boolean; @@ -10,9 +10,39 @@ export interface TranscribeSession { model: string; } +/** NIP-98 event kind for HTTP request authorization. */ +const NIP98_KIND = 27235; + +/** + * Build a NIP-98 `Authorization: Nostr ` header for an HTTP request. + * + * The relay verifies the signed event's `u` tag against its own + * host-derived expected URL, so `url` must be the exact absolute URL being + * fetched (scheme + host + path). The `method` tag must match the request. + */ +async function nip98AuthHeader(url: string, method: string): Promise { + const nonce = crypto.randomUUID(); + const event = await signRelayEvent({ + kind: NIP98_KIND, + content: "", + tags: [ + ["u", url], + ["method", method], + ["nonce", nonce], + ], + }); + const json = JSON.stringify(event); + // btoa needs a binary string; encode UTF-8 first so non-ASCII survives. + const base64 = btoa(String.fromCharCode(...new TextEncoder().encode(json))); + return `Nostr ${base64}`; +} + export async function getTranscribeStatus(): Promise { const baseUrl = await getRelayHttpUrl(); - const response = await fetch(`${baseUrl}/transcribe/status`); + const url = `${baseUrl}/transcribe/status`; + const response = await fetch(url, { + headers: { Authorization: await nip98AuthHeader(url, "GET") }, + }); if (!response.ok) { throw new Error(`Transcribe status check failed: ${response.status}`); } @@ -21,9 +51,13 @@ export async function getTranscribeStatus(): Promise { export async function createTranscribeSession(): Promise { const baseUrl = await getRelayHttpUrl(); - const response = await fetch(`${baseUrl}/transcribe/session`, { + const url = `${baseUrl}/transcribe/session`; + const response = await fetch(url, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + Authorization: await nip98AuthHeader(url, "POST"), + }, }); if (!response.ok) { const body = await response.text().catch(() => ""); diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts index e40f23f2e..d907062be 100644 --- a/desktop/src/features/dictation/hooks/useComposerDictation.ts +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -5,27 +5,38 @@ interface UseComposerDictationOptions { contentRef: React.MutableRefObject; disabled: boolean; isSending: boolean; - setComposerContentFromText: (text: string) => void; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; submitMessageRef: React.MutableRefObject<() => void>; } /** * Thin wrapper around `useDictation` pre-wired for the MessageComposer's - * state management (contentRef, setComposerContentFromText, submitMessageRef). + * state management (contentRef, setComposerContent, editor, submitMessageRef). */ export function useComposerDictation({ contentRef, disabled, isSending, - setComposerContentFromText, + setComposerContent, + setEditorContentRef, submitMessageRef, }: UseComposerDictationOptions) { return useDictation({ text: contentRef.current, - setText: setComposerContentFromText, + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, onSend: (text) => { - setComposerContentFromText(text); - queueMicrotask(() => submitMessageRef.current()); + setComposerContent(text); + setEditorContentRef.current(text); + // Submit synchronously — the content ref is already set above, so + // syncComposerContentFromEditor() will serialize the editor which now + // holds the dictated text. + submitMessageRef.current(); }, sendDisabled: disabled || isSending, }); diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts index 3e2f038f7..ce289e279 100644 --- a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -7,13 +7,15 @@ import { import { type AudioBufferCapture, type TranscriptEvent, + type TranscriptSegmentState, TRANSCRIPT_COMPLETED_EVENT, TRANSCRIPT_DELTA_EVENT, connectPeerConnection, createAudioBufferCapture, createPeerConnection, + createTranscriptSegmentState, flushAudioBuffer, - mergeTranscriptSegment, + mergeTranscriptEvent, } from "../lib/realtimeAudio"; interface UseRealtimeDictationOptions { @@ -50,7 +52,9 @@ export function useRealtimeDictation({ const dataChannelRef = useRef(null); const streamRef = useRef(null); const audioCaptureRef = useRef(null); - const transcriptRef = useRef(""); + const segmentStateRef = useRef( + createTranscriptSegmentState(), + ); const activeRunIdRef = useRef(0); const onRecordingStartRef = useRef(onRecordingStart); const onTranscriptTextRef = useRef(onTranscriptText); @@ -112,12 +116,12 @@ export function useRealtimeDictation({ return; } - const text = event.delta ?? event.transcript ?? ""; - const merged = mergeTranscriptSegment(transcriptRef.current, text, event); + const prevText = + segmentStateRef.current.committed + segmentStateRef.current.pendingDelta; + const merged = mergeTranscriptEvent(segmentStateRef.current, event); - if (merged === transcriptRef.current) return; + if (merged === prevText) return; - transcriptRef.current = merged; onTranscriptTextRef.current(merged); setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT); }, []); @@ -135,7 +139,7 @@ export function useRealtimeDictation({ let dataChannel: RTCDataChannel | null = null; setIsStarting(true); - transcriptRef.current = ""; + segmentStateRef.current = createTranscriptSegmentState(); onRecordingStartRef.current?.(); try { @@ -191,6 +195,12 @@ export function useRealtimeDictation({ const channelToFlush = dataChannel; const captureToFlush = audioCapture; dataChannel.addEventListener("open", () => { + // If the user stopped (or restarted) recording between the SDP + // exchange and the channel opening, drop this run's buffered audio. + if (isStaleRun()) { + captureToFlush.close(); + return; + } flushAudioBuffer(channelToFlush, captureToFlush.chunks); captureToFlush.close(); audioCaptureRef.current = null; diff --git a/desktop/src/features/dictation/lib/realtimeAudio.test.mjs b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs new file mode 100644 index 000000000..e2fe52cb5 --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeAudio.test.mjs @@ -0,0 +1,110 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +// We test the pure logic via dynamic import of the TS source compiled by the +// test runner (vitest/tsx). Since this is an .mjs file run by the Node test +// runner through the desktop vitest config, import the built output or use +// a direct TS import if the runner supports it. + +// Inline the logic to keep the test self-contained and avoid bundler issues. +const TRANSCRIPT_DELTA_EVENT = + "conversation.item.input_audio_transcription.delta"; +const TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; + +function createTranscriptSegmentState() { + return { committed: "", pendingDelta: "" }; +} + +function mergeTranscriptEvent(state, event) { + if (event.type === TRANSCRIPT_DELTA_EVENT) { + const delta = event.delta ?? ""; + if (delta) { + state.pendingDelta += delta; + } + } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { + const finalText = event.transcript ?? ""; + const separator = state.committed && finalText ? "" : ""; + state.committed = state.committed + separator + finalText; + state.pendingDelta = ""; + } + + return state.committed + state.pendingDelta; +} + +describe("mergeTranscriptEvent", () => { + it("accumulates delta events", () => { + const state = createTranscriptSegmentState(); + const r1 = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "hello ", + }); + assert.equal(r1, "hello "); + + const r2 = mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "world", + }); + assert.equal(r2, "hello world"); + }); + + it("replaces deltas with finalized text on completed event", () => { + const state = createTranscriptSegmentState(); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "hello world", + }); + + // Completed event carries corrected/punctuated version + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "Hello, world.", + }); + assert.equal(result, "Hello, world."); + assert.equal(state.committed, "Hello, world."); + assert.equal(state.pendingDelta, ""); + }); + + it("handles multiple segments sequentially", () => { + const state = createTranscriptSegmentState(); + + // First segment + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "first ", + }); + mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "First. ", + }); + + // Second segment + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "second", + }); + assert.equal(state.committed + state.pendingDelta, "First. second"); + + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "Second.", + }); + assert.equal(result, "First. Second."); + }); + + it("does not duplicate text on completed event", () => { + const state = createTranscriptSegmentState(); + + mergeTranscriptEvent(state, { + type: TRANSCRIPT_DELTA_EVENT, + delta: "hello world", + }); + + // Without the fix, this would append: "hello worldHello, world." + const result = mergeTranscriptEvent(state, { + type: TRANSCRIPT_COMPLETED_EVENT, + transcript: "Hello, world.", + }); + assert.equal(result, "Hello, world."); + }); +}); diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts index 59ae5ce0d..bed28a727 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.ts +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -55,24 +55,50 @@ export async function connectPeerConnection(options: { }); } -export function mergeTranscriptSegment( - currentText: string, - segmentText: string, +/** + * State for tracking the current transcription segment. Completed events + * carry the final text for the same segment that prior deltas built up, + * potentially with corrections/punctuation. We track the delta accumulation + * so we can replace it with the finalized text on completion. + */ +export interface TranscriptSegmentState { + /** Text committed from previous (completed) segments. */ + committed: string; + /** Accumulated delta text for the in-progress segment. */ + pendingDelta: string; +} + +export function createTranscriptSegmentState(): TranscriptSegmentState { + return { committed: "", pendingDelta: "" }; +} + +/** + * Merge a transcript event into the segment state. + * + * - Delta events: append to `pendingDelta`. + * - Completed events: replace `pendingDelta` with the finalized transcript, + * then commit it (move to `committed` and reset `pendingDelta`). + * + * Returns the full merged text (committed + pending). + */ +export function mergeTranscriptEvent( + state: TranscriptSegmentState, event: TranscriptEvent, ): string { - if (!segmentText) return currentText; - if (!currentText) return segmentText; - - // Completed events re-send the full segment text; skip if already present. - if (event.type === TRANSCRIPT_COMPLETED_EVENT) { - const normalizedCurrent = currentText.trimEnd().toLowerCase(); - const normalizedText = segmentText.trim().toLowerCase(); - if (normalizedCurrent.endsWith(normalizedText)) { - return currentText; + if (event.type === TRANSCRIPT_DELTA_EVENT) { + const delta = event.delta ?? ""; + if (delta) { + state.pendingDelta += delta; } + } else if (event.type === TRANSCRIPT_COMPLETED_EVENT) { + const finalText = event.transcript ?? ""; + // Replace the accumulated deltas with the finalized text, then commit. + const separator = state.committed && finalText ? "" : ""; + state.committed = state.committed + separator + finalText; + state.pendingDelta = ""; } - return currentText + segmentText; + return state.committed + state.pendingDelta; } // ── Audio buffer capture ────────────────────────────────────────────────── diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 000000000..f56dcf148 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "./voiceInput.ts"; + +// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── + +test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { + assert.deepEqual(parseAutoSubmitPhrases(null), []); + assert.deepEqual(parseAutoSubmitPhrases(undefined), []); + assert.deepEqual(parseAutoSubmitPhrases(""), []); +}); + +test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { + assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ + "submit", + "send it", + ]); +}); + +test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { + assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +}); + +// ── replaceTrailingTranscribedText ────────────────────────────────────────── + +test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", "world"), + "Hello world", + ); +}); + +test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { + assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); +}); + +test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { + // Interim "hello wor" is refined to "hello world". + assert.equal( + replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), + "hello world", + ); +}); + +test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { + // User typed "Note: " then dictated; the manual prefix must survive. + assert.equal( + replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), + "Note: hi there", + ); +}); + +test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { + // If the previous transcript isn't the trailing text anymore, append. + assert.equal( + replaceTrailingTranscribedText("edited text", "old", "new"), + "edited text new", + ); +}); + +test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", ", world"), + "Hello, world", + ); +}); + +// ── getAutoSubmitMatch ────────────────────────────────────────────────────── + +test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { + assert.equal( + getAutoSubmitMatch( + "hello there", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { + const match = getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit"); + assert.equal(match.textWithoutPhrase, "send this message"); +}); + +test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { + // "submit" is not at the end, so it must not auto-send. + assert.equal( + getAutoSubmitMatch( + "submit the form later", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { + // "resubmit" ends with "submit" but is not a standalone word → no match. + assert.equal( + getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { + const match = getAutoSubmitMatch( + "ship it submit.", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ); + assert.ok(match); + assert.equal(match.textWithoutPhrase, "ship it"); +}); + +test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { + const match = getAutoSubmitMatch( + "please do this send it", + parseAutoSubmitPhrases("send it"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "send it"); + assert.equal(match.textWithoutPhrase, "please do this"); +}); + +test("getAutoSubmitMatch_prefersLongestPhrase", () => { + const match = getAutoSubmitMatch( + "text please submit now", + parseAutoSubmitPhrases("submit now, now"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit now"); + assert.equal(match.textWithoutPhrase, "text please"); +}); diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index d8d21d0e8..96a5003c5 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -266,15 +266,15 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); - + const setEditorContentRef = React.useRef<(t: string) => void>(() => {}); const dictation = useComposerDictation({ contentRef, disabled, isSending, - setComposerContentFromText, + setComposerContent, + setEditorContentRef, submitMessageRef, }); - const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -335,6 +335,7 @@ function MessageComposerImpl({ }, }); + setEditorContentRef.current = richText.setContent; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown();