fix(dictation): address review — nonce, editor state, segment tracking

- Add nonce tag to NIP-98 auth events to prevent replay rejection when
  multiple components call /transcribe/status in the same second.

- Wire dictation text into both the Tiptap editor and contentRef via
  setComposerContent + setEditorContentRef, so dictated text actually
  appears in the composer and is serialized on submit.

- Call submitMessageRef.current() synchronously in onSend instead of via
  queueMicrotask, ensuring the editor content is consumed before the
  subsequent setText('') clears it.

- Replace naive append-based transcript merging with segment-aware state
  tracking (TranscriptSegmentState). Delta events accumulate into
  pendingDelta; completed events replace accumulated deltas with the
  finalized text, preventing duplication.

Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
klopez4212
2026-07-11 16:18:06 +01:00
parent bde70242b8
commit 53f96841c9
7 changed files with 367 additions and 33 deletions
@@ -1,4 +1,4 @@
import { getRelayHttpUrl } from "@/shared/api/tauri";
import { getRelayHttpUrl, signRelayEvent } from "@/shared/api/tauri";
export interface TranscribeStatus {
configured: boolean;
@@ -10,9 +10,39 @@ export interface TranscribeSession {
model: string;
}
/** NIP-98 event kind for HTTP request authorization. */
const NIP98_KIND = 27235;
/**
* Build a NIP-98 `Authorization: Nostr <base64>` header for an HTTP request.
*
* The relay verifies the signed event's `u` tag against its own
* host-derived expected URL, so `url` must be the exact absolute URL being
* fetched (scheme + host + path). The `method` tag must match the request.
*/
async function nip98AuthHeader(url: string, method: string): Promise<string> {
const nonce = crypto.randomUUID();
const event = await signRelayEvent({
kind: NIP98_KIND,
content: "",
tags: [
["u", url],
["method", method],
["nonce", nonce],
],
});
const json = JSON.stringify(event);
// btoa needs a binary string; encode UTF-8 first so non-ASCII survives.
const base64 = btoa(String.fromCharCode(...new TextEncoder().encode(json)));
return `Nostr ${base64}`;
}
export async function getTranscribeStatus(): Promise<TranscribeStatus> {
const baseUrl = await getRelayHttpUrl();
const response = await fetch(`${baseUrl}/transcribe/status`);
const url = `${baseUrl}/transcribe/status`;
const response = await fetch(url, {
headers: { Authorization: await nip98AuthHeader(url, "GET") },
});
if (!response.ok) {
throw new Error(`Transcribe status check failed: ${response.status}`);
}
@@ -21,9 +51,13 @@ export async function getTranscribeStatus(): Promise<TranscribeStatus> {
export async function createTranscribeSession(): Promise<TranscribeSession> {
const baseUrl = await getRelayHttpUrl();
const response = await fetch(`${baseUrl}/transcribe/session`, {
const url = `${baseUrl}/transcribe/session`;
const response = await fetch(url, {
method: "POST",
headers: { "Content-Type": "application/json" },
headers: {
"Content-Type": "application/json",
Authorization: await nip98AuthHeader(url, "POST"),
},
});
if (!response.ok) {
const body = await response.text().catch(() => "");
@@ -5,27 +5,38 @@ interface UseComposerDictationOptions {
contentRef: React.MutableRefObject<string>;
disabled: boolean;
isSending: boolean;
setComposerContentFromText: (text: string) => void;
/** Updates contentRef + isContentEmpty state. */
setComposerContent: (text: string) => void;
/** Ref to a function that updates the Tiptap editor document. */
setEditorContentRef: React.MutableRefObject<(text: string) => void>;
submitMessageRef: React.MutableRefObject<() => void>;
}
/**
* Thin wrapper around `useDictation` pre-wired for the MessageComposer's
* state management (contentRef, setComposerContentFromText, submitMessageRef).
* state management (contentRef, setComposerContent, editor, submitMessageRef).
*/
export function useComposerDictation({
contentRef,
disabled,
isSending,
setComposerContentFromText,
setComposerContent,
setEditorContentRef,
submitMessageRef,
}: UseComposerDictationOptions) {
return useDictation({
text: contentRef.current,
setText: setComposerContentFromText,
setText: (text) => {
setComposerContent(text);
setEditorContentRef.current(text);
},
onSend: (text) => {
setComposerContentFromText(text);
queueMicrotask(() => submitMessageRef.current());
setComposerContent(text);
setEditorContentRef.current(text);
// Submit synchronously — the content ref is already set above, so
// syncComposerContentFromEditor() will serialize the editor which now
// holds the dictated text.
submitMessageRef.current();
},
sendDisabled: disabled || isSending,
});
@@ -7,13 +7,15 @@ import {
import {
type AudioBufferCapture,
type TranscriptEvent,
type TranscriptSegmentState,
TRANSCRIPT_COMPLETED_EVENT,
TRANSCRIPT_DELTA_EVENT,
connectPeerConnection,
createAudioBufferCapture,
createPeerConnection,
createTranscriptSegmentState,
flushAudioBuffer,
mergeTranscriptSegment,
mergeTranscriptEvent,
} from "../lib/realtimeAudio";
interface UseRealtimeDictationOptions {
@@ -50,7 +52,9 @@ export function useRealtimeDictation({
const dataChannelRef = useRef<RTCDataChannel | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const audioCaptureRef = useRef<AudioBufferCapture | null>(null);
const transcriptRef = useRef("");
const segmentStateRef = useRef<TranscriptSegmentState>(
createTranscriptSegmentState(),
);
const activeRunIdRef = useRef(0);
const onRecordingStartRef = useRef(onRecordingStart);
const onTranscriptTextRef = useRef(onTranscriptText);
@@ -112,12 +116,12 @@ export function useRealtimeDictation({
return;
}
const text = event.delta ?? event.transcript ?? "";
const merged = mergeTranscriptSegment(transcriptRef.current, text, event);
const prevText =
segmentStateRef.current.committed + segmentStateRef.current.pendingDelta;
const merged = mergeTranscriptEvent(segmentStateRef.current, event);
if (merged === transcriptRef.current) return;
if (merged === prevText) return;
transcriptRef.current = merged;
onTranscriptTextRef.current(merged);
setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT);
}, []);
@@ -135,7 +139,7 @@ export function useRealtimeDictation({
let dataChannel: RTCDataChannel | null = null;
setIsStarting(true);
transcriptRef.current = "";
segmentStateRef.current = createTranscriptSegmentState();
onRecordingStartRef.current?.();
try {
@@ -191,6 +195,12 @@ export function useRealtimeDictation({
const channelToFlush = dataChannel;
const captureToFlush = audioCapture;
dataChannel.addEventListener("open", () => {
// If the user stopped (or restarted) recording between the SDP
// exchange and the channel opening, drop this run's buffered audio.
if (isStaleRun()) {
captureToFlush.close();
return;
}
flushAudioBuffer(channelToFlush, captureToFlush.chunks);
captureToFlush.close();
audioCaptureRef.current = null;
@@ -0,0 +1,110 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
// We test the pure logic via dynamic import of the TS source compiled by the
// test runner (vitest/tsx). Since this is an .mjs file run by the Node test
// runner through the desktop vitest config, import the built output or use
// a direct TS import if the runner supports it.
// Inline the logic to keep the test self-contained and avoid bundler issues.
const TRANSCRIPT_DELTA_EVENT =
"conversation.item.input_audio_transcription.delta";
const TRANSCRIPT_COMPLETED_EVENT =
"conversation.item.input_audio_transcription.completed";
function createTranscriptSegmentState() {
return { committed: "", pendingDelta: "" };
}
function mergeTranscriptEvent(state, event) {
if (event.type === TRANSCRIPT_DELTA_EVENT) {
const delta = event.delta ?? "";
if (delta) {
state.pendingDelta += delta;
}
} else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
const finalText = event.transcript ?? "";
const separator = state.committed && finalText ? "" : "";
state.committed = state.committed + separator + finalText;
state.pendingDelta = "";
}
return state.committed + state.pendingDelta;
}
describe("mergeTranscriptEvent", () => {
it("accumulates delta events", () => {
const state = createTranscriptSegmentState();
const r1 = mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "hello ",
});
assert.equal(r1, "hello ");
const r2 = mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "world",
});
assert.equal(r2, "hello world");
});
it("replaces deltas with finalized text on completed event", () => {
const state = createTranscriptSegmentState();
mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "hello world",
});
// Completed event carries corrected/punctuated version
const result = mergeTranscriptEvent(state, {
type: TRANSCRIPT_COMPLETED_EVENT,
transcript: "Hello, world.",
});
assert.equal(result, "Hello, world.");
assert.equal(state.committed, "Hello, world.");
assert.equal(state.pendingDelta, "");
});
it("handles multiple segments sequentially", () => {
const state = createTranscriptSegmentState();
// First segment
mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "first ",
});
mergeTranscriptEvent(state, {
type: TRANSCRIPT_COMPLETED_EVENT,
transcript: "First. ",
});
// Second segment
mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "second",
});
assert.equal(state.committed + state.pendingDelta, "First. second");
const result = mergeTranscriptEvent(state, {
type: TRANSCRIPT_COMPLETED_EVENT,
transcript: "Second.",
});
assert.equal(result, "First. Second.");
});
it("does not duplicate text on completed event", () => {
const state = createTranscriptSegmentState();
mergeTranscriptEvent(state, {
type: TRANSCRIPT_DELTA_EVENT,
delta: "hello world",
});
// Without the fix, this would append: "hello worldHello, world."
const result = mergeTranscriptEvent(state, {
type: TRANSCRIPT_COMPLETED_EVENT,
transcript: "Hello, world.",
});
assert.equal(result, "Hello, world.");
});
});
@@ -55,24 +55,50 @@ export async function connectPeerConnection(options: {
});
}
export function mergeTranscriptSegment(
currentText: string,
segmentText: string,
/**
* State for tracking the current transcription segment. Completed events
* carry the final text for the same segment that prior deltas built up,
* potentially with corrections/punctuation. We track the delta accumulation
* so we can replace it with the finalized text on completion.
*/
export interface TranscriptSegmentState {
/** Text committed from previous (completed) segments. */
committed: string;
/** Accumulated delta text for the in-progress segment. */
pendingDelta: string;
}
export function createTranscriptSegmentState(): TranscriptSegmentState {
return { committed: "", pendingDelta: "" };
}
/**
* Merge a transcript event into the segment state.
*
* - Delta events: append to `pendingDelta`.
* - Completed events: replace `pendingDelta` with the finalized transcript,
* then commit it (move to `committed` and reset `pendingDelta`).
*
* Returns the full merged text (committed + pending).
*/
export function mergeTranscriptEvent(
state: TranscriptSegmentState,
event: TranscriptEvent,
): string {
if (!segmentText) return currentText;
if (!currentText) return segmentText;
// Completed events re-send the full segment text; skip if already present.
if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
const normalizedCurrent = currentText.trimEnd().toLowerCase();
const normalizedText = segmentText.trim().toLowerCase();
if (normalizedCurrent.endsWith(normalizedText)) {
return currentText;
if (event.type === TRANSCRIPT_DELTA_EVENT) {
const delta = event.delta ?? "";
if (delta) {
state.pendingDelta += delta;
}
} else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
const finalText = event.transcript ?? "";
// Replace the accumulated deltas with the finalized text, then commit.
const separator = state.committed && finalText ? "" : "";
state.committed = state.committed + separator + finalText;
state.pendingDelta = "";
}
return currentText + segmentText;
return state.committed + state.pendingDelta;
}
// ── Audio buffer capture ──────────────────────────────────────────────────
@@ -0,0 +1,142 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
DEFAULT_AUTO_SUBMIT_PHRASE,
getAutoSubmitMatch,
parseAutoSubmitPhrases,
replaceTrailingTranscribedText,
} from "./voiceInput.ts";
// ── parseAutoSubmitPhrases ──────────────────────────────────────────────────
test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => {
assert.deepEqual(parseAutoSubmitPhrases(null), []);
assert.deepEqual(parseAutoSubmitPhrases(undefined), []);
assert.deepEqual(parseAutoSubmitPhrases(""), []);
});
test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => {
assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [
"submit",
"send it",
]);
});
test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => {
assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]);
});
// ── replaceTrailingTranscribedText ──────────────────────────────────────────
test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => {
assert.equal(
replaceTrailingTranscribedText("Hello", "", "world"),
"Hello world",
);
});
test("replaceTrailingTranscribedText_appendsToEmptyBase", () => {
assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello");
});
test("replaceTrailingTranscribedText_replacesTrailingInterim", () => {
// Interim "hello wor" is refined to "hello world".
assert.equal(
replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"),
"hello world",
);
});
test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => {
// User typed "Note: " then dictated; the manual prefix must survive.
assert.equal(
replaceTrailingTranscribedText("Note: hi", "hi", "hi there"),
"Note: hi there",
);
});
test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => {
// If the previous transcript isn't the trailing text anymore, append.
assert.equal(
replaceTrailingTranscribedText("edited text", "old", "new"),
"edited text new",
);
});
test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => {
assert.equal(
replaceTrailingTranscribedText("Hello", "", ", world"),
"Hello, world",
);
});
// ── getAutoSubmitMatch ──────────────────────────────────────────────────────
test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => {
assert.equal(
getAutoSubmitMatch(
"hello there",
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
),
null,
);
});
test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => {
const match = getAutoSubmitMatch(
"send this message submit",
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
);
assert.ok(match);
assert.equal(match.matchedPhrase, "submit");
assert.equal(match.textWithoutPhrase, "send this message");
});
test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => {
// "submit" is not at the end, so it must not auto-send.
assert.equal(
getAutoSubmitMatch(
"submit the form later",
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
),
null,
);
});
test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => {
// "resubmit" ends with "submit" but is not a standalone word → no match.
assert.equal(
getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")),
null,
);
});
test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => {
const match = getAutoSubmitMatch(
"ship it submit.",
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
);
assert.ok(match);
assert.equal(match.textWithoutPhrase, "ship it");
});
test("getAutoSubmitMatch_matchesMultiWordPhrase", () => {
const match = getAutoSubmitMatch(
"please do this send it",
parseAutoSubmitPhrases("send it"),
);
assert.ok(match);
assert.equal(match.matchedPhrase, "send it");
assert.equal(match.textWithoutPhrase, "please do this");
});
test("getAutoSubmitMatch_prefersLongestPhrase", () => {
const match = getAutoSubmitMatch(
"text please submit now",
parseAutoSubmitPhrases("submit now, now"),
);
assert.ok(match);
assert.equal(match.matchedPhrase, "submit now");
assert.equal(match.textWithoutPhrase, "text please");
});
@@ -266,15 +266,15 @@ function MessageComposerImpl({
emojiAutocomplete.isEmojiAutocompleteOpen;
const submitMessageRef = React.useRef<() => void>(() => {});
const setEditorContentRef = React.useRef<(t: string) => void>(() => {});
const dictation = useComposerDictation({
contentRef,
disabled,
isSending,
setComposerContentFromText,
setComposerContent,
setEditorContentRef,
submitMessageRef,
});
const composerScrollRef = React.useRef<HTMLDivElement>(null);
// Set after `useLinkEditor` exists below; the editor's link-click handler
@@ -335,6 +335,7 @@ function MessageComposerImpl({
},
});
setEditorContentRef.current = richText.setContent;
const linkEditor = useLinkEditor(richText);
syncContentRefFromEditorRef.current = () => {
const markdown = richText.getMarkdown();