mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
fix(dictation): address review — nonce, editor state, segment tracking
- Add nonce tag to NIP-98 auth events to prevent replay rejection when
multiple components call /transcribe/status in the same second.
- Wire dictation text into both the Tiptap editor and contentRef via
setComposerContent + setEditorContentRef, so dictated text actually
appears in the composer and is serialized on submit.
- Call submitMessageRef.current() synchronously in onSend instead of via
queueMicrotask, ensuring the editor content is consumed before the
subsequent setText('') clears it.
- Replace naive append-based transcript merging with segment-aware state
tracking (TranscriptSegmentState). Delta events accumulate into
pendingDelta; completed events replace accumulated deltas with the
finalized text, preventing duplication.
Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { getRelayHttpUrl } from "@/shared/api/tauri";
|
||||
import { getRelayHttpUrl, signRelayEvent } from "@/shared/api/tauri";
|
||||
|
||||
export interface TranscribeStatus {
|
||||
configured: boolean;
|
||||
@@ -10,9 +10,39 @@ export interface TranscribeSession {
|
||||
model: string;
|
||||
}
|
||||
|
||||
/** NIP-98 event kind for HTTP request authorization. */
|
||||
const NIP98_KIND = 27235;
|
||||
|
||||
/**
|
||||
* Build a NIP-98 `Authorization: Nostr <base64>` header for an HTTP request.
|
||||
*
|
||||
* The relay verifies the signed event's `u` tag against its own
|
||||
* host-derived expected URL, so `url` must be the exact absolute URL being
|
||||
* fetched (scheme + host + path). The `method` tag must match the request.
|
||||
*/
|
||||
async function nip98AuthHeader(url: string, method: string): Promise<string> {
|
||||
const nonce = crypto.randomUUID();
|
||||
const event = await signRelayEvent({
|
||||
kind: NIP98_KIND,
|
||||
content: "",
|
||||
tags: [
|
||||
["u", url],
|
||||
["method", method],
|
||||
["nonce", nonce],
|
||||
],
|
||||
});
|
||||
const json = JSON.stringify(event);
|
||||
// btoa needs a binary string; encode UTF-8 first so non-ASCII survives.
|
||||
const base64 = btoa(String.fromCharCode(...new TextEncoder().encode(json)));
|
||||
return `Nostr ${base64}`;
|
||||
}
|
||||
|
||||
export async function getTranscribeStatus(): Promise<TranscribeStatus> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/status`);
|
||||
const url = `${baseUrl}/transcribe/status`;
|
||||
const response = await fetch(url, {
|
||||
headers: { Authorization: await nip98AuthHeader(url, "GET") },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Transcribe status check failed: ${response.status}`);
|
||||
}
|
||||
@@ -21,9 +51,13 @@ export async function getTranscribeStatus(): Promise<TranscribeStatus> {
|
||||
|
||||
export async function createTranscribeSession(): Promise<TranscribeSession> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/session`, {
|
||||
const url = `${baseUrl}/transcribe/session`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: await nip98AuthHeader(url, "POST"),
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => "");
|
||||
|
||||
@@ -5,27 +5,38 @@ interface UseComposerDictationOptions {
|
||||
contentRef: React.MutableRefObject<string>;
|
||||
disabled: boolean;
|
||||
isSending: boolean;
|
||||
setComposerContentFromText: (text: string) => void;
|
||||
/** Updates contentRef + isContentEmpty state. */
|
||||
setComposerContent: (text: string) => void;
|
||||
/** Ref to a function that updates the Tiptap editor document. */
|
||||
setEditorContentRef: React.MutableRefObject<(text: string) => void>;
|
||||
submitMessageRef: React.MutableRefObject<() => void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around `useDictation` pre-wired for the MessageComposer's
|
||||
* state management (contentRef, setComposerContentFromText, submitMessageRef).
|
||||
* state management (contentRef, setComposerContent, editor, submitMessageRef).
|
||||
*/
|
||||
export function useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
setComposerContent,
|
||||
setEditorContentRef,
|
||||
submitMessageRef,
|
||||
}: UseComposerDictationOptions) {
|
||||
return useDictation({
|
||||
text: contentRef.current,
|
||||
setText: setComposerContentFromText,
|
||||
setText: (text) => {
|
||||
setComposerContent(text);
|
||||
setEditorContentRef.current(text);
|
||||
},
|
||||
onSend: (text) => {
|
||||
setComposerContentFromText(text);
|
||||
queueMicrotask(() => submitMessageRef.current());
|
||||
setComposerContent(text);
|
||||
setEditorContentRef.current(text);
|
||||
// Submit synchronously — the content ref is already set above, so
|
||||
// syncComposerContentFromEditor() will serialize the editor which now
|
||||
// holds the dictated text.
|
||||
submitMessageRef.current();
|
||||
},
|
||||
sendDisabled: disabled || isSending,
|
||||
});
|
||||
|
||||
@@ -7,13 +7,15 @@ import {
|
||||
import {
|
||||
type AudioBufferCapture,
|
||||
type TranscriptEvent,
|
||||
type TranscriptSegmentState,
|
||||
TRANSCRIPT_COMPLETED_EVENT,
|
||||
TRANSCRIPT_DELTA_EVENT,
|
||||
connectPeerConnection,
|
||||
createAudioBufferCapture,
|
||||
createPeerConnection,
|
||||
createTranscriptSegmentState,
|
||||
flushAudioBuffer,
|
||||
mergeTranscriptSegment,
|
||||
mergeTranscriptEvent,
|
||||
} from "../lib/realtimeAudio";
|
||||
|
||||
interface UseRealtimeDictationOptions {
|
||||
@@ -50,7 +52,9 @@ export function useRealtimeDictation({
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const audioCaptureRef = useRef<AudioBufferCapture | null>(null);
|
||||
const transcriptRef = useRef("");
|
||||
const segmentStateRef = useRef<TranscriptSegmentState>(
|
||||
createTranscriptSegmentState(),
|
||||
);
|
||||
const activeRunIdRef = useRef(0);
|
||||
const onRecordingStartRef = useRef(onRecordingStart);
|
||||
const onTranscriptTextRef = useRef(onTranscriptText);
|
||||
@@ -112,12 +116,12 @@ export function useRealtimeDictation({
|
||||
return;
|
||||
}
|
||||
|
||||
const text = event.delta ?? event.transcript ?? "";
|
||||
const merged = mergeTranscriptSegment(transcriptRef.current, text, event);
|
||||
const prevText =
|
||||
segmentStateRef.current.committed + segmentStateRef.current.pendingDelta;
|
||||
const merged = mergeTranscriptEvent(segmentStateRef.current, event);
|
||||
|
||||
if (merged === transcriptRef.current) return;
|
||||
if (merged === prevText) return;
|
||||
|
||||
transcriptRef.current = merged;
|
||||
onTranscriptTextRef.current(merged);
|
||||
setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT);
|
||||
}, []);
|
||||
@@ -135,7 +139,7 @@ export function useRealtimeDictation({
|
||||
let dataChannel: RTCDataChannel | null = null;
|
||||
|
||||
setIsStarting(true);
|
||||
transcriptRef.current = "";
|
||||
segmentStateRef.current = createTranscriptSegmentState();
|
||||
onRecordingStartRef.current?.();
|
||||
|
||||
try {
|
||||
@@ -191,6 +195,12 @@ export function useRealtimeDictation({
|
||||
const channelToFlush = dataChannel;
|
||||
const captureToFlush = audioCapture;
|
||||
dataChannel.addEventListener("open", () => {
|
||||
// If the user stopped (or restarted) recording between the SDP
|
||||
// exchange and the channel opening, drop this run's buffered audio.
|
||||
if (isStaleRun()) {
|
||||
captureToFlush.close();
|
||||
return;
|
||||
}
|
||||
flushAudioBuffer(channelToFlush, captureToFlush.chunks);
|
||||
captureToFlush.close();
|
||||
audioCaptureRef.current = null;
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// We test the pure logic via dynamic import of the TS source compiled by the
|
||||
// test runner (vitest/tsx). Since this is an .mjs file run by the Node test
|
||||
// runner through the desktop vitest config, import the built output or use
|
||||
// a direct TS import if the runner supports it.
|
||||
|
||||
// Inline the logic to keep the test self-contained and avoid bundler issues.
|
||||
const TRANSCRIPT_DELTA_EVENT =
|
||||
"conversation.item.input_audio_transcription.delta";
|
||||
const TRANSCRIPT_COMPLETED_EVENT =
|
||||
"conversation.item.input_audio_transcription.completed";
|
||||
|
||||
function createTranscriptSegmentState() {
|
||||
return { committed: "", pendingDelta: "" };
|
||||
}
|
||||
|
||||
function mergeTranscriptEvent(state, event) {
|
||||
if (event.type === TRANSCRIPT_DELTA_EVENT) {
|
||||
const delta = event.delta ?? "";
|
||||
if (delta) {
|
||||
state.pendingDelta += delta;
|
||||
}
|
||||
} else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
|
||||
const finalText = event.transcript ?? "";
|
||||
const separator = state.committed && finalText ? "" : "";
|
||||
state.committed = state.committed + separator + finalText;
|
||||
state.pendingDelta = "";
|
||||
}
|
||||
|
||||
return state.committed + state.pendingDelta;
|
||||
}
|
||||
|
||||
describe("mergeTranscriptEvent", () => {
|
||||
it("accumulates delta events", () => {
|
||||
const state = createTranscriptSegmentState();
|
||||
const r1 = mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "hello ",
|
||||
});
|
||||
assert.equal(r1, "hello ");
|
||||
|
||||
const r2 = mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "world",
|
||||
});
|
||||
assert.equal(r2, "hello world");
|
||||
});
|
||||
|
||||
it("replaces deltas with finalized text on completed event", () => {
|
||||
const state = createTranscriptSegmentState();
|
||||
mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "hello world",
|
||||
});
|
||||
|
||||
// Completed event carries corrected/punctuated version
|
||||
const result = mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_COMPLETED_EVENT,
|
||||
transcript: "Hello, world.",
|
||||
});
|
||||
assert.equal(result, "Hello, world.");
|
||||
assert.equal(state.committed, "Hello, world.");
|
||||
assert.equal(state.pendingDelta, "");
|
||||
});
|
||||
|
||||
it("handles multiple segments sequentially", () => {
|
||||
const state = createTranscriptSegmentState();
|
||||
|
||||
// First segment
|
||||
mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "first ",
|
||||
});
|
||||
mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_COMPLETED_EVENT,
|
||||
transcript: "First. ",
|
||||
});
|
||||
|
||||
// Second segment
|
||||
mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "second",
|
||||
});
|
||||
assert.equal(state.committed + state.pendingDelta, "First. second");
|
||||
|
||||
const result = mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_COMPLETED_EVENT,
|
||||
transcript: "Second.",
|
||||
});
|
||||
assert.equal(result, "First. Second.");
|
||||
});
|
||||
|
||||
it("does not duplicate text on completed event", () => {
|
||||
const state = createTranscriptSegmentState();
|
||||
|
||||
mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_DELTA_EVENT,
|
||||
delta: "hello world",
|
||||
});
|
||||
|
||||
// Without the fix, this would append: "hello worldHello, world."
|
||||
const result = mergeTranscriptEvent(state, {
|
||||
type: TRANSCRIPT_COMPLETED_EVENT,
|
||||
transcript: "Hello, world.",
|
||||
});
|
||||
assert.equal(result, "Hello, world.");
|
||||
});
|
||||
});
|
||||
@@ -55,24 +55,50 @@ export async function connectPeerConnection(options: {
|
||||
});
|
||||
}
|
||||
|
||||
export function mergeTranscriptSegment(
|
||||
currentText: string,
|
||||
segmentText: string,
|
||||
/**
|
||||
* State for tracking the current transcription segment. Completed events
|
||||
* carry the final text for the same segment that prior deltas built up,
|
||||
* potentially with corrections/punctuation. We track the delta accumulation
|
||||
* so we can replace it with the finalized text on completion.
|
||||
*/
|
||||
export interface TranscriptSegmentState {
|
||||
/** Text committed from previous (completed) segments. */
|
||||
committed: string;
|
||||
/** Accumulated delta text for the in-progress segment. */
|
||||
pendingDelta: string;
|
||||
}
|
||||
|
||||
export function createTranscriptSegmentState(): TranscriptSegmentState {
|
||||
return { committed: "", pendingDelta: "" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a transcript event into the segment state.
|
||||
*
|
||||
* - Delta events: append to `pendingDelta`.
|
||||
* - Completed events: replace `pendingDelta` with the finalized transcript,
|
||||
* then commit it (move to `committed` and reset `pendingDelta`).
|
||||
*
|
||||
* Returns the full merged text (committed + pending).
|
||||
*/
|
||||
export function mergeTranscriptEvent(
|
||||
state: TranscriptSegmentState,
|
||||
event: TranscriptEvent,
|
||||
): string {
|
||||
if (!segmentText) return currentText;
|
||||
if (!currentText) return segmentText;
|
||||
|
||||
// Completed events re-send the full segment text; skip if already present.
|
||||
if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
|
||||
const normalizedCurrent = currentText.trimEnd().toLowerCase();
|
||||
const normalizedText = segmentText.trim().toLowerCase();
|
||||
if (normalizedCurrent.endsWith(normalizedText)) {
|
||||
return currentText;
|
||||
if (event.type === TRANSCRIPT_DELTA_EVENT) {
|
||||
const delta = event.delta ?? "";
|
||||
if (delta) {
|
||||
state.pendingDelta += delta;
|
||||
}
|
||||
} else if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
|
||||
const finalText = event.transcript ?? "";
|
||||
// Replace the accumulated deltas with the finalized text, then commit.
|
||||
const separator = state.committed && finalText ? "" : "";
|
||||
state.committed = state.committed + separator + finalText;
|
||||
state.pendingDelta = "";
|
||||
}
|
||||
|
||||
return currentText + segmentText;
|
||||
return state.committed + state.pendingDelta;
|
||||
}
|
||||
|
||||
// ── Audio buffer capture ──────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
DEFAULT_AUTO_SUBMIT_PHRASE,
|
||||
getAutoSubmitMatch,
|
||||
parseAutoSubmitPhrases,
|
||||
replaceTrailingTranscribedText,
|
||||
} from "./voiceInput.ts";
|
||||
|
||||
// ── parseAutoSubmitPhrases ──────────────────────────────────────────────────
|
||||
|
||||
test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => {
|
||||
assert.deepEqual(parseAutoSubmitPhrases(null), []);
|
||||
assert.deepEqual(parseAutoSubmitPhrases(undefined), []);
|
||||
assert.deepEqual(parseAutoSubmitPhrases(""), []);
|
||||
});
|
||||
|
||||
test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => {
|
||||
assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [
|
||||
"submit",
|
||||
"send it",
|
||||
]);
|
||||
});
|
||||
|
||||
test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => {
|
||||
assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]);
|
||||
});
|
||||
|
||||
// ── replaceTrailingTranscribedText ──────────────────────────────────────────
|
||||
|
||||
test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => {
|
||||
assert.equal(
|
||||
replaceTrailingTranscribedText("Hello", "", "world"),
|
||||
"Hello world",
|
||||
);
|
||||
});
|
||||
|
||||
test("replaceTrailingTranscribedText_appendsToEmptyBase", () => {
|
||||
assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello");
|
||||
});
|
||||
|
||||
test("replaceTrailingTranscribedText_replacesTrailingInterim", () => {
|
||||
// Interim "hello wor" is refined to "hello world".
|
||||
assert.equal(
|
||||
replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"),
|
||||
"hello world",
|
||||
);
|
||||
});
|
||||
|
||||
test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => {
|
||||
// User typed "Note: " then dictated; the manual prefix must survive.
|
||||
assert.equal(
|
||||
replaceTrailingTranscribedText("Note: hi", "hi", "hi there"),
|
||||
"Note: hi there",
|
||||
);
|
||||
});
|
||||
|
||||
test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => {
|
||||
// If the previous transcript isn't the trailing text anymore, append.
|
||||
assert.equal(
|
||||
replaceTrailingTranscribedText("edited text", "old", "new"),
|
||||
"edited text new",
|
||||
);
|
||||
});
|
||||
|
||||
test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => {
|
||||
assert.equal(
|
||||
replaceTrailingTranscribedText("Hello", "", ", world"),
|
||||
"Hello, world",
|
||||
);
|
||||
});
|
||||
|
||||
// ── getAutoSubmitMatch ──────────────────────────────────────────────────────
|
||||
|
||||
test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => {
|
||||
assert.equal(
|
||||
getAutoSubmitMatch(
|
||||
"hello there",
|
||||
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => {
|
||||
const match = getAutoSubmitMatch(
|
||||
"send this message submit",
|
||||
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
);
|
||||
assert.ok(match);
|
||||
assert.equal(match.matchedPhrase, "submit");
|
||||
assert.equal(match.textWithoutPhrase, "send this message");
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => {
|
||||
// "submit" is not at the end, so it must not auto-send.
|
||||
assert.equal(
|
||||
getAutoSubmitMatch(
|
||||
"submit the form later",
|
||||
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => {
|
||||
// "resubmit" ends with "submit" but is not a standalone word → no match.
|
||||
assert.equal(
|
||||
getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => {
|
||||
const match = getAutoSubmitMatch(
|
||||
"ship it submit.",
|
||||
parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
);
|
||||
assert.ok(match);
|
||||
assert.equal(match.textWithoutPhrase, "ship it");
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_matchesMultiWordPhrase", () => {
|
||||
const match = getAutoSubmitMatch(
|
||||
"please do this send it",
|
||||
parseAutoSubmitPhrases("send it"),
|
||||
);
|
||||
assert.ok(match);
|
||||
assert.equal(match.matchedPhrase, "send it");
|
||||
assert.equal(match.textWithoutPhrase, "please do this");
|
||||
});
|
||||
|
||||
test("getAutoSubmitMatch_prefersLongestPhrase", () => {
|
||||
const match = getAutoSubmitMatch(
|
||||
"text please submit now",
|
||||
parseAutoSubmitPhrases("submit now, now"),
|
||||
);
|
||||
assert.ok(match);
|
||||
assert.equal(match.matchedPhrase, "submit now");
|
||||
assert.equal(match.textWithoutPhrase, "text please");
|
||||
});
|
||||
@@ -266,15 +266,15 @@ function MessageComposerImpl({
|
||||
emojiAutocomplete.isEmojiAutocompleteOpen;
|
||||
|
||||
const submitMessageRef = React.useRef<() => void>(() => {});
|
||||
|
||||
const setEditorContentRef = React.useRef<(t: string) => void>(() => {});
|
||||
const dictation = useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
setComposerContent,
|
||||
setEditorContentRef,
|
||||
submitMessageRef,
|
||||
});
|
||||
|
||||
const composerScrollRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
// Set after `useLinkEditor` exists below; the editor's link-click handler
|
||||
@@ -335,6 +335,7 @@ function MessageComposerImpl({
|
||||
},
|
||||
});
|
||||
|
||||
setEditorContentRef.current = richText.setContent;
|
||||
const linkEditor = useLinkEditor(richText);
|
||||
syncContentRefFromEditorRef.current = () => {
|
||||
const markdown = richText.getMarkdown();
|
||||
|
||||
Reference in New Issue
Block a user