mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
feat: add real-time voice dictation to composer
Adds dictation support using OpenAI's Realtime API over WebRTC:
Relay:
- New /transcribe/status and /transcribe/session endpoints
- BUZZ_OPENAI_API_KEY env var gates the feature (hidden when absent)
- Proxies ephemeral client-secret minting from OpenAI
Desktop:
- New features/dictation module with:
- AudioWorklet for 24kHz PCM capture + buffering
- WebRTC peer connection to OpenAI Realtime API
- Real-time transcript merging into composer
- Auto-submit on trigger phrase ('submit')
- Mic button in composer toolbar (red pulse when recording)
- Integrated into MessageComposer via useComposerDictation hook
Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
klopez4212
@@ -0,0 +1,35 @@
|
||||
import { getRelayHttpUrl } from "@/shared/api/tauri";
|
||||
|
||||
export interface TranscribeStatus {
|
||||
configured: boolean;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface TranscribeSession {
|
||||
clientSecret: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export async function getTranscribeStatus(): Promise<TranscribeStatus> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/status`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Transcribe status check failed: ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function createTranscribeSession(): Promise<TranscribeSession> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/session`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => "");
|
||||
throw new Error(
|
||||
`Failed to create transcribe session (${response.status}): ${body}`,
|
||||
);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type * as React from "react";
|
||||
import { useDictation } from "./useDictation";
|
||||
|
||||
interface UseComposerDictationOptions {
|
||||
contentRef: React.MutableRefObject<string>;
|
||||
disabled: boolean;
|
||||
isSending: boolean;
|
||||
setComposerContentFromText: (text: string) => void;
|
||||
submitMessageRef: React.MutableRefObject<() => void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around `useDictation` pre-wired for the MessageComposer's
|
||||
* state management (contentRef, setComposerContentFromText, submitMessageRef).
|
||||
*/
|
||||
export function useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
submitMessageRef,
|
||||
}: UseComposerDictationOptions) {
|
||||
return useDictation({
|
||||
text: contentRef.current,
|
||||
setText: setComposerContentFromText,
|
||||
onSend: (text) => {
|
||||
setComposerContentFromText(text);
|
||||
queueMicrotask(() => submitMessageRef.current());
|
||||
},
|
||||
sendDisabled: disabled || isSending,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { useCallback, useMemo, useRef } from "react";
|
||||
import {
|
||||
DEFAULT_AUTO_SUBMIT_PHRASE,
|
||||
getAutoSubmitMatch,
|
||||
parseAutoSubmitPhrases,
|
||||
replaceTrailingTranscribedText,
|
||||
} from "../lib/voiceInput";
|
||||
import { useRealtimeDictation } from "./useRealtimeDictation";
|
||||
|
||||
interface UseDictationOptions {
|
||||
/** Current composer text */
|
||||
text: string;
|
||||
/** Set composer text */
|
||||
setText: (value: string) => void;
|
||||
/** Send the message */
|
||||
onSend: (text: string) => void;
|
||||
/** Whether sending is currently blocked */
|
||||
sendDisabled?: boolean;
|
||||
}
|
||||
|
||||
export function useDictation({
|
||||
text,
|
||||
setText,
|
||||
onSend,
|
||||
sendDisabled = false,
|
||||
}: UseDictationOptions) {
|
||||
const autoSubmitPhrases = useMemo(
|
||||
() => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
[],
|
||||
);
|
||||
const stopRecordingRef = useRef<() => void>(() => {});
|
||||
const textRef = useRef(text);
|
||||
textRef.current = text;
|
||||
const lastTranscriptRef = useRef("");
|
||||
|
||||
const handleTranscript = useCallback(
|
||||
(transcript: string) => {
|
||||
const previous = lastTranscriptRef.current;
|
||||
const latest = textRef.current;
|
||||
const merged = replaceTrailingTranscribedText(
|
||||
latest,
|
||||
previous,
|
||||
transcript,
|
||||
);
|
||||
const match = getAutoSubmitMatch(transcript, autoSubmitPhrases);
|
||||
|
||||
if (!match) {
|
||||
setText(merged);
|
||||
textRef.current = merged;
|
||||
lastTranscriptRef.current = transcript;
|
||||
return;
|
||||
}
|
||||
|
||||
const textWithoutPhrase = replaceTrailingTranscribedText(
|
||||
latest,
|
||||
previous,
|
||||
match.textWithoutPhrase,
|
||||
);
|
||||
if (!textWithoutPhrase.trim()) return;
|
||||
|
||||
stopRecordingRef.current();
|
||||
|
||||
if (sendDisabled) {
|
||||
setText(textWithoutPhrase);
|
||||
textRef.current = textWithoutPhrase;
|
||||
return;
|
||||
}
|
||||
|
||||
onSend(textWithoutPhrase.trim());
|
||||
setText("");
|
||||
textRef.current = "";
|
||||
lastTranscriptRef.current = "";
|
||||
},
|
||||
[autoSubmitPhrases, onSend, sendDisabled, setText],
|
||||
);
|
||||
|
||||
const dictation = useRealtimeDictation({
|
||||
onRecordingStart: () => {
|
||||
lastTranscriptRef.current = "";
|
||||
},
|
||||
onTranscriptText: handleTranscript,
|
||||
});
|
||||
stopRecordingRef.current = dictation.stopRecording;
|
||||
|
||||
return dictation;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
createTranscribeSession,
|
||||
getTranscribeStatus,
|
||||
} from "../api/transcribeSession";
|
||||
import {
|
||||
type AudioBufferCapture,
|
||||
type TranscriptEvent,
|
||||
TRANSCRIPT_COMPLETED_EVENT,
|
||||
TRANSCRIPT_DELTA_EVENT,
|
||||
connectPeerConnection,
|
||||
createAudioBufferCapture,
|
||||
createPeerConnection,
|
||||
flushAudioBuffer,
|
||||
mergeTranscriptSegment,
|
||||
} from "../lib/realtimeAudio";
|
||||
|
||||
interface UseRealtimeDictationOptions {
|
||||
disabled?: boolean;
|
||||
onRecordingStart?: () => void;
|
||||
onTranscriptText: (text: string) => void;
|
||||
}
|
||||
|
||||
function closeResources(resources: {
|
||||
audioCapture?: AudioBufferCapture | null;
|
||||
dataChannel?: RTCDataChannel | null;
|
||||
peerConnection?: RTCPeerConnection | null;
|
||||
stream?: MediaStream | null;
|
||||
}) {
|
||||
resources.audioCapture?.close();
|
||||
resources.dataChannel?.close();
|
||||
resources.peerConnection?.close();
|
||||
for (const track of resources.stream?.getTracks() ?? []) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export function useRealtimeDictation({
|
||||
disabled = false,
|
||||
onRecordingStart,
|
||||
onTranscriptText,
|
||||
}: UseRealtimeDictationOptions) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isStarting, setIsStarting] = useState(false);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [isConfigured, setIsConfigured] = useState(false);
|
||||
|
||||
const peerConnectionRef = useRef<RTCPeerConnection | null>(null);
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const audioCaptureRef = useRef<AudioBufferCapture | null>(null);
|
||||
const transcriptRef = useRef("");
|
||||
const activeRunIdRef = useRef(0);
|
||||
const onRecordingStartRef = useRef(onRecordingStart);
|
||||
const onTranscriptTextRef = useRef(onTranscriptText);
|
||||
|
||||
onRecordingStartRef.current = onRecordingStart;
|
||||
onTranscriptTextRef.current = onTranscriptText;
|
||||
|
||||
const isEnabled = !disabled && isConfigured;
|
||||
|
||||
// Check if transcription is configured on mount
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
getTranscribeStatus()
|
||||
.then((status) => {
|
||||
if (!cancelled) setIsConfigured(status.configured);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setIsConfigured(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const cleanupResources = useCallback(() => {
|
||||
activeRunIdRef.current += 1;
|
||||
closeResources({
|
||||
audioCapture: audioCaptureRef.current,
|
||||
dataChannel: dataChannelRef.current,
|
||||
peerConnection: peerConnectionRef.current,
|
||||
stream: streamRef.current,
|
||||
});
|
||||
audioCaptureRef.current = null;
|
||||
dataChannelRef.current = null;
|
||||
peerConnectionRef.current = null;
|
||||
streamRef.current = null;
|
||||
}, []);
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
cleanupResources();
|
||||
setIsRecording(false);
|
||||
setIsStarting(false);
|
||||
setIsTranscribing(false);
|
||||
}, [cleanupResources]);
|
||||
|
||||
useEffect(() => cleanupResources, [cleanupResources]);
|
||||
|
||||
const handleRealtimeEvent = useCallback((event: TranscriptEvent) => {
|
||||
if (event.type === "error") {
|
||||
console.error("OpenAI realtime server error", event);
|
||||
toast.error(event.error?.message ?? "Voice input error");
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
event.type !== TRANSCRIPT_DELTA_EVENT &&
|
||||
event.type !== TRANSCRIPT_COMPLETED_EVENT
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = event.delta ?? event.transcript ?? "";
|
||||
const merged = mergeTranscriptSegment(transcriptRef.current, text, event);
|
||||
|
||||
if (merged === transcriptRef.current) return;
|
||||
|
||||
transcriptRef.current = merged;
|
||||
onTranscriptTextRef.current(merged);
|
||||
setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT);
|
||||
}, []);
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
if (!isEnabled || isStarting || isRecording) return;
|
||||
|
||||
const runId = activeRunIdRef.current + 1;
|
||||
activeRunIdRef.current = runId;
|
||||
const isStaleRun = () => activeRunIdRef.current !== runId;
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
let audioCapture: AudioBufferCapture | null = null;
|
||||
let peerConnection: RTCPeerConnection | null = null;
|
||||
let dataChannel: RTCDataChannel | null = null;
|
||||
|
||||
setIsStarting(true);
|
||||
transcriptRef.current = "";
|
||||
onRecordingStartRef.current?.();
|
||||
|
||||
try {
|
||||
// 1. Capture mic immediately for instant feedback
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
});
|
||||
if (isStaleRun()) {
|
||||
closeResources({ stream });
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
setIsRecording(true);
|
||||
|
||||
// 2. Buffer PCM via AudioWorklet while network calls proceed
|
||||
audioCapture = await createAudioBufferCapture(stream);
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, stream });
|
||||
return;
|
||||
}
|
||||
audioCaptureRef.current = audioCapture;
|
||||
|
||||
// 3. Create session via relay
|
||||
const session = await createTranscribeSession();
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, stream });
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. Set up WebRTC
|
||||
peerConnection = createPeerConnection();
|
||||
peerConnectionRef.current = peerConnection;
|
||||
const activeStream = stream;
|
||||
stream.getAudioTracks().forEach((track) => {
|
||||
peerConnection?.addTrack(track, activeStream);
|
||||
});
|
||||
|
||||
dataChannel = peerConnection.createDataChannel("oai-events");
|
||||
dataChannelRef.current = dataChannel;
|
||||
dataChannel.addEventListener("message", (message) => {
|
||||
try {
|
||||
handleRealtimeEvent(JSON.parse(String(message.data)));
|
||||
} catch {
|
||||
// Ignore non-JSON events
|
||||
}
|
||||
});
|
||||
|
||||
// Flush buffered audio once data channel opens
|
||||
const channelToFlush = dataChannel;
|
||||
const captureToFlush = audioCapture;
|
||||
dataChannel.addEventListener("open", () => {
|
||||
flushAudioBuffer(channelToFlush, captureToFlush.chunks);
|
||||
captureToFlush.close();
|
||||
audioCaptureRef.current = null;
|
||||
});
|
||||
|
||||
// 5. SDP exchange
|
||||
await connectPeerConnection({
|
||||
peerConnection,
|
||||
clientSecret: session.clientSecret,
|
||||
});
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, dataChannel, peerConnection, stream });
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
closeResources({ audioCapture, dataChannel, peerConnection, stream });
|
||||
if (!isStaleRun()) {
|
||||
audioCaptureRef.current = null;
|
||||
dataChannelRef.current = null;
|
||||
peerConnectionRef.current = null;
|
||||
streamRef.current = null;
|
||||
setIsRecording(false);
|
||||
setIsTranscribing(false);
|
||||
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Voice input failed";
|
||||
if (/not allowed|denied|permission/i.test(message)) {
|
||||
toast.error("Microphone access denied", {
|
||||
description:
|
||||
"Allow microphone access in System Settings to use dictation.",
|
||||
});
|
||||
} else if (/not found|no audio/i.test(message)) {
|
||||
toast.error("No microphone found", {
|
||||
description: "Connect a microphone and try again.",
|
||||
});
|
||||
} else {
|
||||
toast.error("Voice input failed", { description: message });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (!isStaleRun()) setIsStarting(false);
|
||||
}
|
||||
}, [handleRealtimeEvent, isEnabled, isRecording, isStarting]);
|
||||
|
||||
const stopRecording = useCallback(() => cleanup(), [cleanup]);
|
||||
|
||||
const toggleRecording = useCallback(() => {
|
||||
if (isRecording || isStarting) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
void startRecording();
|
||||
}, [isRecording, isStarting, startRecording, stopRecording]);
|
||||
|
||||
return {
|
||||
isEnabled,
|
||||
isRecording,
|
||||
isStarting,
|
||||
isTranscribing,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
toggleRecording,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export { useComposerDictation } from "./hooks/useComposerDictation";
|
||||
export { useDictation } from "./hooks/useDictation";
|
||||
export { DictationButton } from "./ui/DictationButton";
|
||||
@@ -0,0 +1,145 @@
|
||||
import {
|
||||
REALTIME_BUFFER_PROCESSOR_NAME,
|
||||
createWorkletBlobUrl,
|
||||
} from "./realtimeBufferWorklet";
|
||||
|
||||
export const OPENAI_REALTIME_WEBRTC_URL =
|
||||
"https://api.openai.com/v1/realtime/calls";
|
||||
export const TRANSCRIPT_DELTA_EVENT =
|
||||
"conversation.item.input_audio_transcription.delta";
|
||||
export const TRANSCRIPT_COMPLETED_EVENT =
|
||||
"conversation.item.input_audio_transcription.completed";
|
||||
|
||||
const MAX_BUFFER_CHUNKS = 500; // ~10s at 20ms per chunk
|
||||
|
||||
export type TranscriptEvent = {
|
||||
type?: string;
|
||||
item_id?: string;
|
||||
content_index?: number;
|
||||
delta?: string;
|
||||
transcript?: string;
|
||||
message?: string;
|
||||
error?: { message?: string };
|
||||
};
|
||||
|
||||
export function createPeerConnection(): RTCPeerConnection {
|
||||
return new RTCPeerConnection();
|
||||
}
|
||||
|
||||
export async function connectPeerConnection(options: {
|
||||
peerConnection: RTCPeerConnection;
|
||||
clientSecret: string;
|
||||
}): Promise<void> {
|
||||
const offer = await options.peerConnection.createOffer();
|
||||
await options.peerConnection.setLocalDescription(offer);
|
||||
|
||||
const response = await fetch(OPENAI_REALTIME_WEBRTC_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${options.clientSecret}`,
|
||||
"Content-Type": "application/sdp",
|
||||
},
|
||||
body: offer.sdp ?? "",
|
||||
});
|
||||
|
||||
const body = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`OpenAI realtime connection failed (${response.status}): ${body}`,
|
||||
);
|
||||
}
|
||||
|
||||
await options.peerConnection.setRemoteDescription({
|
||||
type: "answer",
|
||||
sdp: body,
|
||||
});
|
||||
}
|
||||
|
||||
export function mergeTranscriptSegment(
|
||||
currentText: string,
|
||||
segmentText: string,
|
||||
event: TranscriptEvent,
|
||||
): string {
|
||||
if (!segmentText) return currentText;
|
||||
if (!currentText) return segmentText;
|
||||
|
||||
// Completed events re-send the full segment text; skip if already present.
|
||||
if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
|
||||
const normalizedCurrent = currentText.trimEnd().toLowerCase();
|
||||
const normalizedText = segmentText.trim().toLowerCase();
|
||||
if (normalizedCurrent.endsWith(normalizedText)) {
|
||||
return currentText;
|
||||
}
|
||||
}
|
||||
|
||||
return currentText + segmentText;
|
||||
}
|
||||
|
||||
// ── Audio buffer capture ──────────────────────────────────────────────────
|
||||
|
||||
export interface AudioBufferCapture {
|
||||
chunks: Int16Array[];
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export async function createAudioBufferCapture(
|
||||
stream: MediaStream,
|
||||
): Promise<AudioBufferCapture> {
|
||||
const audioContext = new AudioContext();
|
||||
const blobUrl = createWorkletBlobUrl();
|
||||
try {
|
||||
await audioContext.audioWorklet.addModule(blobUrl);
|
||||
} finally {
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
}
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const worklet = new AudioWorkletNode(
|
||||
audioContext,
|
||||
REALTIME_BUFFER_PROCESSOR_NAME,
|
||||
);
|
||||
source.connect(worklet);
|
||||
worklet.connect(audioContext.destination);
|
||||
|
||||
const chunks: Int16Array[] = [];
|
||||
worklet.port.onmessage = (event: MessageEvent<ArrayBuffer>) => {
|
||||
if (chunks.length < MAX_BUFFER_CHUNKS) {
|
||||
chunks.push(new Int16Array(event.data));
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
chunks,
|
||||
close() {
|
||||
worklet.disconnect();
|
||||
source.disconnect();
|
||||
void audioContext.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Flush buffered PCM into the data channel ──────────────────────────────
|
||||
|
||||
function int16ToBase64(pcm: Int16Array): string {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
export function flushAudioBuffer(
|
||||
dataChannel: RTCDataChannel,
|
||||
chunks: Int16Array[],
|
||||
): void {
|
||||
for (const chunk of chunks) {
|
||||
dataChannel.send(
|
||||
JSON.stringify({
|
||||
type: "input_audio_buffer.append",
|
||||
audio: int16ToBase64(chunk),
|
||||
}),
|
||||
);
|
||||
}
|
||||
chunks.length = 0;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
const FRAME_SAMPLES = 480; // 20ms at 24kHz
|
||||
|
||||
export const REALTIME_BUFFER_PROCESSOR_NAME = "realtime-buffer-processor";
|
||||
|
||||
export const REALTIME_BUFFER_WORKLET_SOURCE = /* js */ `
|
||||
class RealtimeBufferProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this._ratio = sampleRate / ${TARGET_SAMPLE_RATE};
|
||||
this._offset = 0;
|
||||
this._buf = new Float32Array(${FRAME_SAMPLES});
|
||||
this._idx = 0;
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0]?.[0];
|
||||
if (!input) return true;
|
||||
|
||||
while (this._offset < input.length) {
|
||||
const i = Math.floor(this._offset);
|
||||
const frac = this._offset - i;
|
||||
const s0 = input[i];
|
||||
const s1 = i + 1 < input.length ? input[i + 1] : s0;
|
||||
this._buf[this._idx++] = s0 + frac * (s1 - s0);
|
||||
|
||||
if (this._idx >= ${FRAME_SAMPLES}) {
|
||||
const pcm = new Int16Array(${FRAME_SAMPLES});
|
||||
for (let j = 0; j < ${FRAME_SAMPLES}; j++) {
|
||||
const s = Math.max(-1, Math.min(1, this._buf[j]));
|
||||
pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
this.port.postMessage(pcm.buffer, [pcm.buffer]);
|
||||
this._idx = 0;
|
||||
}
|
||||
this._offset += this._ratio;
|
||||
}
|
||||
this._offset -= input.length;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('${REALTIME_BUFFER_PROCESSOR_NAME}', RealtimeBufferProcessor);
|
||||
`;
|
||||
|
||||
/** Create a blob URL that can be passed to `audioWorklet.addModule()`. */
|
||||
export function createWorkletBlobUrl(): string {
|
||||
return URL.createObjectURL(
|
||||
new Blob([REALTIME_BUFFER_WORKLET_SOURCE], {
|
||||
type: "application/javascript",
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
export const DEFAULT_AUTO_SUBMIT_PHRASE = "submit";
|
||||
|
||||
const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u;
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function normalizePhrase(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.replace(TRAILING_PUNCTUATION_REGEX, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function parseAutoSubmitPhrases(
|
||||
rawValue: string | null | undefined,
|
||||
): string[] {
|
||||
if (!rawValue) return [];
|
||||
return Array.from(
|
||||
new Set(
|
||||
rawValue
|
||||
.split(",")
|
||||
.map((value) => normalizePhrase(value))
|
||||
.filter(Boolean),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function appendTranscribedText(baseText: string, fragment: string): string {
|
||||
const normalizedFragment = fragment.replace(/\s+/g, " ").trim();
|
||||
if (!normalizedFragment) return baseText;
|
||||
if (!baseText.trim()) return normalizedFragment;
|
||||
if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) {
|
||||
return `${baseText}${normalizedFragment}`;
|
||||
}
|
||||
return `${baseText} ${normalizedFragment}`;
|
||||
}
|
||||
|
||||
export function replaceTrailingTranscribedText(
|
||||
fullText: string,
|
||||
previousTranscribedText: string,
|
||||
nextTranscribedText: string,
|
||||
): string {
|
||||
if (!previousTranscribedText) {
|
||||
return appendTranscribedText(fullText, nextTranscribedText);
|
||||
}
|
||||
|
||||
if (fullText.endsWith(previousTranscribedText)) {
|
||||
return appendTranscribedText(
|
||||
fullText.slice(0, -previousTranscribedText.length),
|
||||
nextTranscribedText,
|
||||
);
|
||||
}
|
||||
|
||||
const trimmedPreviousText = previousTranscribedText.trim();
|
||||
if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) {
|
||||
return appendTranscribedText(
|
||||
fullText.slice(0, -trimmedPreviousText.length),
|
||||
nextTranscribedText,
|
||||
);
|
||||
}
|
||||
|
||||
return appendTranscribedText(fullText, nextTranscribedText);
|
||||
}
|
||||
|
||||
export function getAutoSubmitMatch(
|
||||
transcribedText: string,
|
||||
autoSubmitPhrases: string[],
|
||||
): { matchedPhrase: string; textWithoutPhrase: string } | null {
|
||||
const normalizedTranscribedText = normalizePhrase(transcribedText);
|
||||
if (!normalizedTranscribedText) return null;
|
||||
|
||||
const sortedPhrases = [...autoSubmitPhrases].sort(
|
||||
(left, right) => right.length - left.length,
|
||||
);
|
||||
|
||||
for (const phrase of sortedPhrases) {
|
||||
if (!normalizedTranscribedText.endsWith(phrase)) continue;
|
||||
|
||||
const phraseStartIndex = normalizedTranscribedText.length - phrase.length;
|
||||
if (
|
||||
phraseStartIndex > 0 &&
|
||||
normalizedTranscribedText[phraseStartIndex - 1] !== " "
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, "");
|
||||
const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp);
|
||||
const phrasePattern = new RegExp(
|
||||
`(^|\\s)(${phraseWords.join("\\s+")})\\s*$`,
|
||||
"iu",
|
||||
);
|
||||
const rawMatch = trimmedText.match(phrasePattern);
|
||||
const phraseStartOffset =
|
||||
rawMatch && rawMatch.index !== undefined
|
||||
? rawMatch.index + (rawMatch[1]?.length ?? 0)
|
||||
: trimmedText.length - phrase.length;
|
||||
const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd();
|
||||
|
||||
return { matchedPhrase: phrase, textWithoutPhrase };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Mic } from "lucide-react";
|
||||
import { Button } from "@/shared/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip";
|
||||
import { cn } from "@/shared/lib/cn";
|
||||
|
||||
interface DictationState {
|
||||
isEnabled: boolean;
|
||||
isRecording: boolean;
|
||||
isStarting: boolean;
|
||||
isTranscribing: boolean;
|
||||
toggleRecording: () => void;
|
||||
}
|
||||
|
||||
interface DictationButtonProps {
|
||||
dictation: DictationState;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
export function DictationButton({
|
||||
dictation,
|
||||
disabled = false,
|
||||
}: DictationButtonProps) {
|
||||
if (!dictation.isEnabled) return null;
|
||||
|
||||
const tooltipText = dictation.isRecording
|
||||
? "Stop recording"
|
||||
: dictation.isTranscribing
|
||||
? "Transcribing…"
|
||||
: "Dictate message";
|
||||
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
aria-label={tooltipText}
|
||||
aria-pressed={dictation.isRecording}
|
||||
className={cn(
|
||||
dictation.isRecording &&
|
||||
"bg-destructive text-destructive-foreground hover:bg-destructive/90 hover:text-destructive-foreground active:bg-destructive active:text-destructive-foreground",
|
||||
dictation.isTranscribing && "animate-pulse",
|
||||
)}
|
||||
disabled={disabled || dictation.isStarting}
|
||||
onClick={dictation.toggleRecording}
|
||||
size="icon"
|
||||
type="button"
|
||||
variant={dictation.isRecording ? "default" : "ghost"}
|
||||
>
|
||||
<Mic />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{tooltipText}</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
@@ -54,6 +54,7 @@ import { NonMemberMentionDialog } from "./NonMemberMentionDialog";
|
||||
import { useMentionSendFlow } from "./useMentionSendFlow";
|
||||
import { useComposerContentState } from "./useComposerContentState";
|
||||
import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot";
|
||||
import { DictationButton, useComposerDictation } from "@/features/dictation";
|
||||
|
||||
type MessageComposerProps = {
|
||||
channelId?: string | null;
|
||||
@@ -265,6 +266,15 @@ function MessageComposerImpl({
|
||||
emojiAutocomplete.isEmojiAutocompleteOpen;
|
||||
|
||||
const submitMessageRef = React.useRef<() => void>(() => {});
|
||||
|
||||
const dictation = useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
submitMessageRef,
|
||||
});
|
||||
|
||||
const composerScrollRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
// Set after `useLinkEditor` exists below; the editor's link-click handler
|
||||
@@ -997,7 +1007,12 @@ function MessageComposerImpl({
|
||||
<MessageComposerToolbar
|
||||
composerDisabled={disabled}
|
||||
editor={richText.editor}
|
||||
extraActions={toolbarExtraActions}
|
||||
extraActions={
|
||||
<>
|
||||
<DictationButton dictation={dictation} disabled={disabled} />
|
||||
{toolbarExtraActions}
|
||||
</>
|
||||
}
|
||||
formattingDisabled={disabled}
|
||||
isEmojiPickerOpen={isEmojiPickerOpen}
|
||||
isFormattingOpen={isFormattingOpen}
|
||||
|
||||
Reference in New Issue
Block a user