From f7ee8ff98017e2ebb5ade8323653ba20df73a8b1 Mon Sep 17 00:00:00 2001 From: klopez4212 Date: Sat, 4 Jul 2026 15:02:32 +0100 Subject: [PATCH] feat: add real-time voice dictation to composer Adds dictation support using OpenAI's Realtime API over WebRTC: Relay: - New /transcribe/status and /transcribe/session endpoints - BUZZ_OPENAI_API_KEY env var gates the feature (hidden when absent) - Proxies ephemeral client-secret minting from OpenAI Desktop: - New features/dictation module with: - AudioWorklet for 24kHz PCM capture + buffering - WebRTC peer connection to OpenAI Realtime API - Real-time transcript merging into composer - Auto-submit on trigger phrase ('submit') - Mic button in composer toolbar (red pulse when recording) - Integrated into MessageComposer via useComposerDictation hook Signed-off-by: klopez4212 --- .env.example | 8 + crates/buzz-relay/Cargo.toml | 1 + crates/buzz-relay/src/api/mod.rs | 3 +- crates/buzz-relay/src/api/transcribe.rs | 182 +++++++++++++ crates/buzz-relay/src/config.rs | 11 + crates/buzz-relay/src/router.rs | 10 + .../dictation/api/transcribeSession.ts | 35 +++ .../dictation/hooks/useComposerDictation.ts | 32 +++ .../features/dictation/hooks/useDictation.ts | 86 ++++++ .../dictation/hooks/useRealtimeDictation.ts | 257 ++++++++++++++++++ desktop/src/features/dictation/index.ts | 3 + .../features/dictation/lib/realtimeAudio.ts | 145 ++++++++++ .../dictation/lib/realtimeBufferWorklet.ts | 53 ++++ .../src/features/dictation/lib/voiceInput.ts | 108 ++++++++ .../features/dictation/ui/DictationButton.tsx | 54 ++++ .../features/messages/ui/MessageComposer.tsx | 17 +- 16 files changed, 1003 insertions(+), 2 deletions(-) create mode 100644 crates/buzz-relay/src/api/transcribe.rs create mode 100644 desktop/src/features/dictation/api/transcribeSession.ts create mode 100644 desktop/src/features/dictation/hooks/useComposerDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useDictation.ts create mode 100644 desktop/src/features/dictation/hooks/useRealtimeDictation.ts create mode 100644 desktop/src/features/dictation/index.ts create mode 100644 desktop/src/features/dictation/lib/realtimeAudio.ts create mode 100644 desktop/src/features/dictation/lib/realtimeBufferWorklet.ts create mode 100644 desktop/src/features/dictation/lib/voiceInput.ts create mode 100644 desktop/src/features/dictation/ui/DictationButton.tsx diff --git a/.env.example b/.env.example index 696d3a061..f4ba0cc8a 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,14 @@ RELAY_URL=ws://localhost:3000 # (use `just web` for Vite HMR instead). # BUZZ_WEB_DIR=./web/dist +# ----------------------------------------------------------------------------- +# Transcription (dictation) +# ----------------------------------------------------------------------------- +# OpenAI API key for real-time voice transcription in the composer. +# When absent, the dictation mic button is hidden. +# BUZZ_OPENAI_API_KEY=sk-... +# BUZZ_TRANSCRIPTION_MODEL=whisper-1 + # ----------------------------------------------------------------------------- # Git (NIP-34 bare repositories) # ----------------------------------------------------------------------------- diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index 7cf665b1b..ff7d06cb3 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -72,6 +72,7 @@ url = { workspace = true } moka = { workspace = true } metrics = { workspace = true } metrics-exporter-prometheus = { workspace = true } +reqwest = { workspace = true } [features] dev = ["buzz-auth/dev"] diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index f180b3255..9cb5c761c 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -1,4 +1,4 @@ -//! HTTP API — media, git, NIP-05, and the Nostr HTTP bridge. +//! HTTP API — media, git, NIP-05, transcription, and the Nostr HTTP bridge. pub mod bridge; pub mod events; @@ -7,6 +7,7 @@ pub mod invites; pub mod media; pub mod nip05; pub mod operator; +pub mod transcribe; // Re-export imeta helpers used by ingest pipeline. pub use crate::handlers::imeta::{validate_imeta_tags, verify_imeta_blobs}; diff --git a/crates/buzz-relay/src/api/transcribe.rs b/crates/buzz-relay/src/api/transcribe.rs new file mode 100644 index 000000000..915dcf5b4 --- /dev/null +++ b/crates/buzz-relay/src/api/transcribe.rs @@ -0,0 +1,182 @@ +//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting. +//! +//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client +//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a +//! WebRTC connection for real-time speech-to-text dictation. + +use axum::{extract::State, http::StatusCode, response::Json}; +use serde::Serialize; +use std::sync::Arc; + +use crate::state::AppState; + +const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions"; +const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1"; + +#[derive(Serialize)] +pub struct TranscribeStatus { + configured: bool, + model: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TranscribeSession { + client_secret: String, + model: String, +} + +/// `GET /transcribe/status` — check if transcription is configured. +pub async fn transcribe_status(State(state): State>) -> Json { + Json(TranscribeStatus { + configured: state.config.openai_api_key.is_some(), + model: transcription_model(), + }) +} + +/// `POST /transcribe/session` — create an ephemeral OpenAI Realtime session. +/// +/// Returns a short-lived client secret that the frontend uses to establish +/// a WebRTC connection directly with OpenAI for real-time transcription. +pub async fn create_transcribe_session( + State(state): State>, +) -> Result, (StatusCode, Json)> { + let api_key = state.config.openai_api_key.as_deref().ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "error": "transcription_not_configured", + "message": "Transcription is not configured on this relay" + })), + ) + })?; + + let model = transcription_model(); + + let client = reqwest::Client::new(); + let response = client + .post(OPENAI_REALTIME_SESSIONS_URL) + .header("Authorization", format!("Bearer {api_key}")) + .header("Content-Type", "application/json") + .json(&serde_json::json!({ + "model": "gpt-4o-mini-realtime-preview", + "modalities": ["text"], + "input_audio_transcription": { + "model": model, + }, + "turn_detection": { + "type": "server_vad", + } + })) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .map_err(|e| { + tracing::error!("OpenAI realtime session request failed: {e}"); + ( + StatusCode::BAD_GATEWAY, + Json(serde_json::json!({ + "error": "upstream_error", + "message": "Failed to create transcription session" + })), + ) + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::error!("OpenAI realtime session error ({status}): {body}"); + return Err(( + StatusCode::BAD_GATEWAY, + Json(serde_json::json!({ + "error": "upstream_error", + "message": "OpenAI rejected the transcription session request" + })), + )); + } + + let body: serde_json::Value = response.json().await.map_err(|e| { + tracing::error!("OpenAI realtime session response parse error: {e}"); + ( + StatusCode::BAD_GATEWAY, + Json(serde_json::json!({ + "error": "upstream_error", + "message": "Invalid response from transcription service" + })), + ) + })?; + + let client_secret = extract_client_secret(&body).ok_or_else(|| { + tracing::error!("OpenAI realtime session response missing client_secret: {body}"); + ( + StatusCode::BAD_GATEWAY, + Json(serde_json::json!({ + "error": "upstream_error", + "message": "Transcription service returned unexpected response" + })), + ) + })?; + + Ok(Json(TranscribeSession { + client_secret, + model, + })) +} + +fn transcription_model() -> String { + std::env::var("BUZZ_TRANSCRIPTION_MODEL") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) +} + +fn extract_client_secret(value: &serde_json::Value) -> Option { + // Shape 1: { "client_secret": { "value": "..." } } + if let Some(cs) = value.get("client_secret") { + if let Some(v) = cs.get("value").and_then(|v| v.as_str()) { + return Some(v.to_string()); + } + // Shape 2: { "client_secret": "..." } + if let Some(v) = cs.as_str() { + return Some(v.to_string()); + } + } + // Shape 3: { "value": "..." } + value + .get("value") + .and_then(|v| v.as_str()) + .map(String::from) +} + +#[cfg(test)] +mod tests { + use super::extract_client_secret; + use serde_json::json; + + #[test] + fn parses_nested_client_secret() { + let body = json!({ "client_secret": { "value": "sec_abc123", "expires_at": 9999 } }); + assert_eq!(extract_client_secret(&body), Some("sec_abc123".to_string())); + } + + #[test] + fn parses_direct_string_client_secret() { + let body = json!({ "client_secret": "sec_direct" }); + assert_eq!(extract_client_secret(&body), Some("sec_direct".to_string())); + } + + #[test] + fn parses_top_level_value() { + let body = json!({ "value": "sec_toplevel" }); + assert_eq!( + extract_client_secret(&body), + Some("sec_toplevel".to_string()) + ); + } + + #[test] + fn returns_none_for_missing_secret() { + let body = json!({ "id": "sess_123", "model": "gpt-4o" }); + assert_eq!(extract_client_secret(&body), None); + } +} diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 24e9604f6..48d1fd535 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -173,6 +173,11 @@ pub struct Config { /// Used to authenticate internal policy endpoint requests. pub git_hook_hmac_secret: String, + /// Optional OpenAI API key for real-time transcription (dictation). + /// When absent, the `/transcribe/session` endpoint returns 503 and the + /// desktop mic button stays hidden. + pub openai_api_key: Option, + /// Optional path to the web UI `dist/` directory. /// When set, the relay serves the SPA from this directory for browser requests. /// When unset, no static file serving happens (relay behaves as before). @@ -476,6 +481,11 @@ impl Config { let secret: [u8; 32] = rand::random(); hex::encode(secret) }); + let openai_api_key = std::env::var("BUZZ_OPENAI_API_KEY") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + // Web UI static file serving let web_dir = std::env::var("BUZZ_WEB_DIR") .ok() @@ -538,6 +548,7 @@ impl Config { git_max_repos_per_pubkey, git_max_concurrent_ops, git_hook_hmac_secret, + openai_api_key, web_dir, }) } diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 5f4bbee7e..d6c67af18 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -80,6 +80,15 @@ pub fn build_router(state: Arc) -> Router { ) // Webhook trigger (secret-authenticated, no NIP-98) .route("/hooks/{id}", post(api::bridge::workflow_webhook)) + // Transcription (dictation) — proxies OpenAI Realtime client-secret minting + .route( + "/transcribe/status", + get(api::transcribe::transcribe_status), + ) + .route( + "/transcribe/session", + post(api::transcribe::create_transcribe_session), + ) // Huddle audio WebSocket route .route( "/huddle/{channel_id}/audio", @@ -112,6 +121,7 @@ pub fn build_router(state: Arc) -> Router { || path.starts_with("/internal/") || path.starts_with("/.well-known/") || path.starts_with("/huddle/") + || path.starts_with("/transcribe/") || path == "/health" || path == "/_liveness" || path == "/_readiness" diff --git a/desktop/src/features/dictation/api/transcribeSession.ts b/desktop/src/features/dictation/api/transcribeSession.ts new file mode 100644 index 000000000..dddfe89c3 --- /dev/null +++ b/desktop/src/features/dictation/api/transcribeSession.ts @@ -0,0 +1,35 @@ +import { getRelayHttpUrl } from "@/shared/api/tauri"; + +export interface TranscribeStatus { + configured: boolean; + model: string; +} + +export interface TranscribeSession { + clientSecret: string; + model: string; +} + +export async function getTranscribeStatus(): Promise { + const baseUrl = await getRelayHttpUrl(); + const response = await fetch(`${baseUrl}/transcribe/status`); + if (!response.ok) { + throw new Error(`Transcribe status check failed: ${response.status}`); + } + return response.json(); +} + +export async function createTranscribeSession(): Promise { + const baseUrl = await getRelayHttpUrl(); + const response = await fetch(`${baseUrl}/transcribe/session`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + }); + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `Failed to create transcribe session (${response.status}): ${body}`, + ); + } + return response.json(); +} diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 000000000..e40f23f2e --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,32 @@ +import type * as React from "react"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + contentRef: React.MutableRefObject; + disabled: boolean; + isSending: boolean; + setComposerContentFromText: (text: string) => void; + submitMessageRef: React.MutableRefObject<() => void>; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (contentRef, setComposerContentFromText, submitMessageRef). + */ +export function useComposerDictation({ + contentRef, + disabled, + isSending, + setComposerContentFromText, + submitMessageRef, +}: UseComposerDictationOptions) { + return useDictation({ + text: contentRef.current, + setText: setComposerContentFromText, + onSend: (text) => { + setComposerContentFromText(text); + queueMicrotask(() => submitMessageRef.current()); + }, + sendDisabled: disabled || isSending, + }); +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 000000000..abf687704 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,86 @@ +import { useCallback, useMemo, useRef } from "react"; +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "../lib/voiceInput"; +import { useRealtimeDictation } from "./useRealtimeDictation"; + +interface UseDictationOptions { + /** Current composer text */ + text: string; + /** Set composer text */ + setText: (value: string) => void; + /** Send the message */ + onSend: (text: string) => void; + /** Whether sending is currently blocked */ + sendDisabled?: boolean; +} + +export function useDictation({ + text, + setText, + onSend, + sendDisabled = false, +}: UseDictationOptions) { + const autoSubmitPhrases = useMemo( + () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + [], + ); + const stopRecordingRef = useRef<() => void>(() => {}); + const textRef = useRef(text); + textRef.current = text; + const lastTranscriptRef = useRef(""); + + const handleTranscript = useCallback( + (transcript: string) => { + const previous = lastTranscriptRef.current; + const latest = textRef.current; + const merged = replaceTrailingTranscribedText( + latest, + previous, + transcript, + ); + const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); + + if (!match) { + setText(merged); + textRef.current = merged; + lastTranscriptRef.current = transcript; + return; + } + + const textWithoutPhrase = replaceTrailingTranscribedText( + latest, + previous, + match.textWithoutPhrase, + ); + if (!textWithoutPhrase.trim()) return; + + stopRecordingRef.current(); + + if (sendDisabled) { + setText(textWithoutPhrase); + textRef.current = textWithoutPhrase; + return; + } + + onSend(textWithoutPhrase.trim()); + setText(""); + textRef.current = ""; + lastTranscriptRef.current = ""; + }, + [autoSubmitPhrases, onSend, sendDisabled, setText], + ); + + const dictation = useRealtimeDictation({ + onRecordingStart: () => { + lastTranscriptRef.current = ""; + }, + onTranscriptText: handleTranscript, + }); + stopRecordingRef.current = dictation.stopRecording; + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts new file mode 100644 index 000000000..3e2f038f7 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -0,0 +1,257 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { + createTranscribeSession, + getTranscribeStatus, +} from "../api/transcribeSession"; +import { + type AudioBufferCapture, + type TranscriptEvent, + TRANSCRIPT_COMPLETED_EVENT, + TRANSCRIPT_DELTA_EVENT, + connectPeerConnection, + createAudioBufferCapture, + createPeerConnection, + flushAudioBuffer, + mergeTranscriptSegment, +} from "../lib/realtimeAudio"; + +interface UseRealtimeDictationOptions { + disabled?: boolean; + onRecordingStart?: () => void; + onTranscriptText: (text: string) => void; +} + +function closeResources(resources: { + audioCapture?: AudioBufferCapture | null; + dataChannel?: RTCDataChannel | null; + peerConnection?: RTCPeerConnection | null; + stream?: MediaStream | null; +}) { + resources.audioCapture?.close(); + resources.dataChannel?.close(); + resources.peerConnection?.close(); + for (const track of resources.stream?.getTracks() ?? []) { + track.stop(); + } +} + +export function useRealtimeDictation({ + disabled = false, + onRecordingStart, + onTranscriptText, +}: UseRealtimeDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isConfigured, setIsConfigured] = useState(false); + + const peerConnectionRef = useRef(null); + const dataChannelRef = useRef(null); + const streamRef = useRef(null); + const audioCaptureRef = useRef(null); + const transcriptRef = useRef(""); + const activeRunIdRef = useRef(0); + const onRecordingStartRef = useRef(onRecordingStart); + const onTranscriptTextRef = useRef(onTranscriptText); + + onRecordingStartRef.current = onRecordingStart; + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isConfigured; + + // Check if transcription is configured on mount + useEffect(() => { + let cancelled = false; + getTranscribeStatus() + .then((status) => { + if (!cancelled) setIsConfigured(status.configured); + }) + .catch(() => { + if (!cancelled) setIsConfigured(false); + }); + return () => { + cancelled = true; + }; + }, []); + + const cleanupResources = useCallback(() => { + activeRunIdRef.current += 1; + closeResources({ + audioCapture: audioCaptureRef.current, + dataChannel: dataChannelRef.current, + peerConnection: peerConnectionRef.current, + stream: streamRef.current, + }); + audioCaptureRef.current = null; + dataChannelRef.current = null; + peerConnectionRef.current = null; + streamRef.current = null; + }, []); + + const cleanup = useCallback(() => { + cleanupResources(); + setIsRecording(false); + setIsStarting(false); + setIsTranscribing(false); + }, [cleanupResources]); + + useEffect(() => cleanupResources, [cleanupResources]); + + const handleRealtimeEvent = useCallback((event: TranscriptEvent) => { + if (event.type === "error") { + console.error("OpenAI realtime server error", event); + toast.error(event.error?.message ?? "Voice input error"); + return; + } + + if ( + event.type !== TRANSCRIPT_DELTA_EVENT && + event.type !== TRANSCRIPT_COMPLETED_EVENT + ) { + return; + } + + const text = event.delta ?? event.transcript ?? ""; + const merged = mergeTranscriptSegment(transcriptRef.current, text, event); + + if (merged === transcriptRef.current) return; + + transcriptRef.current = merged; + onTranscriptTextRef.current(merged); + setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT); + }, []); + + const startRecording = useCallback(async () => { + if (!isEnabled || isStarting || isRecording) return; + + const runId = activeRunIdRef.current + 1; + activeRunIdRef.current = runId; + const isStaleRun = () => activeRunIdRef.current !== runId; + + let stream: MediaStream | null = null; + let audioCapture: AudioBufferCapture | null = null; + let peerConnection: RTCPeerConnection | null = null; + let dataChannel: RTCDataChannel | null = null; + + setIsStarting(true); + transcriptRef.current = ""; + onRecordingStartRef.current?.(); + + try { + // 1. Capture mic immediately for instant feedback + stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + if (isStaleRun()) { + closeResources({ stream }); + return; + } + streamRef.current = stream; + setIsRecording(true); + + // 2. Buffer PCM via AudioWorklet while network calls proceed + audioCapture = await createAudioBufferCapture(stream); + if (isStaleRun()) { + closeResources({ audioCapture, stream }); + return; + } + audioCaptureRef.current = audioCapture; + + // 3. Create session via relay + const session = await createTranscribeSession(); + if (isStaleRun()) { + closeResources({ audioCapture, stream }); + return; + } + + // 4. Set up WebRTC + peerConnection = createPeerConnection(); + peerConnectionRef.current = peerConnection; + const activeStream = stream; + stream.getAudioTracks().forEach((track) => { + peerConnection?.addTrack(track, activeStream); + }); + + dataChannel = peerConnection.createDataChannel("oai-events"); + dataChannelRef.current = dataChannel; + dataChannel.addEventListener("message", (message) => { + try { + handleRealtimeEvent(JSON.parse(String(message.data))); + } catch { + // Ignore non-JSON events + } + }); + + // Flush buffered audio once data channel opens + const channelToFlush = dataChannel; + const captureToFlush = audioCapture; + dataChannel.addEventListener("open", () => { + flushAudioBuffer(channelToFlush, captureToFlush.chunks); + captureToFlush.close(); + audioCaptureRef.current = null; + }); + + // 5. SDP exchange + await connectPeerConnection({ + peerConnection, + clientSecret: session.clientSecret, + }); + if (isStaleRun()) { + closeResources({ audioCapture, dataChannel, peerConnection, stream }); + return; + } + } catch (error) { + closeResources({ audioCapture, dataChannel, peerConnection, stream }); + if (!isStaleRun()) { + audioCaptureRef.current = null; + dataChannelRef.current = null; + peerConnectionRef.current = null; + streamRef.current = null; + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Voice input failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else { + toast.error("Voice input failed", { description: message }); + } + } + } finally { + if (!isStaleRun()) setIsStarting(false); + } + }, [handleRealtimeEvent, isEnabled, isRecording, isStarting]); + + const stopRecording = useCallback(() => cleanup(), [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 000000000..1578aaf65 --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,3 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts new file mode 100644 index 000000000..59ae5ce0d --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -0,0 +1,145 @@ +import { + REALTIME_BUFFER_PROCESSOR_NAME, + createWorkletBlobUrl, +} from "./realtimeBufferWorklet"; + +export const OPENAI_REALTIME_WEBRTC_URL = + "https://api.openai.com/v1/realtime/calls"; +export const TRANSCRIPT_DELTA_EVENT = + "conversation.item.input_audio_transcription.delta"; +export const TRANSCRIPT_COMPLETED_EVENT = + "conversation.item.input_audio_transcription.completed"; + +const MAX_BUFFER_CHUNKS = 500; // ~10s at 20ms per chunk + +export type TranscriptEvent = { + type?: string; + item_id?: string; + content_index?: number; + delta?: string; + transcript?: string; + message?: string; + error?: { message?: string }; +}; + +export function createPeerConnection(): RTCPeerConnection { + return new RTCPeerConnection(); +} + +export async function connectPeerConnection(options: { + peerConnection: RTCPeerConnection; + clientSecret: string; +}): Promise { + const offer = await options.peerConnection.createOffer(); + await options.peerConnection.setLocalDescription(offer); + + const response = await fetch(OPENAI_REALTIME_WEBRTC_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${options.clientSecret}`, + "Content-Type": "application/sdp", + }, + body: offer.sdp ?? "", + }); + + const body = await response.text(); + if (!response.ok) { + throw new Error( + `OpenAI realtime connection failed (${response.status}): ${body}`, + ); + } + + await options.peerConnection.setRemoteDescription({ + type: "answer", + sdp: body, + }); +} + +export function mergeTranscriptSegment( + currentText: string, + segmentText: string, + event: TranscriptEvent, +): string { + if (!segmentText) return currentText; + if (!currentText) return segmentText; + + // Completed events re-send the full segment text; skip if already present. + if (event.type === TRANSCRIPT_COMPLETED_EVENT) { + const normalizedCurrent = currentText.trimEnd().toLowerCase(); + const normalizedText = segmentText.trim().toLowerCase(); + if (normalizedCurrent.endsWith(normalizedText)) { + return currentText; + } + } + + return currentText + segmentText; +} + +// ── Audio buffer capture ────────────────────────────────────────────────── + +export interface AudioBufferCapture { + chunks: Int16Array[]; + close(): void; +} + +export async function createAudioBufferCapture( + stream: MediaStream, +): Promise { + const audioContext = new AudioContext(); + const blobUrl = createWorkletBlobUrl(); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode( + audioContext, + REALTIME_BUFFER_PROCESSOR_NAME, + ); + source.connect(worklet); + worklet.connect(audioContext.destination); + + const chunks: Int16Array[] = []; + worklet.port.onmessage = (event: MessageEvent) => { + if (chunks.length < MAX_BUFFER_CHUNKS) { + chunks.push(new Int16Array(event.data)); + } + }; + + return { + chunks, + close() { + worklet.disconnect(); + source.disconnect(); + void audioContext.close(); + }, + }; +} + +// ── Flush buffered PCM into the data channel ────────────────────────────── + +function int16ToBase64(pcm: Int16Array): string { + const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength); + let binary = ""; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +export function flushAudioBuffer( + dataChannel: RTCDataChannel, + chunks: Int16Array[], +): void { + for (const chunk of chunks) { + dataChannel.send( + JSON.stringify({ + type: "input_audio_buffer.append", + audio: int16ToBase64(chunk), + }), + ); + } + chunks.length = 0; +} diff --git a/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts b/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts new file mode 100644 index 000000000..ac8ffaabd --- /dev/null +++ b/desktop/src/features/dictation/lib/realtimeBufferWorklet.ts @@ -0,0 +1,53 @@ +const TARGET_SAMPLE_RATE = 24000; +const FRAME_SAMPLES = 480; // 20ms at 24kHz + +export const REALTIME_BUFFER_PROCESSOR_NAME = "realtime-buffer-processor"; + +export const REALTIME_BUFFER_WORKLET_SOURCE = /* js */ ` +class RealtimeBufferProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._ratio = sampleRate / ${TARGET_SAMPLE_RATE}; + this._offset = 0; + this._buf = new Float32Array(${FRAME_SAMPLES}); + this._idx = 0; + } + + process(inputs) { + const input = inputs[0]?.[0]; + if (!input) return true; + + while (this._offset < input.length) { + const i = Math.floor(this._offset); + const frac = this._offset - i; + const s0 = input[i]; + const s1 = i + 1 < input.length ? input[i + 1] : s0; + this._buf[this._idx++] = s0 + frac * (s1 - s0); + + if (this._idx >= ${FRAME_SAMPLES}) { + const pcm = new Int16Array(${FRAME_SAMPLES}); + for (let j = 0; j < ${FRAME_SAMPLES}; j++) { + const s = Math.max(-1, Math.min(1, this._buf[j])); + pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + this._idx = 0; + } + this._offset += this._ratio; + } + this._offset -= input.length; + return true; + } +} + +registerProcessor('${REALTIME_BUFFER_PROCESSOR_NAME}', RealtimeBufferProcessor); +`; + +/** Create a blob URL that can be passed to `audioWorklet.addModule()`. */ +export function createWorkletBlobUrl(): string { + return URL.createObjectURL( + new Blob([REALTIME_BUFFER_WORKLET_SOURCE], { + type: "application/javascript", + }), + ); +} diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 000000000..334deda02 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,108 @@ +export const DEFAULT_AUTO_SUBMIT_PHRASE = "submit"; + +const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizePhrase(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, " ") + .trim() + .replace(TRAILING_PUNCTUATION_REGEX, "") + .trim(); +} + +export function parseAutoSubmitPhrases( + rawValue: string | null | undefined, +): string[] { + if (!rawValue) return []; + return Array.from( + new Set( + rawValue + .split(",") + .map((value) => normalizePhrase(value)) + .filter(Boolean), + ), + ); +} + +function appendTranscribedText(baseText: string, fragment: string): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export function replaceTrailingTranscribedText( + fullText: string, + previousTranscribedText: string, + nextTranscribedText: string, +): string { + if (!previousTranscribedText) { + return appendTranscribedText(fullText, nextTranscribedText); + } + + if (fullText.endsWith(previousTranscribedText)) { + return appendTranscribedText( + fullText.slice(0, -previousTranscribedText.length), + nextTranscribedText, + ); + } + + const trimmedPreviousText = previousTranscribedText.trim(); + if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { + return appendTranscribedText( + fullText.slice(0, -trimmedPreviousText.length), + nextTranscribedText, + ); + } + + return appendTranscribedText(fullText, nextTranscribedText); +} + +export function getAutoSubmitMatch( + transcribedText: string, + autoSubmitPhrases: string[], +): { matchedPhrase: string; textWithoutPhrase: string } | null { + const normalizedTranscribedText = normalizePhrase(transcribedText); + if (!normalizedTranscribedText) return null; + + const sortedPhrases = [...autoSubmitPhrases].sort( + (left, right) => right.length - left.length, + ); + + for (const phrase of sortedPhrases) { + if (!normalizedTranscribedText.endsWith(phrase)) continue; + + const phraseStartIndex = normalizedTranscribedText.length - phrase.length; + if ( + phraseStartIndex > 0 && + normalizedTranscribedText[phraseStartIndex - 1] !== " " + ) { + continue; + } + + const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); + const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); + const phrasePattern = new RegExp( + `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, + "iu", + ); + const rawMatch = trimmedText.match(phrasePattern); + const phraseStartOffset = + rawMatch && rawMatch.index !== undefined + ? rawMatch.index + (rawMatch[1]?.length ?? 0) + : trimmedText.length - phrase.length; + const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); + + return { matchedPhrase: phrase, textWithoutPhrase }; + } + + return null; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 000000000..b43249cdf --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,54 @@ +import { Mic } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : "Dictate message"; + + return ( + + + + + {tooltipText} + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 11e619012..d8d21d0e8 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -54,6 +54,7 @@ import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { DictationButton, useComposerDictation } from "@/features/dictation"; type MessageComposerProps = { channelId?: string | null; @@ -265,6 +266,15 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + + const dictation = useComposerDictation({ + contentRef, + disabled, + isSending, + setComposerContentFromText, + submitMessageRef, + }); + const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler @@ -997,7 +1007,12 @@ function MessageComposerImpl({ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen}