mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
feat: add real-time voice dictation to composer
Adds dictation support using OpenAI's Realtime API over WebRTC:
Relay:
- New /transcribe/status and /transcribe/session endpoints
- BUZZ_OPENAI_API_KEY env var gates the feature (hidden when absent)
- Proxies ephemeral client-secret minting from OpenAI
Desktop:
- New features/dictation module with:
- AudioWorklet for 24kHz PCM capture + buffering
- WebRTC peer connection to OpenAI Realtime API
- Real-time transcript merging into composer
- Auto-submit on trigger phrase ('submit')
- Mic button in composer toolbar (red pulse when recording)
- Integrated into MessageComposer via useComposerDictation hook
Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
klopez4212
@@ -51,6 +51,14 @@ RELAY_URL=ws://localhost:3000
|
||||
# (use `just web` for Vite HMR instead).
|
||||
# BUZZ_WEB_DIR=./web/dist
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Transcription (dictation)
|
||||
# -----------------------------------------------------------------------------
|
||||
# OpenAI API key for real-time voice transcription in the composer.
|
||||
# When absent, the dictation mic button is hidden.
|
||||
# BUZZ_OPENAI_API_KEY=sk-...
|
||||
# BUZZ_TRANSCRIPTION_MODEL=whisper-1
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Git (NIP-34 bare repositories)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -72,6 +72,7 @@ url = { workspace = true }
|
||||
moka = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
metrics-exporter-prometheus = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
|
||||
[features]
|
||||
dev = ["buzz-auth/dev"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! HTTP API — media, git, NIP-05, and the Nostr HTTP bridge.
|
||||
//! HTTP API — media, git, NIP-05, transcription, and the Nostr HTTP bridge.
|
||||
|
||||
pub mod bridge;
|
||||
pub mod events;
|
||||
@@ -7,6 +7,7 @@ pub mod invites;
|
||||
pub mod media;
|
||||
pub mod nip05;
|
||||
pub mod operator;
|
||||
pub mod transcribe;
|
||||
|
||||
// Re-export imeta helpers used by ingest pipeline.
|
||||
pub use crate::handlers::imeta::{validate_imeta_tags, verify_imeta_blobs};
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting.
|
||||
//!
|
||||
//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client
|
||||
//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a
|
||||
//! WebRTC connection for real-time speech-to-text dictation.
|
||||
|
||||
use axum::{extract::State, http::StatusCode, response::Json};
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions";
|
||||
const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1";
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct TranscribeStatus {
|
||||
configured: bool,
|
||||
model: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TranscribeSession {
|
||||
client_secret: String,
|
||||
model: String,
|
||||
}
|
||||
|
||||
/// `GET /transcribe/status` — check if transcription is configured.
|
||||
pub async fn transcribe_status(State(state): State<Arc<AppState>>) -> Json<TranscribeStatus> {
|
||||
Json(TranscribeStatus {
|
||||
configured: state.config.openai_api_key.is_some(),
|
||||
model: transcription_model(),
|
||||
})
|
||||
}
|
||||
|
||||
/// `POST /transcribe/session` — create an ephemeral OpenAI Realtime session.
|
||||
///
|
||||
/// Returns a short-lived client secret that the frontend uses to establish
|
||||
/// a WebRTC connection directly with OpenAI for real-time transcription.
|
||||
pub async fn create_transcribe_session(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<TranscribeSession>, (StatusCode, Json<serde_json::Value>)> {
|
||||
let api_key = state.config.openai_api_key.as_deref().ok_or_else(|| {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(serde_json::json!({
|
||||
"error": "transcription_not_configured",
|
||||
"message": "Transcription is not configured on this relay"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let model = transcription_model();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let response = client
|
||||
.post(OPENAI_REALTIME_SESSIONS_URL)
|
||||
.header("Authorization", format!("Bearer {api_key}"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&serde_json::json!({
|
||||
"model": "gpt-4o-mini-realtime-preview",
|
||||
"modalities": ["text"],
|
||||
"input_audio_transcription": {
|
||||
"model": model,
|
||||
},
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
}
|
||||
}))
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("OpenAI realtime session request failed: {e}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Failed to create transcription session"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
tracing::error!("OpenAI realtime session error ({status}): {body}");
|
||||
return Err((
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "OpenAI rejected the transcription session request"
|
||||
})),
|
||||
));
|
||||
}
|
||||
|
||||
let body: serde_json::Value = response.json().await.map_err(|e| {
|
||||
tracing::error!("OpenAI realtime session response parse error: {e}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Invalid response from transcription service"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let client_secret = extract_client_secret(&body).ok_or_else(|| {
|
||||
tracing::error!("OpenAI realtime session response missing client_secret: {body}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Transcription service returned unexpected response"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(Json(TranscribeSession {
|
||||
client_secret,
|
||||
model,
|
||||
}))
|
||||
}
|
||||
|
||||
fn transcription_model() -> String {
|
||||
std::env::var("BUZZ_TRANSCRIPTION_MODEL")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string())
|
||||
}
|
||||
|
||||
fn extract_client_secret(value: &serde_json::Value) -> Option<String> {
|
||||
// Shape 1: { "client_secret": { "value": "..." } }
|
||||
if let Some(cs) = value.get("client_secret") {
|
||||
if let Some(v) = cs.get("value").and_then(|v| v.as_str()) {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
// Shape 2: { "client_secret": "..." }
|
||||
if let Some(v) = cs.as_str() {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
}
|
||||
// Shape 3: { "value": "..." }
|
||||
value
|
||||
.get("value")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::extract_client_secret;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parses_nested_client_secret() {
|
||||
let body = json!({ "client_secret": { "value": "sec_abc123", "expires_at": 9999 } });
|
||||
assert_eq!(extract_client_secret(&body), Some("sec_abc123".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_direct_string_client_secret() {
|
||||
let body = json!({ "client_secret": "sec_direct" });
|
||||
assert_eq!(extract_client_secret(&body), Some("sec_direct".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_top_level_value() {
|
||||
let body = json!({ "value": "sec_toplevel" });
|
||||
assert_eq!(
|
||||
extract_client_secret(&body),
|
||||
Some("sec_toplevel".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_for_missing_secret() {
|
||||
let body = json!({ "id": "sess_123", "model": "gpt-4o" });
|
||||
assert_eq!(extract_client_secret(&body), None);
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,11 @@ pub struct Config {
|
||||
/// Used to authenticate internal policy endpoint requests.
|
||||
pub git_hook_hmac_secret: String,
|
||||
|
||||
/// Optional OpenAI API key for real-time transcription (dictation).
|
||||
/// When absent, the `/transcribe/session` endpoint returns 503 and the
|
||||
/// desktop mic button stays hidden.
|
||||
pub openai_api_key: Option<String>,
|
||||
|
||||
/// Optional path to the web UI `dist/` directory.
|
||||
/// When set, the relay serves the SPA from this directory for browser requests.
|
||||
/// When unset, no static file serving happens (relay behaves as before).
|
||||
@@ -476,6 +481,11 @@ impl Config {
|
||||
let secret: [u8; 32] = rand::random();
|
||||
hex::encode(secret)
|
||||
});
|
||||
let openai_api_key = std::env::var("BUZZ_OPENAI_API_KEY")
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
// Web UI static file serving
|
||||
let web_dir = std::env::var("BUZZ_WEB_DIR")
|
||||
.ok()
|
||||
@@ -538,6 +548,7 @@ impl Config {
|
||||
git_max_repos_per_pubkey,
|
||||
git_max_concurrent_ops,
|
||||
git_hook_hmac_secret,
|
||||
openai_api_key,
|
||||
web_dir,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -80,6 +80,15 @@ pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
)
|
||||
// Webhook trigger (secret-authenticated, no NIP-98)
|
||||
.route("/hooks/{id}", post(api::bridge::workflow_webhook))
|
||||
// Transcription (dictation) — proxies OpenAI Realtime client-secret minting
|
||||
.route(
|
||||
"/transcribe/status",
|
||||
get(api::transcribe::transcribe_status),
|
||||
)
|
||||
.route(
|
||||
"/transcribe/session",
|
||||
post(api::transcribe::create_transcribe_session),
|
||||
)
|
||||
// Huddle audio WebSocket route
|
||||
.route(
|
||||
"/huddle/{channel_id}/audio",
|
||||
@@ -112,6 +121,7 @@ pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
|| path.starts_with("/internal/")
|
||||
|| path.starts_with("/.well-known/")
|
||||
|| path.starts_with("/huddle/")
|
||||
|| path.starts_with("/transcribe/")
|
||||
|| path == "/health"
|
||||
|| path == "/_liveness"
|
||||
|| path == "/_readiness"
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { getRelayHttpUrl } from "@/shared/api/tauri";
|
||||
|
||||
export interface TranscribeStatus {
|
||||
configured: boolean;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface TranscribeSession {
|
||||
clientSecret: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export async function getTranscribeStatus(): Promise<TranscribeStatus> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/status`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Transcribe status check failed: ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function createTranscribeSession(): Promise<TranscribeSession> {
|
||||
const baseUrl = await getRelayHttpUrl();
|
||||
const response = await fetch(`${baseUrl}/transcribe/session`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => "");
|
||||
throw new Error(
|
||||
`Failed to create transcribe session (${response.status}): ${body}`,
|
||||
);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type * as React from "react";
|
||||
import { useDictation } from "./useDictation";
|
||||
|
||||
interface UseComposerDictationOptions {
|
||||
contentRef: React.MutableRefObject<string>;
|
||||
disabled: boolean;
|
||||
isSending: boolean;
|
||||
setComposerContentFromText: (text: string) => void;
|
||||
submitMessageRef: React.MutableRefObject<() => void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around `useDictation` pre-wired for the MessageComposer's
|
||||
* state management (contentRef, setComposerContentFromText, submitMessageRef).
|
||||
*/
|
||||
export function useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
submitMessageRef,
|
||||
}: UseComposerDictationOptions) {
|
||||
return useDictation({
|
||||
text: contentRef.current,
|
||||
setText: setComposerContentFromText,
|
||||
onSend: (text) => {
|
||||
setComposerContentFromText(text);
|
||||
queueMicrotask(() => submitMessageRef.current());
|
||||
},
|
||||
sendDisabled: disabled || isSending,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { useCallback, useMemo, useRef } from "react";
|
||||
import {
|
||||
DEFAULT_AUTO_SUBMIT_PHRASE,
|
||||
getAutoSubmitMatch,
|
||||
parseAutoSubmitPhrases,
|
||||
replaceTrailingTranscribedText,
|
||||
} from "../lib/voiceInput";
|
||||
import { useRealtimeDictation } from "./useRealtimeDictation";
|
||||
|
||||
interface UseDictationOptions {
|
||||
/** Current composer text */
|
||||
text: string;
|
||||
/** Set composer text */
|
||||
setText: (value: string) => void;
|
||||
/** Send the message */
|
||||
onSend: (text: string) => void;
|
||||
/** Whether sending is currently blocked */
|
||||
sendDisabled?: boolean;
|
||||
}
|
||||
|
||||
export function useDictation({
|
||||
text,
|
||||
setText,
|
||||
onSend,
|
||||
sendDisabled = false,
|
||||
}: UseDictationOptions) {
|
||||
const autoSubmitPhrases = useMemo(
|
||||
() => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE),
|
||||
[],
|
||||
);
|
||||
const stopRecordingRef = useRef<() => void>(() => {});
|
||||
const textRef = useRef(text);
|
||||
textRef.current = text;
|
||||
const lastTranscriptRef = useRef("");
|
||||
|
||||
const handleTranscript = useCallback(
|
||||
(transcript: string) => {
|
||||
const previous = lastTranscriptRef.current;
|
||||
const latest = textRef.current;
|
||||
const merged = replaceTrailingTranscribedText(
|
||||
latest,
|
||||
previous,
|
||||
transcript,
|
||||
);
|
||||
const match = getAutoSubmitMatch(transcript, autoSubmitPhrases);
|
||||
|
||||
if (!match) {
|
||||
setText(merged);
|
||||
textRef.current = merged;
|
||||
lastTranscriptRef.current = transcript;
|
||||
return;
|
||||
}
|
||||
|
||||
const textWithoutPhrase = replaceTrailingTranscribedText(
|
||||
latest,
|
||||
previous,
|
||||
match.textWithoutPhrase,
|
||||
);
|
||||
if (!textWithoutPhrase.trim()) return;
|
||||
|
||||
stopRecordingRef.current();
|
||||
|
||||
if (sendDisabled) {
|
||||
setText(textWithoutPhrase);
|
||||
textRef.current = textWithoutPhrase;
|
||||
return;
|
||||
}
|
||||
|
||||
onSend(textWithoutPhrase.trim());
|
||||
setText("");
|
||||
textRef.current = "";
|
||||
lastTranscriptRef.current = "";
|
||||
},
|
||||
[autoSubmitPhrases, onSend, sendDisabled, setText],
|
||||
);
|
||||
|
||||
const dictation = useRealtimeDictation({
|
||||
onRecordingStart: () => {
|
||||
lastTranscriptRef.current = "";
|
||||
},
|
||||
onTranscriptText: handleTranscript,
|
||||
});
|
||||
stopRecordingRef.current = dictation.stopRecording;
|
||||
|
||||
return dictation;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
createTranscribeSession,
|
||||
getTranscribeStatus,
|
||||
} from "../api/transcribeSession";
|
||||
import {
|
||||
type AudioBufferCapture,
|
||||
type TranscriptEvent,
|
||||
TRANSCRIPT_COMPLETED_EVENT,
|
||||
TRANSCRIPT_DELTA_EVENT,
|
||||
connectPeerConnection,
|
||||
createAudioBufferCapture,
|
||||
createPeerConnection,
|
||||
flushAudioBuffer,
|
||||
mergeTranscriptSegment,
|
||||
} from "../lib/realtimeAudio";
|
||||
|
||||
interface UseRealtimeDictationOptions {
|
||||
disabled?: boolean;
|
||||
onRecordingStart?: () => void;
|
||||
onTranscriptText: (text: string) => void;
|
||||
}
|
||||
|
||||
function closeResources(resources: {
|
||||
audioCapture?: AudioBufferCapture | null;
|
||||
dataChannel?: RTCDataChannel | null;
|
||||
peerConnection?: RTCPeerConnection | null;
|
||||
stream?: MediaStream | null;
|
||||
}) {
|
||||
resources.audioCapture?.close();
|
||||
resources.dataChannel?.close();
|
||||
resources.peerConnection?.close();
|
||||
for (const track of resources.stream?.getTracks() ?? []) {
|
||||
track.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export function useRealtimeDictation({
|
||||
disabled = false,
|
||||
onRecordingStart,
|
||||
onTranscriptText,
|
||||
}: UseRealtimeDictationOptions) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isStarting, setIsStarting] = useState(false);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [isConfigured, setIsConfigured] = useState(false);
|
||||
|
||||
const peerConnectionRef = useRef<RTCPeerConnection | null>(null);
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const audioCaptureRef = useRef<AudioBufferCapture | null>(null);
|
||||
const transcriptRef = useRef("");
|
||||
const activeRunIdRef = useRef(0);
|
||||
const onRecordingStartRef = useRef(onRecordingStart);
|
||||
const onTranscriptTextRef = useRef(onTranscriptText);
|
||||
|
||||
onRecordingStartRef.current = onRecordingStart;
|
||||
onTranscriptTextRef.current = onTranscriptText;
|
||||
|
||||
const isEnabled = !disabled && isConfigured;
|
||||
|
||||
// Check if transcription is configured on mount
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
getTranscribeStatus()
|
||||
.then((status) => {
|
||||
if (!cancelled) setIsConfigured(status.configured);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setIsConfigured(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const cleanupResources = useCallback(() => {
|
||||
activeRunIdRef.current += 1;
|
||||
closeResources({
|
||||
audioCapture: audioCaptureRef.current,
|
||||
dataChannel: dataChannelRef.current,
|
||||
peerConnection: peerConnectionRef.current,
|
||||
stream: streamRef.current,
|
||||
});
|
||||
audioCaptureRef.current = null;
|
||||
dataChannelRef.current = null;
|
||||
peerConnectionRef.current = null;
|
||||
streamRef.current = null;
|
||||
}, []);
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
cleanupResources();
|
||||
setIsRecording(false);
|
||||
setIsStarting(false);
|
||||
setIsTranscribing(false);
|
||||
}, [cleanupResources]);
|
||||
|
||||
useEffect(() => cleanupResources, [cleanupResources]);
|
||||
|
||||
const handleRealtimeEvent = useCallback((event: TranscriptEvent) => {
|
||||
if (event.type === "error") {
|
||||
console.error("OpenAI realtime server error", event);
|
||||
toast.error(event.error?.message ?? "Voice input error");
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
event.type !== TRANSCRIPT_DELTA_EVENT &&
|
||||
event.type !== TRANSCRIPT_COMPLETED_EVENT
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = event.delta ?? event.transcript ?? "";
|
||||
const merged = mergeTranscriptSegment(transcriptRef.current, text, event);
|
||||
|
||||
if (merged === transcriptRef.current) return;
|
||||
|
||||
transcriptRef.current = merged;
|
||||
onTranscriptTextRef.current(merged);
|
||||
setIsTranscribing(event.type !== TRANSCRIPT_COMPLETED_EVENT);
|
||||
}, []);
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
if (!isEnabled || isStarting || isRecording) return;
|
||||
|
||||
const runId = activeRunIdRef.current + 1;
|
||||
activeRunIdRef.current = runId;
|
||||
const isStaleRun = () => activeRunIdRef.current !== runId;
|
||||
|
||||
let stream: MediaStream | null = null;
|
||||
let audioCapture: AudioBufferCapture | null = null;
|
||||
let peerConnection: RTCPeerConnection | null = null;
|
||||
let dataChannel: RTCDataChannel | null = null;
|
||||
|
||||
setIsStarting(true);
|
||||
transcriptRef.current = "";
|
||||
onRecordingStartRef.current?.();
|
||||
|
||||
try {
|
||||
// 1. Capture mic immediately for instant feedback
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
});
|
||||
if (isStaleRun()) {
|
||||
closeResources({ stream });
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
setIsRecording(true);
|
||||
|
||||
// 2. Buffer PCM via AudioWorklet while network calls proceed
|
||||
audioCapture = await createAudioBufferCapture(stream);
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, stream });
|
||||
return;
|
||||
}
|
||||
audioCaptureRef.current = audioCapture;
|
||||
|
||||
// 3. Create session via relay
|
||||
const session = await createTranscribeSession();
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, stream });
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. Set up WebRTC
|
||||
peerConnection = createPeerConnection();
|
||||
peerConnectionRef.current = peerConnection;
|
||||
const activeStream = stream;
|
||||
stream.getAudioTracks().forEach((track) => {
|
||||
peerConnection?.addTrack(track, activeStream);
|
||||
});
|
||||
|
||||
dataChannel = peerConnection.createDataChannel("oai-events");
|
||||
dataChannelRef.current = dataChannel;
|
||||
dataChannel.addEventListener("message", (message) => {
|
||||
try {
|
||||
handleRealtimeEvent(JSON.parse(String(message.data)));
|
||||
} catch {
|
||||
// Ignore non-JSON events
|
||||
}
|
||||
});
|
||||
|
||||
// Flush buffered audio once data channel opens
|
||||
const channelToFlush = dataChannel;
|
||||
const captureToFlush = audioCapture;
|
||||
dataChannel.addEventListener("open", () => {
|
||||
flushAudioBuffer(channelToFlush, captureToFlush.chunks);
|
||||
captureToFlush.close();
|
||||
audioCaptureRef.current = null;
|
||||
});
|
||||
|
||||
// 5. SDP exchange
|
||||
await connectPeerConnection({
|
||||
peerConnection,
|
||||
clientSecret: session.clientSecret,
|
||||
});
|
||||
if (isStaleRun()) {
|
||||
closeResources({ audioCapture, dataChannel, peerConnection, stream });
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
closeResources({ audioCapture, dataChannel, peerConnection, stream });
|
||||
if (!isStaleRun()) {
|
||||
audioCaptureRef.current = null;
|
||||
dataChannelRef.current = null;
|
||||
peerConnectionRef.current = null;
|
||||
streamRef.current = null;
|
||||
setIsRecording(false);
|
||||
setIsTranscribing(false);
|
||||
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Voice input failed";
|
||||
if (/not allowed|denied|permission/i.test(message)) {
|
||||
toast.error("Microphone access denied", {
|
||||
description:
|
||||
"Allow microphone access in System Settings to use dictation.",
|
||||
});
|
||||
} else if (/not found|no audio/i.test(message)) {
|
||||
toast.error("No microphone found", {
|
||||
description: "Connect a microphone and try again.",
|
||||
});
|
||||
} else {
|
||||
toast.error("Voice input failed", { description: message });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (!isStaleRun()) setIsStarting(false);
|
||||
}
|
||||
}, [handleRealtimeEvent, isEnabled, isRecording, isStarting]);
|
||||
|
||||
const stopRecording = useCallback(() => cleanup(), [cleanup]);
|
||||
|
||||
const toggleRecording = useCallback(() => {
|
||||
if (isRecording || isStarting) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
void startRecording();
|
||||
}, [isRecording, isStarting, startRecording, stopRecording]);
|
||||
|
||||
return {
|
||||
isEnabled,
|
||||
isRecording,
|
||||
isStarting,
|
||||
isTranscribing,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
toggleRecording,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export { useComposerDictation } from "./hooks/useComposerDictation";
|
||||
export { useDictation } from "./hooks/useDictation";
|
||||
export { DictationButton } from "./ui/DictationButton";
|
||||
@@ -0,0 +1,145 @@
|
||||
import {
|
||||
REALTIME_BUFFER_PROCESSOR_NAME,
|
||||
createWorkletBlobUrl,
|
||||
} from "./realtimeBufferWorklet";
|
||||
|
||||
export const OPENAI_REALTIME_WEBRTC_URL =
|
||||
"https://api.openai.com/v1/realtime/calls";
|
||||
export const TRANSCRIPT_DELTA_EVENT =
|
||||
"conversation.item.input_audio_transcription.delta";
|
||||
export const TRANSCRIPT_COMPLETED_EVENT =
|
||||
"conversation.item.input_audio_transcription.completed";
|
||||
|
||||
const MAX_BUFFER_CHUNKS = 500; // ~10s at 20ms per chunk
|
||||
|
||||
export type TranscriptEvent = {
|
||||
type?: string;
|
||||
item_id?: string;
|
||||
content_index?: number;
|
||||
delta?: string;
|
||||
transcript?: string;
|
||||
message?: string;
|
||||
error?: { message?: string };
|
||||
};
|
||||
|
||||
export function createPeerConnection(): RTCPeerConnection {
|
||||
return new RTCPeerConnection();
|
||||
}
|
||||
|
||||
export async function connectPeerConnection(options: {
|
||||
peerConnection: RTCPeerConnection;
|
||||
clientSecret: string;
|
||||
}): Promise<void> {
|
||||
const offer = await options.peerConnection.createOffer();
|
||||
await options.peerConnection.setLocalDescription(offer);
|
||||
|
||||
const response = await fetch(OPENAI_REALTIME_WEBRTC_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${options.clientSecret}`,
|
||||
"Content-Type": "application/sdp",
|
||||
},
|
||||
body: offer.sdp ?? "",
|
||||
});
|
||||
|
||||
const body = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`OpenAI realtime connection failed (${response.status}): ${body}`,
|
||||
);
|
||||
}
|
||||
|
||||
await options.peerConnection.setRemoteDescription({
|
||||
type: "answer",
|
||||
sdp: body,
|
||||
});
|
||||
}
|
||||
|
||||
export function mergeTranscriptSegment(
|
||||
currentText: string,
|
||||
segmentText: string,
|
||||
event: TranscriptEvent,
|
||||
): string {
|
||||
if (!segmentText) return currentText;
|
||||
if (!currentText) return segmentText;
|
||||
|
||||
// Completed events re-send the full segment text; skip if already present.
|
||||
if (event.type === TRANSCRIPT_COMPLETED_EVENT) {
|
||||
const normalizedCurrent = currentText.trimEnd().toLowerCase();
|
||||
const normalizedText = segmentText.trim().toLowerCase();
|
||||
if (normalizedCurrent.endsWith(normalizedText)) {
|
||||
return currentText;
|
||||
}
|
||||
}
|
||||
|
||||
return currentText + segmentText;
|
||||
}
|
||||
|
||||
// ── Audio buffer capture ──────────────────────────────────────────────────
|
||||
|
||||
export interface AudioBufferCapture {
|
||||
chunks: Int16Array[];
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export async function createAudioBufferCapture(
|
||||
stream: MediaStream,
|
||||
): Promise<AudioBufferCapture> {
|
||||
const audioContext = new AudioContext();
|
||||
const blobUrl = createWorkletBlobUrl();
|
||||
try {
|
||||
await audioContext.audioWorklet.addModule(blobUrl);
|
||||
} finally {
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
}
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const worklet = new AudioWorkletNode(
|
||||
audioContext,
|
||||
REALTIME_BUFFER_PROCESSOR_NAME,
|
||||
);
|
||||
source.connect(worklet);
|
||||
worklet.connect(audioContext.destination);
|
||||
|
||||
const chunks: Int16Array[] = [];
|
||||
worklet.port.onmessage = (event: MessageEvent<ArrayBuffer>) => {
|
||||
if (chunks.length < MAX_BUFFER_CHUNKS) {
|
||||
chunks.push(new Int16Array(event.data));
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
chunks,
|
||||
close() {
|
||||
worklet.disconnect();
|
||||
source.disconnect();
|
||||
void audioContext.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Flush buffered PCM into the data channel ──────────────────────────────
|
||||
|
||||
function int16ToBase64(pcm: Int16Array): string {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
export function flushAudioBuffer(
|
||||
dataChannel: RTCDataChannel,
|
||||
chunks: Int16Array[],
|
||||
): void {
|
||||
for (const chunk of chunks) {
|
||||
dataChannel.send(
|
||||
JSON.stringify({
|
||||
type: "input_audio_buffer.append",
|
||||
audio: int16ToBase64(chunk),
|
||||
}),
|
||||
);
|
||||
}
|
||||
chunks.length = 0;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
const FRAME_SAMPLES = 480; // 20ms at 24kHz
|
||||
|
||||
export const REALTIME_BUFFER_PROCESSOR_NAME = "realtime-buffer-processor";
|
||||
|
||||
export const REALTIME_BUFFER_WORKLET_SOURCE = /* js */ `
|
||||
class RealtimeBufferProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this._ratio = sampleRate / ${TARGET_SAMPLE_RATE};
|
||||
this._offset = 0;
|
||||
this._buf = new Float32Array(${FRAME_SAMPLES});
|
||||
this._idx = 0;
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0]?.[0];
|
||||
if (!input) return true;
|
||||
|
||||
while (this._offset < input.length) {
|
||||
const i = Math.floor(this._offset);
|
||||
const frac = this._offset - i;
|
||||
const s0 = input[i];
|
||||
const s1 = i + 1 < input.length ? input[i + 1] : s0;
|
||||
this._buf[this._idx++] = s0 + frac * (s1 - s0);
|
||||
|
||||
if (this._idx >= ${FRAME_SAMPLES}) {
|
||||
const pcm = new Int16Array(${FRAME_SAMPLES});
|
||||
for (let j = 0; j < ${FRAME_SAMPLES}; j++) {
|
||||
const s = Math.max(-1, Math.min(1, this._buf[j]));
|
||||
pcm[j] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
this.port.postMessage(pcm.buffer, [pcm.buffer]);
|
||||
this._idx = 0;
|
||||
}
|
||||
this._offset += this._ratio;
|
||||
}
|
||||
this._offset -= input.length;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('${REALTIME_BUFFER_PROCESSOR_NAME}', RealtimeBufferProcessor);
|
||||
`;
|
||||
|
||||
/** Create a blob URL that can be passed to `audioWorklet.addModule()`. */
|
||||
export function createWorkletBlobUrl(): string {
|
||||
return URL.createObjectURL(
|
||||
new Blob([REALTIME_BUFFER_WORKLET_SOURCE], {
|
||||
type: "application/javascript",
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
export const DEFAULT_AUTO_SUBMIT_PHRASE = "submit";
|
||||
|
||||
const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u;
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function normalizePhrase(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.replace(TRAILING_PUNCTUATION_REGEX, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function parseAutoSubmitPhrases(
|
||||
rawValue: string | null | undefined,
|
||||
): string[] {
|
||||
if (!rawValue) return [];
|
||||
return Array.from(
|
||||
new Set(
|
||||
rawValue
|
||||
.split(",")
|
||||
.map((value) => normalizePhrase(value))
|
||||
.filter(Boolean),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function appendTranscribedText(baseText: string, fragment: string): string {
|
||||
const normalizedFragment = fragment.replace(/\s+/g, " ").trim();
|
||||
if (!normalizedFragment) return baseText;
|
||||
if (!baseText.trim()) return normalizedFragment;
|
||||
if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) {
|
||||
return `${baseText}${normalizedFragment}`;
|
||||
}
|
||||
return `${baseText} ${normalizedFragment}`;
|
||||
}
|
||||
|
||||
export function replaceTrailingTranscribedText(
|
||||
fullText: string,
|
||||
previousTranscribedText: string,
|
||||
nextTranscribedText: string,
|
||||
): string {
|
||||
if (!previousTranscribedText) {
|
||||
return appendTranscribedText(fullText, nextTranscribedText);
|
||||
}
|
||||
|
||||
if (fullText.endsWith(previousTranscribedText)) {
|
||||
return appendTranscribedText(
|
||||
fullText.slice(0, -previousTranscribedText.length),
|
||||
nextTranscribedText,
|
||||
);
|
||||
}
|
||||
|
||||
const trimmedPreviousText = previousTranscribedText.trim();
|
||||
if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) {
|
||||
return appendTranscribedText(
|
||||
fullText.slice(0, -trimmedPreviousText.length),
|
||||
nextTranscribedText,
|
||||
);
|
||||
}
|
||||
|
||||
return appendTranscribedText(fullText, nextTranscribedText);
|
||||
}
|
||||
|
||||
export function getAutoSubmitMatch(
|
||||
transcribedText: string,
|
||||
autoSubmitPhrases: string[],
|
||||
): { matchedPhrase: string; textWithoutPhrase: string } | null {
|
||||
const normalizedTranscribedText = normalizePhrase(transcribedText);
|
||||
if (!normalizedTranscribedText) return null;
|
||||
|
||||
const sortedPhrases = [...autoSubmitPhrases].sort(
|
||||
(left, right) => right.length - left.length,
|
||||
);
|
||||
|
||||
for (const phrase of sortedPhrases) {
|
||||
if (!normalizedTranscribedText.endsWith(phrase)) continue;
|
||||
|
||||
const phraseStartIndex = normalizedTranscribedText.length - phrase.length;
|
||||
if (
|
||||
phraseStartIndex > 0 &&
|
||||
normalizedTranscribedText[phraseStartIndex - 1] !== " "
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, "");
|
||||
const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp);
|
||||
const phrasePattern = new RegExp(
|
||||
`(^|\\s)(${phraseWords.join("\\s+")})\\s*$`,
|
||||
"iu",
|
||||
);
|
||||
const rawMatch = trimmedText.match(phrasePattern);
|
||||
const phraseStartOffset =
|
||||
rawMatch && rawMatch.index !== undefined
|
||||
? rawMatch.index + (rawMatch[1]?.length ?? 0)
|
||||
: trimmedText.length - phrase.length;
|
||||
const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd();
|
||||
|
||||
return { matchedPhrase: phrase, textWithoutPhrase };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Mic } from "lucide-react";
|
||||
import { Button } from "@/shared/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip";
|
||||
import { cn } from "@/shared/lib/cn";
|
||||
|
||||
interface DictationState {
|
||||
isEnabled: boolean;
|
||||
isRecording: boolean;
|
||||
isStarting: boolean;
|
||||
isTranscribing: boolean;
|
||||
toggleRecording: () => void;
|
||||
}
|
||||
|
||||
interface DictationButtonProps {
|
||||
dictation: DictationState;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
export function DictationButton({
|
||||
dictation,
|
||||
disabled = false,
|
||||
}: DictationButtonProps) {
|
||||
if (!dictation.isEnabled) return null;
|
||||
|
||||
const tooltipText = dictation.isRecording
|
||||
? "Stop recording"
|
||||
: dictation.isTranscribing
|
||||
? "Transcribing…"
|
||||
: "Dictate message";
|
||||
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
aria-label={tooltipText}
|
||||
aria-pressed={dictation.isRecording}
|
||||
className={cn(
|
||||
dictation.isRecording &&
|
||||
"bg-destructive text-destructive-foreground hover:bg-destructive/90 hover:text-destructive-foreground active:bg-destructive active:text-destructive-foreground",
|
||||
dictation.isTranscribing && "animate-pulse",
|
||||
)}
|
||||
disabled={disabled || dictation.isStarting}
|
||||
onClick={dictation.toggleRecording}
|
||||
size="icon"
|
||||
type="button"
|
||||
variant={dictation.isRecording ? "default" : "ghost"}
|
||||
>
|
||||
<Mic />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{tooltipText}</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
@@ -54,6 +54,7 @@ import { NonMemberMentionDialog } from "./NonMemberMentionDialog";
|
||||
import { useMentionSendFlow } from "./useMentionSendFlow";
|
||||
import { useComposerContentState } from "./useComposerContentState";
|
||||
import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot";
|
||||
import { DictationButton, useComposerDictation } from "@/features/dictation";
|
||||
|
||||
type MessageComposerProps = {
|
||||
channelId?: string | null;
|
||||
@@ -265,6 +266,15 @@ function MessageComposerImpl({
|
||||
emojiAutocomplete.isEmojiAutocompleteOpen;
|
||||
|
||||
const submitMessageRef = React.useRef<() => void>(() => {});
|
||||
|
||||
const dictation = useComposerDictation({
|
||||
contentRef,
|
||||
disabled,
|
||||
isSending,
|
||||
setComposerContentFromText,
|
||||
submitMessageRef,
|
||||
});
|
||||
|
||||
const composerScrollRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
// Set after `useLinkEditor` exists below; the editor's link-click handler
|
||||
@@ -997,7 +1007,12 @@ function MessageComposerImpl({
|
||||
<MessageComposerToolbar
|
||||
composerDisabled={disabled}
|
||||
editor={richText.editor}
|
||||
extraActions={toolbarExtraActions}
|
||||
extraActions={
|
||||
<>
|
||||
<DictationButton dictation={dictation} disabled={disabled} />
|
||||
{toolbarExtraActions}
|
||||
</>
|
||||
}
|
||||
formattingDisabled={disabled}
|
||||
isEmojiPickerOpen={isEmojiPickerOpen}
|
||||
isFormattingOpen={isFormattingOpen}
|
||||
|
||||
Reference in New Issue
Block a user