feat: add real-time voice dictation to composer

Adds dictation support using OpenAI's Realtime API over WebRTC:

Relay:
- New /transcribe/status and /transcribe/session endpoints
- BUZZ_OPENAI_API_KEY env var gates the feature (hidden when absent)
- Proxies ephemeral client-secret minting from OpenAI

Desktop:
- New features/dictation module with:
  - AudioWorklet for 24kHz PCM capture + buffering
  - WebRTC peer connection to OpenAI Realtime API
  - Real-time transcript merging into composer
  - Auto-submit on trigger phrase ('submit')
  - Mic button in composer toolbar (red pulse when recording)
- Integrated into MessageComposer via useComposerDictation hook

Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
klopez4212
2026-07-11 16:18:04 +01:00
parent 51234bd89f
commit f7ee8ff980
16 changed files with 1003 additions and 2 deletions
+1
View File
@@ -72,6 +72,7 @@ url = { workspace = true }
moka = { workspace = true }
metrics = { workspace = true }
metrics-exporter-prometheus = { workspace = true }
reqwest = { workspace = true }
[features]
dev = ["buzz-auth/dev"]
+2 -1
View File
@@ -1,4 +1,4 @@
//! HTTP API — media, git, NIP-05, and the Nostr HTTP bridge.
//! HTTP API — media, git, NIP-05, transcription, and the Nostr HTTP bridge.
pub mod bridge;
pub mod events;
@@ -7,6 +7,7 @@ pub mod invites;
pub mod media;
pub mod nip05;
pub mod operator;
pub mod transcribe;
// Re-export imeta helpers used by ingest pipeline.
pub use crate::handlers::imeta::{validate_imeta_tags, verify_imeta_blobs};
+182
View File
@@ -0,0 +1,182 @@
//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting.
//!
//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client
//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a
//! WebRTC connection for real-time speech-to-text dictation.
use axum::{extract::State, http::StatusCode, response::Json};
use serde::Serialize;
use std::sync::Arc;
use crate::state::AppState;
const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions";
const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1";
#[derive(Serialize)]
pub struct TranscribeStatus {
configured: bool,
model: String,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TranscribeSession {
client_secret: String,
model: String,
}
/// `GET /transcribe/status` — check if transcription is configured.
pub async fn transcribe_status(State(state): State<Arc<AppState>>) -> Json<TranscribeStatus> {
Json(TranscribeStatus {
configured: state.config.openai_api_key.is_some(),
model: transcription_model(),
})
}
/// `POST /transcribe/session` — create an ephemeral OpenAI Realtime session.
///
/// Returns a short-lived client secret that the frontend uses to establish
/// a WebRTC connection directly with OpenAI for real-time transcription.
pub async fn create_transcribe_session(
State(state): State<Arc<AppState>>,
) -> Result<Json<TranscribeSession>, (StatusCode, Json<serde_json::Value>)> {
let api_key = state.config.openai_api_key.as_deref().ok_or_else(|| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(serde_json::json!({
"error": "transcription_not_configured",
"message": "Transcription is not configured on this relay"
})),
)
})?;
let model = transcription_model();
let client = reqwest::Client::new();
let response = client
.post(OPENAI_REALTIME_SESSIONS_URL)
.header("Authorization", format!("Bearer {api_key}"))
.header("Content-Type", "application/json")
.json(&serde_json::json!({
"model": "gpt-4o-mini-realtime-preview",
"modalities": ["text"],
"input_audio_transcription": {
"model": model,
},
"turn_detection": {
"type": "server_vad",
}
}))
.timeout(std::time::Duration::from_secs(10))
.send()
.await
.map_err(|e| {
tracing::error!("OpenAI realtime session request failed: {e}");
(
StatusCode::BAD_GATEWAY,
Json(serde_json::json!({
"error": "upstream_error",
"message": "Failed to create transcription session"
})),
)
})?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
tracing::error!("OpenAI realtime session error ({status}): {body}");
return Err((
StatusCode::BAD_GATEWAY,
Json(serde_json::json!({
"error": "upstream_error",
"message": "OpenAI rejected the transcription session request"
})),
));
}
let body: serde_json::Value = response.json().await.map_err(|e| {
tracing::error!("OpenAI realtime session response parse error: {e}");
(
StatusCode::BAD_GATEWAY,
Json(serde_json::json!({
"error": "upstream_error",
"message": "Invalid response from transcription service"
})),
)
})?;
let client_secret = extract_client_secret(&body).ok_or_else(|| {
tracing::error!("OpenAI realtime session response missing client_secret: {body}");
(
StatusCode::BAD_GATEWAY,
Json(serde_json::json!({
"error": "upstream_error",
"message": "Transcription service returned unexpected response"
})),
)
})?;
Ok(Json(TranscribeSession {
client_secret,
model,
}))
}
fn transcription_model() -> String {
std::env::var("BUZZ_TRANSCRIPTION_MODEL")
.ok()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string())
}
fn extract_client_secret(value: &serde_json::Value) -> Option<String> {
// Shape 1: { "client_secret": { "value": "..." } }
if let Some(cs) = value.get("client_secret") {
if let Some(v) = cs.get("value").and_then(|v| v.as_str()) {
return Some(v.to_string());
}
// Shape 2: { "client_secret": "..." }
if let Some(v) = cs.as_str() {
return Some(v.to_string());
}
}
// Shape 3: { "value": "..." }
value
.get("value")
.and_then(|v| v.as_str())
.map(String::from)
}
#[cfg(test)]
mod tests {
use super::extract_client_secret;
use serde_json::json;
#[test]
fn parses_nested_client_secret() {
let body = json!({ "client_secret": { "value": "sec_abc123", "expires_at": 9999 } });
assert_eq!(extract_client_secret(&body), Some("sec_abc123".to_string()));
}
#[test]
fn parses_direct_string_client_secret() {
let body = json!({ "client_secret": "sec_direct" });
assert_eq!(extract_client_secret(&body), Some("sec_direct".to_string()));
}
#[test]
fn parses_top_level_value() {
let body = json!({ "value": "sec_toplevel" });
assert_eq!(
extract_client_secret(&body),
Some("sec_toplevel".to_string())
);
}
#[test]
fn returns_none_for_missing_secret() {
let body = json!({ "id": "sess_123", "model": "gpt-4o" });
assert_eq!(extract_client_secret(&body), None);
}
}
+11
View File
@@ -173,6 +173,11 @@ pub struct Config {
/// Used to authenticate internal policy endpoint requests.
pub git_hook_hmac_secret: String,
/// Optional OpenAI API key for real-time transcription (dictation).
/// When absent, the `/transcribe/session` endpoint returns 503 and the
/// desktop mic button stays hidden.
pub openai_api_key: Option<String>,
/// Optional path to the web UI `dist/` directory.
/// When set, the relay serves the SPA from this directory for browser requests.
/// When unset, no static file serving happens (relay behaves as before).
@@ -476,6 +481,11 @@ impl Config {
let secret: [u8; 32] = rand::random();
hex::encode(secret)
});
let openai_api_key = std::env::var("BUZZ_OPENAI_API_KEY")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty());
// Web UI static file serving
let web_dir = std::env::var("BUZZ_WEB_DIR")
.ok()
@@ -538,6 +548,7 @@ impl Config {
git_max_repos_per_pubkey,
git_max_concurrent_ops,
git_hook_hmac_secret,
openai_api_key,
web_dir,
})
}
+10
View File
@@ -80,6 +80,15 @@ pub fn build_router(state: Arc<AppState>) -> Router {
)
// Webhook trigger (secret-authenticated, no NIP-98)
.route("/hooks/{id}", post(api::bridge::workflow_webhook))
// Transcription (dictation) — proxies OpenAI Realtime client-secret minting
.route(
"/transcribe/status",
get(api::transcribe::transcribe_status),
)
.route(
"/transcribe/session",
post(api::transcribe::create_transcribe_session),
)
// Huddle audio WebSocket route
.route(
"/huddle/{channel_id}/audio",
@@ -112,6 +121,7 @@ pub fn build_router(state: Arc<AppState>) -> Router {
|| path.starts_with("/internal/")
|| path.starts_with("/.well-known/")
|| path.starts_with("/huddle/")
|| path.starts_with("/transcribe/")
|| path == "/health"
|| path == "/_liveness"
|| path == "/_readiness"