mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
feat: add real-time voice dictation to composer
Adds dictation support using OpenAI's Realtime API over WebRTC:
Relay:
- New /transcribe/status and /transcribe/session endpoints
- BUZZ_OPENAI_API_KEY env var gates the feature (hidden when absent)
- Proxies ephemeral client-secret minting from OpenAI
Desktop:
- New features/dictation module with:
- AudioWorklet for 24kHz PCM capture + buffering
- WebRTC peer connection to OpenAI Realtime API
- Real-time transcript merging into composer
- Auto-submit on trigger phrase ('submit')
- Mic button in composer toolbar (red pulse when recording)
- Integrated into MessageComposer via useComposerDictation hook
Signed-off-by: klopez4212 <klopez4212@gmail.com>
This commit is contained in:
@@ -72,6 +72,7 @@ url = { workspace = true }
|
||||
moka = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
metrics-exporter-prometheus = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
|
||||
[features]
|
||||
dev = ["buzz-auth/dev"]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! HTTP API — media, git, NIP-05, and the Nostr HTTP bridge.
|
||||
//! HTTP API — media, git, NIP-05, transcription, and the Nostr HTTP bridge.
|
||||
|
||||
pub mod bridge;
|
||||
pub mod events;
|
||||
@@ -7,6 +7,7 @@ pub mod invites;
|
||||
pub mod media;
|
||||
pub mod nip05;
|
||||
pub mod operator;
|
||||
pub mod transcribe;
|
||||
|
||||
// Re-export imeta helpers used by ingest pipeline.
|
||||
pub use crate::handlers::imeta::{validate_imeta_tags, verify_imeta_blobs};
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting.
|
||||
//!
|
||||
//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client
|
||||
//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a
|
||||
//! WebRTC connection for real-time speech-to-text dictation.
|
||||
|
||||
use axum::{extract::State, http::StatusCode, response::Json};
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
const OPENAI_REALTIME_SESSIONS_URL: &str = "https://api.openai.com/v1/realtime/sessions";
|
||||
const DEFAULT_TRANSCRIPTION_MODEL: &str = "whisper-1";
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct TranscribeStatus {
|
||||
configured: bool,
|
||||
model: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TranscribeSession {
|
||||
client_secret: String,
|
||||
model: String,
|
||||
}
|
||||
|
||||
/// `GET /transcribe/status` — check if transcription is configured.
|
||||
pub async fn transcribe_status(State(state): State<Arc<AppState>>) -> Json<TranscribeStatus> {
|
||||
Json(TranscribeStatus {
|
||||
configured: state.config.openai_api_key.is_some(),
|
||||
model: transcription_model(),
|
||||
})
|
||||
}
|
||||
|
||||
/// `POST /transcribe/session` — create an ephemeral OpenAI Realtime session.
|
||||
///
|
||||
/// Returns a short-lived client secret that the frontend uses to establish
|
||||
/// a WebRTC connection directly with OpenAI for real-time transcription.
|
||||
pub async fn create_transcribe_session(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<TranscribeSession>, (StatusCode, Json<serde_json::Value>)> {
|
||||
let api_key = state.config.openai_api_key.as_deref().ok_or_else(|| {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(serde_json::json!({
|
||||
"error": "transcription_not_configured",
|
||||
"message": "Transcription is not configured on this relay"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let model = transcription_model();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let response = client
|
||||
.post(OPENAI_REALTIME_SESSIONS_URL)
|
||||
.header("Authorization", format!("Bearer {api_key}"))
|
||||
.header("Content-Type", "application/json")
|
||||
.json(&serde_json::json!({
|
||||
"model": "gpt-4o-mini-realtime-preview",
|
||||
"modalities": ["text"],
|
||||
"input_audio_transcription": {
|
||||
"model": model,
|
||||
},
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
}
|
||||
}))
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("OpenAI realtime session request failed: {e}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Failed to create transcription session"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
tracing::error!("OpenAI realtime session error ({status}): {body}");
|
||||
return Err((
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "OpenAI rejected the transcription session request"
|
||||
})),
|
||||
));
|
||||
}
|
||||
|
||||
let body: serde_json::Value = response.json().await.map_err(|e| {
|
||||
tracing::error!("OpenAI realtime session response parse error: {e}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Invalid response from transcription service"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
let client_secret = extract_client_secret(&body).ok_or_else(|| {
|
||||
tracing::error!("OpenAI realtime session response missing client_secret: {body}");
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "upstream_error",
|
||||
"message": "Transcription service returned unexpected response"
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(Json(TranscribeSession {
|
||||
client_secret,
|
||||
model,
|
||||
}))
|
||||
}
|
||||
|
||||
fn transcription_model() -> String {
|
||||
std::env::var("BUZZ_TRANSCRIPTION_MODEL")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string())
|
||||
}
|
||||
|
||||
fn extract_client_secret(value: &serde_json::Value) -> Option<String> {
|
||||
// Shape 1: { "client_secret": { "value": "..." } }
|
||||
if let Some(cs) = value.get("client_secret") {
|
||||
if let Some(v) = cs.get("value").and_then(|v| v.as_str()) {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
// Shape 2: { "client_secret": "..." }
|
||||
if let Some(v) = cs.as_str() {
|
||||
return Some(v.to_string());
|
||||
}
|
||||
}
|
||||
// Shape 3: { "value": "..." }
|
||||
value
|
||||
.get("value")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::extract_client_secret;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parses_nested_client_secret() {
|
||||
let body = json!({ "client_secret": { "value": "sec_abc123", "expires_at": 9999 } });
|
||||
assert_eq!(extract_client_secret(&body), Some("sec_abc123".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_direct_string_client_secret() {
|
||||
let body = json!({ "client_secret": "sec_direct" });
|
||||
assert_eq!(extract_client_secret(&body), Some("sec_direct".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_top_level_value() {
|
||||
let body = json!({ "value": "sec_toplevel" });
|
||||
assert_eq!(
|
||||
extract_client_secret(&body),
|
||||
Some("sec_toplevel".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_none_for_missing_secret() {
|
||||
let body = json!({ "id": "sess_123", "model": "gpt-4o" });
|
||||
assert_eq!(extract_client_secret(&body), None);
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,11 @@ pub struct Config {
|
||||
/// Used to authenticate internal policy endpoint requests.
|
||||
pub git_hook_hmac_secret: String,
|
||||
|
||||
/// Optional OpenAI API key for real-time transcription (dictation).
|
||||
/// When absent, the `/transcribe/session` endpoint returns 503 and the
|
||||
/// desktop mic button stays hidden.
|
||||
pub openai_api_key: Option<String>,
|
||||
|
||||
/// Optional path to the web UI `dist/` directory.
|
||||
/// When set, the relay serves the SPA from this directory for browser requests.
|
||||
/// When unset, no static file serving happens (relay behaves as before).
|
||||
@@ -476,6 +481,11 @@ impl Config {
|
||||
let secret: [u8; 32] = rand::random();
|
||||
hex::encode(secret)
|
||||
});
|
||||
let openai_api_key = std::env::var("BUZZ_OPENAI_API_KEY")
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
// Web UI static file serving
|
||||
let web_dir = std::env::var("BUZZ_WEB_DIR")
|
||||
.ok()
|
||||
@@ -538,6 +548,7 @@ impl Config {
|
||||
git_max_repos_per_pubkey,
|
||||
git_max_concurrent_ops,
|
||||
git_hook_hmac_secret,
|
||||
openai_api_key,
|
||||
web_dir,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -80,6 +80,15 @@ pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
)
|
||||
// Webhook trigger (secret-authenticated, no NIP-98)
|
||||
.route("/hooks/{id}", post(api::bridge::workflow_webhook))
|
||||
// Transcription (dictation) — proxies OpenAI Realtime client-secret minting
|
||||
.route(
|
||||
"/transcribe/status",
|
||||
get(api::transcribe::transcribe_status),
|
||||
)
|
||||
.route(
|
||||
"/transcribe/session",
|
||||
post(api::transcribe::create_transcribe_session),
|
||||
)
|
||||
// Huddle audio WebSocket route
|
||||
.route(
|
||||
"/huddle/{channel_id}/audio",
|
||||
@@ -112,6 +121,7 @@ pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
|| path.starts_with("/internal/")
|
||||
|| path.starts_with("/.well-known/")
|
||||
|| path.starts_with("/huddle/")
|
||||
|| path.starts_with("/transcribe/")
|
||||
|| path == "/health"
|
||||
|| path == "/_liveness"
|
||||
|| path == "/_readiness"
|
||||
|
||||
Reference in New Issue
Block a user