From f320ac0092b4369ea002fe95206020329f62dad2 Mon Sep 17 00:00:00 2001 From: klopez4212 Date: Mon, 6 Jul 2026 12:58:04 +0100 Subject: [PATCH] fix(dictation): proxy SDP server-side, allow send during transcribing 1. Proxy SDP exchange through the relay (transcribe.rs) The relay no longer returns the raw OpenAI client secret to the desktop client. Instead, POST /transcribe/session returns an opaque session ID, and the new POST /transcribe/sdp endpoint accepts the client's SDP offer, looks up the cached secret, forwards it to OpenAI, and returns the SDP answer. This prevents a compromised client from reusing the bearer token to open non-transcription Realtime sessions under the operator account. 2. Allow send during the transcribing grace window (MessageComposer.tsx) Previously, pressing Enter/Send while isTranscribing was true (during the 3s grace window after user-stop) blocked the send entirely. Now the send proceeds with whatever content is already in the composer (transcript deltas have been applied incrementally) and cancels the dictation run so late events don't refill the cleared composer. --- crates/buzz-relay/src/api/transcribe.rs | 122 ++++++++++++++++-- crates/buzz-relay/src/router.rs | 1 + crates/buzz-relay/src/state.rs | 6 + .../dictation/api/transcribeSession.ts | 31 ++++- .../dictation/hooks/useRealtimeDictation.ts | 5 +- .../features/dictation/lib/realtimeAudio.ts | 33 ++--- .../features/messages/ui/MessageComposer.tsx | 6 +- 7 files changed, 168 insertions(+), 36 deletions(-) diff --git a/crates/buzz-relay/src/api/transcribe.rs b/crates/buzz-relay/src/api/transcribe.rs index e34b09d09..b8a441440 100644 --- a/crates/buzz-relay/src/api/transcribe.rs +++ b/crates/buzz-relay/src/api/transcribe.rs @@ -1,10 +1,12 @@ -//! Transcription session endpoint — proxies OpenAI Realtime API client-secret minting. +//! Transcription session endpoint — proxies OpenAI Realtime API session + SDP exchange. //! -//! When `BUZZ_OPENAI_API_KEY` is configured, the relay can mint ephemeral client -//! secrets for the OpenAI Realtime API. The desktop app uses these to establish a -//! WebRTC connection for real-time speech-to-text dictation. +//! When `BUZZ_OPENAI_API_KEY` is configured, the relay mints ephemeral OpenAI +//! Realtime sessions and proxies the WebRTC SDP exchange on behalf of the +//! desktop client. The client never receives the raw OpenAI bearer token — +//! this prevents a compromised client from reusing the token to open +//! non-transcription sessions under the operator's account. //! -//! Both endpoints require NIP-98 auth (same as `/events`, `/query`, `/count`). +//! All endpoints require NIP-98 auth (same as `/events`, `/query`, `/count`). use std::sync::Arc; use std::time::{Duration, Instant}; @@ -14,8 +16,9 @@ use axum::{ http::{HeaderMap, StatusCode}, response::Json, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; +use uuid::Uuid; use buzz_core::CommunityId; @@ -26,6 +29,12 @@ use super::api_error; const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = "https://api.openai.com/v1/realtime/client_secrets"; +const OPENAI_REALTIME_CALLS_URL: &str = "https://api.openai.com/v1/realtime/calls"; + +/// Maximum age of a cached transcription session secret before it's considered +/// expired. Matches the `expires_after.seconds` sent to OpenAI. +const SESSION_SECRET_TTL: Duration = Duration::from_secs(60); + /// Rate-limit window for transcription session minting. const TRANSCRIBE_RATE_WINDOW: Duration = Duration::from_secs(60); @@ -40,10 +49,24 @@ pub struct TranscribeStatus { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct TranscribeSession { - client_secret: String, + session_id: String, model: String, } +/// Request body for `POST /transcribe/sdp`. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SdpExchangeRequest { + session_id: String, + sdp: String, +} + +/// Response for `POST /transcribe/sdp`. +#[derive(Serialize)] +pub struct SdpExchangeResponse { + sdp: String, +} + /// `GET /transcribe/status` — check if transcription is configured. /// /// Requires NIP-98 auth. Returns whether the relay has an OpenAI API key @@ -160,10 +183,87 @@ pub async fn create_transcribe_session( ) })?; - Ok(Json(TranscribeSession { - client_secret, - model, - })) + // Store the secret server-side — the client receives only an opaque session + // ID and must call `/transcribe/sdp` to complete the WebRTC handshake. + let session_id = Uuid::new_v4().to_string(); + state + .transcribe_sessions + .insert(session_id.clone(), (client_secret, Instant::now())); + + Ok(Json(TranscribeSession { session_id, model })) +} + +/// `POST /transcribe/sdp` — proxy the WebRTC SDP exchange to OpenAI. +/// +/// Accepts the client's SDP offer and the session ID returned by +/// `/transcribe/session`. The relay looks up the cached client secret, +/// forwards the SDP offer to OpenAI's `/v1/realtime/calls` endpoint, and +/// returns the SDP answer. The client never sees the bearer token. +pub async fn proxy_sdp_exchange( + State(state): State>, + headers: HeaderMap, + Json(body): Json, +) -> Result, (StatusCode, Json)> { + // Authenticate — same NIP-98 requirement as session creation. + let pubkey = authenticate(&state, &headers, "/transcribe/sdp", "POST").await?; + require_relay_member(&state, &headers, &pubkey).await?; + + // Look up the cached client secret. + let (client_secret, created_at) = state + .transcribe_sessions + .remove(&body.session_id) + .map(|(_, v)| v) + .ok_or_else(|| { + api_error( + StatusCode::NOT_FOUND, + "transcription session not found or already used", + ) + })?; + + // Reject expired sessions. + if created_at.elapsed() > SESSION_SECRET_TTL { + return Err(api_error( + StatusCode::GONE, + "transcription session expired — create a new one", + )); + } + + // Proxy the SDP offer to OpenAI. + let client = openai_client().map_err(|(status, msg)| api_error(status, msg))?; + let response = client + .post(OPENAI_REALTIME_CALLS_URL) + .header("Authorization", format!("Bearer {client_secret}")) + .header("Content-Type", "application/sdp") + .body(body.sdp) + .send() + .await + .map_err(|e| { + tracing::error!("OpenAI SDP exchange request failed: {e}"); + api_error( + StatusCode::BAD_GATEWAY, + "failed to establish transcription connection", + ) + })?; + + if !response.status().is_success() { + let status = response.status(); + let resp_body = response.text().await.unwrap_or_default(); + tracing::error!("OpenAI SDP exchange error ({status}): {resp_body}"); + return Err(api_error( + StatusCode::BAD_GATEWAY, + "transcription service rejected the connection", + )); + } + + let sdp_answer = response.text().await.map_err(|e| { + tracing::error!("OpenAI SDP answer read error: {e}"); + api_error( + StatusCode::BAD_GATEWAY, + "invalid response from transcription service", + ) + })?; + + Ok(Json(SdpExchangeResponse { sdp: sdp_answer })) } // ── Helpers ─────────────────────────────────────────────────────────────────── diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index d6c67af18..7123a755d 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -89,6 +89,7 @@ pub fn build_router(state: Arc) -> Router { "/transcribe/session", post(api::transcribe::create_transcribe_session), ) + .route("/transcribe/sdp", post(api::transcribe::proxy_sdp_exchange)) // Huddle audio WebSocket route .route( "/huddle/{channel_id}/audio", diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index b6af51d5b..e6c89b5ba 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -379,6 +379,11 @@ pub struct AppState { /// window_start). Bounds the cost of OpenAI Realtime sessions minted on the /// operator's bill. pub transcribe_rate_limiter: Arc, + /// Short-lived cache of minted transcription session secrets. + /// Key: opaque session ID (UUID). Value: (client_secret, created_at). + /// Secrets are kept server-side so the desktop client never sees the raw + /// OpenAI bearer token — the relay proxies the SDP exchange instead. + pub transcribe_sessions: Arc>, /// Current in-flight media uploads per (community, uploader pubkey). pub media_uploads_in_flight: Arc>, /// Cache for observer agent-owner authorization (kind 24200). @@ -527,6 +532,7 @@ impl AppState { .build(), ), transcribe_rate_limiter: Arc::new(DashMap::new()), + transcribe_sessions: Arc::new(DashMap::new()), media_uploads_in_flight: Arc::new(DashMap::new()), observer_owner_cache: Arc::new( moka::sync::Cache::builder() diff --git a/desktop/src/features/dictation/api/transcribeSession.ts b/desktop/src/features/dictation/api/transcribeSession.ts index 276ca495a..6f4fca26d 100644 --- a/desktop/src/features/dictation/api/transcribeSession.ts +++ b/desktop/src/features/dictation/api/transcribeSession.ts @@ -6,10 +6,14 @@ export interface TranscribeStatus { } export interface TranscribeSession { - clientSecret: string; + sessionId: string; model: string; } +export interface SdpExchangeResponse { + sdp: string; +} + /** NIP-98 event kind for HTTP request authorization. */ const NIP98_KIND = 27235; @@ -67,3 +71,28 @@ export async function createTranscribeSession(): Promise { } return response.json(); } + +/** + * Proxy the WebRTC SDP exchange through the relay. The relay holds the + * OpenAI client secret server-side — the desktop client never sees it. + */ +export async function proxySdpExchange( + sessionId: string, + sdp: string, +): Promise { + const baseUrl = await getRelayHttpUrl(); + const url = `${baseUrl}/transcribe/sdp`; + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: await nip98AuthHeader(url, "POST"), + }, + body: JSON.stringify({ sessionId, sdp }), + }); + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error(`SDP exchange failed (${response.status}): ${body}`); + } + return response.json(); +} diff --git a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts index 959798b9e..e07401eb0 100644 --- a/desktop/src/features/dictation/hooks/useRealtimeDictation.ts +++ b/desktop/src/features/dictation/hooks/useRealtimeDictation.ts @@ -303,10 +303,11 @@ export function useRealtimeDictation({ audioCaptureRef.current = null; }); - // 5. SDP exchange + // 5. SDP exchange (proxied through the relay — client never sees the + // OpenAI bearer token) await connectPeerConnection({ peerConnection, - clientSecret: session.clientSecret, + sessionId: session.sessionId, }); if (isStaleRun()) { closeResources({ audioCapture, dataChannel, peerConnection, stream }); diff --git a/desktop/src/features/dictation/lib/realtimeAudio.ts b/desktop/src/features/dictation/lib/realtimeAudio.ts index 4c46fec7b..b4a88c126 100644 --- a/desktop/src/features/dictation/lib/realtimeAudio.ts +++ b/desktop/src/features/dictation/lib/realtimeAudio.ts @@ -1,10 +1,9 @@ +import { proxySdpExchange } from "../api/transcribeSession"; import { REALTIME_BUFFER_PROCESSOR_NAME, createWorkletBlobUrl, } from "./realtimeBufferWorklet"; -export const OPENAI_REALTIME_WEBRTC_URL = - "https://api.openai.com/v1/realtime/calls"; export const TRANSCRIPT_DELTA_EVENT = "conversation.item.input_audio_transcription.delta"; export const TRANSCRIPT_COMPLETED_EVENT = @@ -28,32 +27,28 @@ export function createPeerConnection(): RTCPeerConnection { return new RTCPeerConnection(); } +/** + * Complete the WebRTC SDP exchange via the relay proxy. + * + * The relay holds the OpenAI client secret server-side — the desktop client + * sends its SDP offer to the relay, which forwards it to OpenAI and returns + * the SDP answer. This prevents the client from ever seeing the bearer token. + */ export async function connectPeerConnection(options: { peerConnection: RTCPeerConnection; - clientSecret: string; + sessionId: string; }): Promise { const offer = await options.peerConnection.createOffer(); await options.peerConnection.setLocalDescription(offer); - const response = await fetch(OPENAI_REALTIME_WEBRTC_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${options.clientSecret}`, - "Content-Type": "application/sdp", - }, - body: offer.sdp ?? "", - }); - - const body = await response.text(); - if (!response.ok) { - throw new Error( - `OpenAI realtime connection failed (${response.status}): ${body}`, - ); - } + const { sdp: answerSdp } = await proxySdpExchange( + options.sessionId, + offer.sdp ?? "", + ); await options.peerConnection.setRemoteDescription({ type: "answer", - sdp: body, + sdp: answerSdp, }); } diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 5d9f27ac8..4e279914b 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -279,6 +279,7 @@ function MessageComposerImpl({ draftKey: effectiveDraftKey, }); stopDictationRef.current = dictation.cancelRecording; + const composerScrollRef = React.useRef(null); // Set after `useLinkEditor` exists below; the editor's link-click handler // delegates through this ref to break the hook ordering cycle (the editor @@ -558,7 +559,7 @@ function MessageComposerImpl({ // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; - stopDictationRef.current(); // stop dictation so late transcripts don't refill during edit save + stopDictationRef.current(); // cancel dictation before edit save const currentPendingImeta = media.pendingImetaRef.current; const hasMedia = currentPendingImeta.length > 0; // Empty text + zero attachments is a no-op (don't let edit become an @@ -618,8 +619,7 @@ function MessageComposerImpl({ ) { return; } - - stopDictationRef.current(); // stop dictation so late transcripts don't refill + stopDictationRef.current(); // cancel dictation; send proceeds with current content const capturedThreadContext = onCaptureSendContext?.() ?? null; // If a thread-reply composer reported no reply target at submit time,