From 12dca80e1a8499a7ca3bba983de2f32d6845341d Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 16 Apr 2026 13:36:46 -0600 Subject: [PATCH] feat(desktop): improve huddle bar UX + audio device selection (#340) Co-authored-by: Claude Opus 4.6 (1M context) --- desktop/src-tauri/src/app_state.rs | 4 + desktop/src-tauri/src/huddle/audio_output.rs | 83 +++++ desktop/src-tauri/src/huddle/mod.rs | 1 + desktop/src-tauri/src/huddle/pipeline.rs | 7 +- desktop/src-tauri/src/huddle/relay_api.rs | 10 +- desktop/src-tauri/src/huddle/tts.rs | 11 +- desktop/src-tauri/src/lib.rs | 6 + desktop/src/app/AppShell.tsx | 310 +++++++-------- desktop/src/features/huddle/HuddleContext.tsx | 199 +++++----- .../huddle/components/AddAgentDialog.tsx | 115 +++--- .../features/huddle/components/HuddleBar.tsx | 352 +++++++++--------- .../huddle/components/HuddleIndicator.tsx | 24 ++ .../huddle/components/MicControls.tsx | 225 +++++++++++ .../huddle/components/ParticipantList.tsx | 34 +- .../src/features/huddle/lib/audioWorklet.ts | 14 +- .../features/huddle/lib/useAudioDevices.ts | 56 +++ .../features/huddle/lib/useTtsSubscription.ts | 111 ++++++ desktop/src/shared/ui/sidebar.tsx | 6 +- 18 files changed, 1059 insertions(+), 509 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/audio_output.rs create mode 100644 desktop/src/features/huddle/components/MicControls.tsx create mode 100644 desktop/src/features/huddle/lib/useAudioDevices.ts create mode 100644 desktop/src/features/huddle/lib/useTtsSubscription.ts diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index a1d8ca6dd..d2b05a126 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -20,6 +20,9 @@ pub struct AppState { /// /// Set once during `setup()` in `lib.rs`; never cleared. pub app_handle: Mutex>, + /// Selected audio output device name. `None` = system default. + /// Used by `connect_audio_relay` and TTS pipeline when opening sinks. + pub audio_output_device: Mutex>, } pub fn build_app_state() -> AppState { @@ -65,6 +68,7 @@ pub fn build_app_state() -> AppState { managed_agent_processes: Mutex::new(HashMap::new()), huddle_state: Mutex::new(HuddleState::default()), app_handle: Mutex::new(None), + audio_output_device: Mutex::new(None), } } diff --git a/desktop/src-tauri/src/huddle/audio_output.rs b/desktop/src-tauri/src/huddle/audio_output.rs new file mode 100644 index 000000000..7e5519ce2 --- /dev/null +++ b/desktop/src-tauri/src/huddle/audio_output.rs @@ -0,0 +1,83 @@ +//! Audio output device enumeration, selection, and sink creation. + +use tauri::State; + +use crate::app_state::AppState; + +/// List available audio output devices. Returns (name, is_default) pairs. +#[tauri::command] +pub fn list_audio_output_devices() -> Result, String> { + use rodio::cpal::traits::HostTrait; + use rodio::DeviceTrait; + + let host = rodio::cpal::default_host(); + let default_name = host.default_output_device().and_then(|d| d.name().ok()); + let devices = host + .output_devices() + .map_err(|e| format!("enumerate output devices: {e}"))?; + + let mut result = Vec::new(); + for device in devices { + if let Ok(name) = device.name() { + let is_default = default_name.as_deref() == Some(&name); + result.push(AudioOutputDevice { name, is_default }); + } + } + Ok(result) +} + +/// Set the preferred audio output device by name. Empty string = system default. +/// Takes effect on the next huddle start/join (does not change a live stream). +#[tauri::command] +pub fn set_audio_output_device(name: String, state: State<'_, AppState>) -> Result<(), String> { + let mut guard = state + .audio_output_device + .lock() + .map_err(|e| e.to_string())?; + *guard = if name.is_empty() { None } else { Some(name) }; + Ok(()) +} + +/// Get the currently selected audio output device name (empty = system default). +#[tauri::command] +pub fn get_audio_output_device(state: State<'_, AppState>) -> Result { + let guard = state + .audio_output_device + .lock() + .map_err(|e| e.to_string())?; + Ok(guard.clone().unwrap_or_default()) +} + +#[derive(Debug, serde::Serialize)] +pub struct AudioOutputDevice { + pub name: String, + pub is_default: bool, +} + +/// Open a rodio sink for a named output device, falling back to default. +pub(crate) fn open_output_sink_by_name( + preferred: Option<&str>, +) -> Result { + use rodio::cpal::traits::HostTrait; + use rodio::DeviceTrait; + + if let Some(name) = preferred { + let host = rodio::cpal::default_host(); + if let Ok(devices) = host.output_devices() { + for device in devices { + if device.name().ok().as_deref() == Some(name) { + if let Ok(sink) = rodio::DeviceSinkBuilder::from_device(device) { + return sink + .open_stream() + .map_err(|e| format!("audio output ({name}): {e}")); + } + } + } + } + eprintln!( + "sprout-desktop: preferred output device {name:?} not found, falling back to default" + ); + } + + rodio::DeviceSinkBuilder::open_default_sink().map_err(|e| format!("audio output: {e}")) +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 0e56c591b..892dd025b 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -24,6 +24,7 @@ //! and drops them outside the lock (thread joins can block ~200ms). pub mod agents; +pub mod audio_output; pub mod kokoro; pub mod models; pub mod pipeline; diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index 32e3102ab..586900240 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -193,7 +193,12 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result Arc::new(p), Err(e) => { let hs = state.huddle()?; diff --git a/desktop/src-tauri/src/huddle/relay_api.rs b/desktop/src-tauri/src/huddle/relay_api.rs index 522c352b8..241729eb8 100644 --- a/desktop/src-tauri/src/huddle/relay_api.rs +++ b/desktop/src-tauri/src/huddle/relay_api.rs @@ -162,6 +162,11 @@ pub(crate) async fn connect_audio_relay( let cancel = CancellationToken::new(); let cancel_clone = cancel.clone(); let (pcm_tx, pcm_rx) = tokio::sync::mpsc::channel::>(50); + let output_device_name = state + .audio_output_device + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); tokio::spawn(async move { if let Err(e) = audio_relay_pipeline( @@ -173,6 +178,7 @@ pub(crate) async fn connect_audio_relay( initial_peers, tts_cancel, tts_active, + output_device_name, ) .await { @@ -206,6 +212,7 @@ async fn audio_relay_pipeline( initial_peers: Vec<(u8, String)>, tts_cancel: Arc, tts_active: Arc, + output_device_name: Option, ) -> Result<(), String> { let mut encoder = opus::Encoder::new(48000, opus::Channels::Mono, opus::Application::Voip) .map_err(|e| format!("opus encoder: {e}"))?; @@ -216,8 +223,7 @@ async fn audio_relay_pipeline( .set_dtx(true) .map_err(|e| format!("opus dtx: {e}"))?; - let sink_handle = - rodio::DeviceSinkBuilder::open_default_sink().map_err(|e| format!("audio output: {e}"))?; + let sink_handle = super::audio_output::open_output_sink_by_name(output_device_name.as_deref())?; let player = rodio::Player::connect_new(&sink_handle.mixer()); let decoders: std::collections::HashMap = std::collections::HashMap::new(); diff --git a/desktop/src-tauri/src/huddle/tts.rs b/desktop/src-tauri/src/huddle/tts.rs index bdd459a41..bb696c1eb 100644 --- a/desktop/src-tauri/src/huddle/tts.rs +++ b/desktop/src-tauri/src/huddle/tts.rs @@ -110,9 +110,10 @@ impl TtsPipeline { model_dir: PathBuf, tts_active: Arc, cancel: Arc, + output_device: Option, ) -> Result { use super::kokoro::DEFAULT_VOICE; - Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE) + Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE, output_device) } /// Spawn the TTS pipeline thread with a specific voice name (e.g. `"af_heart"`, `"am_michael"`). @@ -121,6 +122,7 @@ impl TtsPipeline { tts_active: Arc, cancel: Arc, voice: &str, + output_device: Option, ) -> Result { let (text_tx, text_rx) = mpsc::sync_channel::(TEXT_QUEUE_DEPTH); let shutdown = Arc::new(AtomicBool::new(false)); @@ -142,6 +144,7 @@ impl TtsPipeline { tts_active_worker, shutdown_worker, cancel_worker, + output_device, ) }) .map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?; @@ -199,6 +202,7 @@ fn tts_worker( tts_active: Arc, shutdown: Arc, cancel: Arc, + output_device: Option, ) { // ── 1. Initialise Kokoro engine ─────────────────────────────────────────── let model_dir_str = model_dir.to_string_lossy().to_string(); @@ -229,9 +233,10 @@ fn tts_worker( }; // ── 3. Initialise rodio output device ───────────────────────────────────── - use rodio::{DeviceSinkBuilder, Player}; + use rodio::Player; - let sink_handle = match DeviceSinkBuilder::open_default_sink() { + let sink_handle = match super::audio_output::open_output_sink_by_name(output_device.as_deref()) + { Ok(h) => h, Err(e) => { eprintln!("sprout-desktop: TTS audio output failed: {e}. TTS disabled."); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 57e2ccfce..6c4f0ab33 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -10,6 +10,9 @@ mod util; use app_state::{build_app_state, resolve_persisted_identity, AppState}; use commands::*; +use huddle::audio_output::{ + get_audio_output_device, list_audio_output_devices, set_audio_output_device, +}; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models, end_huddle, get_huddle_agent_pubkeys, get_huddle_state, get_model_status, get_voice_input_mode, @@ -641,6 +644,9 @@ pub fn run() { get_huddle_agent_pubkeys, set_voice_input_mode, get_voice_input_mode, + list_audio_output_devices, + set_audio_output_device, + get_audio_output_device, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index 1683e5840..fe1cb2e10 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -387,172 +387,174 @@ export function AppShell() { }} > - -
- - - -
- { - const createdChannel = await createChannelMutation.mutateAsync({ - name, +
+ +
+ + + +
+ { + const createdChannel = + await createChannelMutation.mutateAsync({ + name, + description, + channelType: "stream", + visibility, + ttlSeconds, + }); - await goChannel(createdChannel.id); - }} - onCreateForum={async ({ - description, - name, - visibility, - ttlSeconds, - }) => { - const createdForum = await createForumMutation.mutateAsync({ - name, + await goChannel(createdChannel.id); + }} + onCreateForum={async ({ description, - channelType: "forum", + name, visibility, ttlSeconds, - }); + }) => { + const createdForum = await createForumMutation.mutateAsync({ + name, + description, + channelType: "forum", + visibility, + ttlSeconds, + }); - await goChannel(createdForum.id); - }} - onHideDm={handleHideDm} - onOpenBrowseChannels={handleOpenBrowseChannels} - onOpenBrowseForums={handleOpenBrowseForums} - onOpenDm={async ({ pubkeys }) => { - const directMessage = await openDmMutation.mutateAsync({ - pubkeys, - }); - await goChannel(directMessage.id); - }} - onOpenSearch={handleOpenSearch} - onSelectAgents={() => { - void goAgents(); - }} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - onSelectHome={() => { - void goHome(); - }} - onSelectPulse={() => { - void goPulse(); - }} - onSelectSettings={handleOpenSettings} - onSelectWorkflows={() => { - void goWorkflows(); - }} - onSetPresenceStatus={(status) => - presenceSession.setStatus(status) - } - profile={profileQuery.data} - selectedChannelId={selectedChannelId} - selectedView={selectedView} - unreadChannelIds={unreadChannelIds} - /> + await goChannel(createdForum.id); + }} + onHideDm={handleHideDm} + onOpenBrowseChannels={handleOpenBrowseChannels} + onOpenBrowseForums={handleOpenBrowseForums} + onOpenDm={async ({ pubkeys }) => { + const directMessage = await openDmMutation.mutateAsync({ + pubkeys, + }); + await goChannel(directMessage.id); + }} + onOpenSearch={handleOpenSearch} + onSelectAgents={() => { + void goAgents(); + }} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + onSelectHome={() => { + void goHome(); + }} + onSelectPulse={() => { + void goPulse(); + }} + onSelectSettings={handleOpenSettings} + onSelectWorkflows={() => { + void goWorkflows(); + }} + onSetPresenceStatus={(status) => + presenceSession.setStatus(status) + } + profile={profileQuery.data} + selectedChannelId={selectedChannelId} + selectedView={selectedView} + unreadChannelIds={unreadChannelIds} + /> - - - + + + - { - setIsChannelManagementOpen(false); - void goHome({ replace: true }); - }} - onOpenSearchResult={handleOpenSearchResult} - onSearchOpenChange={setIsSearchOpen} - onSelectChannel={(channelId) => { - void goChannel(channelId); - }} - /> + { + setIsChannelManagementOpen(false); + void goHome({ replace: true }); + }} + onOpenSearchResult={handleOpenSearchResult} + onSearchOpenChange={setIsSearchOpen} + onSelectChannel={(channelId) => { + void goChannel(channelId); + }} + /> + {settingsOpen ? ( + + + + ) : null} + - - {settingsOpen ? ( - - - - ) : null} - +
diff --git a/desktop/src/features/huddle/HuddleContext.tsx b/desktop/src/features/huddle/HuddleContext.tsx index e6f7f21b4..9e036cba1 100644 --- a/desktop/src/features/huddle/HuddleContext.tsx +++ b/desktop/src/features/huddle/HuddleContext.tsx @@ -2,8 +2,9 @@ import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; import * as React from "react"; -import { relayClient } from "@/shared/api/relayClient"; import { setupAudioWorklet, type AudioWorkletHandle } from "./lib/audioWorklet"; +import { useAudioDevices } from "./lib/useAudioDevices"; +import { useTtsSubscription } from "./lib/useTtsSubscription"; /** * Huddle lifecycle (React context): @@ -41,6 +42,22 @@ interface HuddleContextValue { setVoiceInputMode: (mode: VoiceInputMode) => Promise; /** Pubkeys of currently speaking participants (from Rust backend) */ activeSpeakers: string[]; + /** Available audio input devices */ + audioDevices: MediaDeviceInfo[]; + /** Currently selected mic device ID (empty string = system default) */ + selectedDeviceId: string; + /** Select a different mic — takes effect on next huddle start/join */ + setSelectedDeviceId: (id: string) => void; + /** Mic input gain 0–1 */ + micGain: number; + /** Adjust mic input gain — applied immediately to the active audio graph */ + setMicGain: (value: number) => void; + /** Available audio output devices */ + outputDevices: { name: string; is_default: boolean }[]; + /** Currently selected output device name (empty = system default) */ + selectedOutputDevice: string; + /** Select a different speaker — takes effect on next huddle start/join */ + setSelectedOutputDevice: (name: string) => void; /** Start a new huddle — calls Rust start_huddle, then connects mic + AudioWorklet */ startHuddle: ( parentChannelId: string, @@ -90,6 +107,59 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { const selfPubkeyRef = React.useRef(null); /** Pubkeys of participants currently speaking (from Rust backend via Tauri event) */ const [activeSpeakers, setActiveSpeakers] = React.useState([]); + const { + audioDevices, + selectedDeviceId, + setSelectedDeviceId, + micGain, + setMicGain, + } = useAudioDevices(workletRef); + /** Audio output devices from Rust backend */ + const [outputDevices, setOutputDevices] = React.useState< + { name: string; is_default: boolean }[] + >([]); + const [selectedOutputDevice, setSelectedOutputDeviceState] = + React.useState(""); + const setSelectedOutputDevice = React.useCallback((name: string) => { + setSelectedOutputDeviceState(name); + invoke("set_audio_output_device", { name }).catch(() => { + /* best-effort */ + }); + }, []); + + // Fetch output devices on mount and when system devices change. + React.useEffect(() => { + function refreshOutputDevices() { + invoke<{ name: string; is_default: boolean }[]>( + "list_audio_output_devices", + ) + .then(setOutputDevices) + .catch(() => { + /* best-effort */ + }); + } + refreshOutputDevices(); + invoke("get_audio_output_device") + .then(setSelectedOutputDeviceState) + .catch(() => { + /* best-effort */ + }); + navigator.mediaDevices.addEventListener( + "devicechange", + refreshOutputDevices, + ); + return () => { + navigator.mediaDevices.removeEventListener( + "devicechange", + refreshOutputDevices, + ); + }; + }, []); + + /** Ref tracking latest micGain — read inside connectAndSetupMedia to + * avoid stale closure capture. */ + const micGainRef = React.useRef(1); + micGainRef.current = micGain; // Bootstrap voice input mode from Rust backend on mount. // Ensures frontend stays in sync after remount/recovery. @@ -296,12 +366,16 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { // Get mic — Rust backend owns the audio WS connection. // Request 48 kHz to match the Opus encoder and worklet buffer size (960 samples = 20ms). + const audioConstraints: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + sampleRate: 48000, + }; + if (selectedDeviceId) { + audioConstraints.deviceId = { exact: selectedDeviceId }; + } const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - sampleRate: 48000, - }, + audio: audioConstraints, }); const audioTrack = stream.getAudioTracks()[0]; @@ -322,6 +396,8 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { audioTrack, initialTransmitting, ); + // Apply current gain level to the new audio graph. + worklet.setGain(micGainRef.current); if (tokenRef.current !== myToken) { worklet.stop(); @@ -343,7 +419,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { throw err; } }, - [], + [selectedDeviceId], ); const startHuddle = React.useCallback( @@ -354,6 +430,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { tokenRef.current += 1; const myToken = tokenRef.current; + setHuddleError(null); setIsStarting(true); try { // Step 1: Call Rust to create ephemeral channel @@ -394,6 +471,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { busyRef.current = true; tokenRef.current += 1; const myToken = tokenRef.current; + setHuddleError(null); setIsStarting(true); try { @@ -430,104 +508,7 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { [cleanupFailedStart, connectAndSetupMedia], ); - // TTS subscription — pipe AGENT messages from ephemeral channel to speak_agent_message. - // Human STT transcripts are also kind:9 in this channel, so we must filter them out - // using an authoritative agent list fetched from the relay membership API. - React.useEffect(() => { - if (!ephemeralChannelId) return; - - let disposed = false; - let cleanup: (() => void) | null = null; - - // ── Agent identity (authoritative, fail-closed) ─────────────────────── - // - // Fetch the ephemeral channel's member list from the relay REST API and - // identify agents by their "bot" role. This is authoritative — it works - // for both creators and joiners, and reflects mid-huddle agent additions. - // - // FAIL-CLOSED: agentsLoaded starts false. Until the fetch succeeds and - // populates agentPubkeys, NO messages are spoken. An empty set after a - // successful fetch means "no agents in the huddle" → still mute. - let agentsLoaded = false; - const agentPubkeys = new Set(); - - async function loadAgentPubkeys() { - try { - const pubkeys = await invoke("get_huddle_agent_pubkeys"); - agentPubkeys.clear(); - for (const pk of pubkeys) agentPubkeys.add(pk); - agentsLoaded = true; - } catch (e) { - // Fail-closed on ALL failures, including refresh after prior success. - // Clear the set and mark as not loaded — TTS goes mute until the - // next successful refresh. Stale membership must never authorize speech. - agentPubkeys.clear(); - agentsLoaded = false; - console.error("[huddle] Failed to load agent pubkeys:", e); - } - } - - // Initial load + periodic refresh (catches mid-huddle agent additions). - void loadAgentPubkeys(); - const agentRefreshId = window.setInterval(() => { - void loadAgentPubkeys(); - }, 10_000); - - // ── Live-only subscription ─────────────────────────────────────────── - // subscribeToChannelLive uses `since: now` — the relay never sends - // historical backlog. Every event delivered is a live message. - // Event-ID dedup handles reconnect replay (same event arriving twice). - const seenEventIds = new Set(); - const seenOrder: string[] = []; - const MAX_SEEN_EVENTS = 5000; - - relayClient - .subscribeToChannelLive(ephemeralChannelId, (event) => { - if (disposed) return; - // Defense-in-depth: subscription already filters to kind:9 only. - if (event.kind !== 9) return; - - // Dedup by event ID (covers reconnect replay). - if (seenEventIds.has(event.id)) return; - seenEventIds.add(event.id); - seenOrder.push(event.id); - if (seenOrder.length > MAX_SEEN_EVENTS) { - const oldest = seenOrder.shift(); - if (oldest !== undefined) seenEventIds.delete(oldest); - } - - // Fail-closed: don't speak until agent list is loaded. - if (!agentsLoaded) return; - // Only speak agent messages — skip human STT transcripts. - if (!agentPubkeys.has(event.pubkey)) return; - if (event.pubkey === selfPubkeyRef.current) return; - if (event.content.trim().length <= 1) return; - // Legacy: skip [System]-prefixed messages from before kind:48106. - if (event.content.startsWith("[System]")) return; - invoke("speak_agent_message", { text: event.content }).catch((err) => { - console.warn( - "[huddle] TTS speak failed (backpressure or pipeline unavailable):", - err, - ); - }); - }) - .then((dispose) => { - if (disposed) { - void dispose(); - return; - } - cleanup = () => void dispose(); - }) - .catch((err) => { - console.error("[huddle] TTS subscription failed:", err); - }); - - return () => { - disposed = true; - cleanup?.(); - window.clearInterval(agentRefreshId); - }; - }, [ephemeralChannelId]); + useTtsSubscription(ephemeralChannelId, selfPubkeyRef); // Pipeline hot-start — check if voice models finished downloading mid-huddle React.useEffect(() => { @@ -620,6 +601,14 @@ export function HuddleProvider({ children }: { children: React.ReactNode }) { voiceInputMode, setVoiceInputMode, activeSpeakers, + audioDevices, + selectedDeviceId, + setSelectedDeviceId, + micGain, + setMicGain, + outputDevices, + selectedOutputDevice, + setSelectedOutputDevice, startHuddle, joinHuddle, leaveHuddle, diff --git a/desktop/src/features/huddle/components/AddAgentDialog.tsx b/desktop/src/features/huddle/components/AddAgentDialog.tsx index 361df552c..7dd3a2075 100644 --- a/desktop/src/features/huddle/components/AddAgentDialog.tsx +++ b/desktop/src/features/huddle/components/AddAgentDialog.tsx @@ -2,9 +2,11 @@ import { invoke } from "@tauri-apps/api/core"; import { Bot } from "lucide-react"; import * as React from "react"; +import { Button } from "@/shared/ui/button"; import { Dialog, DialogContent, + DialogDescription, DialogHeader, DialogTitle, } from "@/shared/ui/dialog"; @@ -85,62 +87,73 @@ export function AddAgentDialog({ }} open > - - + + Add Agent to Huddle + + Select a running agent to join the huddle. + - {error && ( -

- {error} -

- )} +
+ {error && ( +

+ {error} +

+ )} - {warning && ( -
- {warning} - -
- )} + {warning && ( +
+ {warning} + +
+ )} - {loading ? ( -

- Loading agents… -

- ) : runningAgents.length === 0 ? ( -

- {agents.filter((a) => a.status === "running").length > 0 - ? "All running agents are already in this huddle." - : "No running agents found."} -

- ) : ( -
    - {runningAgents.map((agent) => ( -
  • - -
    • - ))} -
- )} + {loading ? ( +

+ Loading agents… +

+ ) : runningAgents.length === 0 ? ( +

+ {agents.filter((a) => a.status === "running").length > 0 + ? "All running agents are already in this huddle." + : "No running agents found."} +

+ ) : ( +
    + {runningAgents.map((agent) => ( +
  • + +
    • + ))} +
+ )} +
+ +
+ +
); diff --git a/desktop/src/features/huddle/components/HuddleBar.tsx b/desktop/src/features/huddle/components/HuddleBar.tsx index 8c6d0dc93..af6fcf0e0 100644 --- a/desktop/src/features/huddle/components/HuddleBar.tsx +++ b/desktop/src/features/huddle/components/HuddleBar.tsx @@ -1,25 +1,16 @@ import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; -import { - Mic, - MicOff, - PhoneOff, - Plus, - Users, - Volume2, - VolumeX, -} from "lucide-react"; +import { Headphones, PhoneOff, Plus } from "lucide-react"; import * as React from "react"; import { cn } from "@/shared/lib/cn"; import { Button } from "@/shared/ui/button"; import { useHuddle } from "../HuddleContext"; import { AddAgentDialog, type AgentAddResult } from "./AddAgentDialog"; +import { MicControls, SpeakerControls } from "./MicControls"; import { ParticipantList } from "./ParticipantList"; -// Shape returned by the `get_huddle_state` Tauri command. -// NOTE: This mirrors the HuddleState struct in the Rust backend (src-tauri/src/huddle/mod.rs). -// If you add/remove fields here, update the Rust struct (and vice versa). +// Mirrors HuddleState in src-tauri/src/huddle/mod.rs. type HuddleState = { phase: | "idle" @@ -46,6 +37,7 @@ export function HuddleBar({ className }: HuddleBarProps) { localAudioTrack, leaveHuddle, endHuddle, + isStarting, micConnected, micLevel, pttActive, @@ -54,13 +46,19 @@ export function HuddleBar({ className }: HuddleBarProps) { activeSpeakers, huddleError, clearHuddleError, + audioDevices, + selectedDeviceId, + setSelectedDeviceId, + micGain, + setMicGain, + outputDevices, + selectedOutputDevice, + setSelectedOutputDevice, } = useHuddle(); const isPttMode = voiceInputMode === "push_to_talk"; const [state, setState] = React.useState(null); const [isMuted, setIsMuted] = React.useState(false); - // Derive TTS enabled from backend state (single source of truth). - // Fall back to true if state hasn't loaded yet. const ttsEnabled = state?.tts_enabled ?? true; const [isLeaving, setIsLeaving] = React.useState(false); const [showAddAgent, setShowAddAgent] = React.useState(false); @@ -69,8 +67,7 @@ export function HuddleBar({ className }: HuddleBarProps) { moonshine: string; kokoro: string; } | null>(null); - - // Huddle state: event-driven primary path + 10s fallback poll. + // Huddle state: event-driven + 10s fallback poll. React.useEffect(() => { let cancelled = false; let unlisten: (() => void) | null = null; @@ -112,15 +109,12 @@ export function HuddleBar({ className }: HuddleBarProps) { }; }, []); - // Poll model download status while huddle is active const huddlePhase = state?.phase; React.useEffect(() => { if (huddlePhase !== "active" && huddlePhase !== "connected") return; let cancelled = false; - // ModelStatus serializes as: "ready" | "not_downloaded" (strings) - // or { downloading: { progress_percent: N } } | { error: "msg" } (objects). const fmt = (s: unknown): string => { if (typeof s === "string") return s === "ready" ? "ready" : "pending"; if (typeof s === "object" && s !== null) { @@ -161,7 +155,6 @@ export function HuddleBar({ className }: HuddleBarProps) { }; }, [huddlePhase]); - // Sync mute state to the audio track React.useEffect(() => { if (localAudioTrack) { localAudioTrack.enabled = !isMuted; @@ -179,8 +172,7 @@ export function HuddleBar({ className }: HuddleBarProps) { if (backendClean) { setState(null); } - // If backend cleanup failed, keep the bar visible so the user can retry. - // leaveHuddle retains rustActiveRef=true for the next attempt. + // If cleanup failed, keep the bar visible so the user can retry. } catch (e) { console.error("Failed to leave huddle:", e); } finally { @@ -200,7 +192,7 @@ export function HuddleBar({ className }: HuddleBarProps) { if (backendClean) { setState(null); } - // If backend cleanup failed, keep the bar visible so the user can retry. + // If cleanup failed, keep the bar visible so the user can retry. } catch (e) { console.error("Failed to end huddle:", e); } finally { @@ -211,13 +203,11 @@ export function HuddleBar({ className }: HuddleBarProps) { return (
- {/* Error banner — dismissible, shown when start/join fails */} + {/* Error banner */} {huddleError && (
)} - {/* Room label */} - Huddle - - {/* Huddle status */} -
- - In huddle -
- {/* Model download progress */} {modelStatus && (modelStatus.moonshine !== "ready" || @@ -260,89 +241,117 @@ export function HuddleBar({ className }: HuddleBarProps) { )} - {/* Participant avatars */} - {state.participants.length > 0 && ( + {/* Participants */} +
+ { + if (!state.ephemeral_channel_id) return; + const confirmed = window.confirm( + "Remove this agent from the huddle?", + ); + if (!confirmed) return; + try { + await invoke("remove_channel_member", { + channelId: state.ephemeral_channel_id, + pubkey, + }); + // Optimistically remove from local state — the backend's + // 15s membership poll will eventually converge. + setState((prev) => { + if (!prev) return prev; + return { + ...prev, + participants: prev.participants.filter((p) => p !== pubkey), + agent_pubkeys: prev.agent_pubkeys.filter((p) => p !== pubkey), + }; + }); + } catch (e) { + console.error("Failed to remove agent from huddle:", e); + } + }} /> - )} - - {/* Voice input mode indicator */} -
- {micConnected ? ( - isPttMode ? ( - <> -
- PTT - Ctrl+Space - - ) : ( - <> -
0.05 - ? `rgba(34, 197, 94, ${0.4 + micLevel * 0.6})` - : "rgba(100, 116, 139, 0.4)", - }} - title={`Mic level: ${Math.round(micLevel * 100)}%`} - /> - VAD - - ) - ) : ( - no mic - )} +
- {/* Voice input mode toggle */} - - - {/* Add agent button */} - + {/* Voice input mode — single toggle combining indicator + switch */} + {micConnected ? ( + <> + + {isPttMode && ( + + {navigator.platform?.includes("Mac") ? "⌃Space" : "Ctrl+Space"} + + )} + + ) : ( + +
+ {isStarting ? "Connecting…" : "No mic"} + + )} {agentAddError && ( @@ -357,9 +366,14 @@ export function HuddleBar({ className }: HuddleBarProps) { onAdd={async (pubkey: string): Promise => { setAgentAddError(null); try { - return await invoke("add_agent_to_huddle", { - agentPubkey: pubkey, - }); + const result = await invoke( + "add_agent_to_huddle", + { agentPubkey: pubkey }, + ); + // Refresh huddle state so the participant list updates immediately. + const s = await invoke("get_huddle_state"); + setState(s); + return result; } catch (e: unknown) { const msg = e instanceof Error ? e.message : String(e); setAgentAddError(`Failed to add agent: ${msg}`); @@ -369,83 +383,65 @@ export function HuddleBar({ className }: HuddleBarProps) { /> )} - {/* Mute toggle — in PTT mode acts as hard mute override (even PTT won't transmit) */} - + setIsMuted((m) => !m)} + isPttMode={isPttMode} + pttActive={pttActive} + micConnected={micConnected} + audioDevices={audioDevices} + selectedDeviceId={selectedDeviceId} + onSelectDevice={setSelectedDeviceId} + micGain={micGain} + onGainChange={setMicGain} + /> - {/* TTS toggle */} - + outputDevices={outputDevices} + selectedOutputDevice={selectedOutputDevice} + onSelectOutputDevice={setSelectedOutputDevice} + /> - {/* Leave / End buttons — available to all participants */} - - - {state?.is_creator && ( + {/* Leave / End buttons — pushed to the right */} +
- )} + {state?.is_creator && ( + + )} +
{/* Screen reader announcements for huddle state changes */} diff --git a/desktop/src/features/huddle/components/HuddleIndicator.tsx b/desktop/src/features/huddle/components/HuddleIndicator.tsx index 6cdea4555..b486d04a6 100644 --- a/desktop/src/features/huddle/components/HuddleIndicator.tsx +++ b/desktop/src/features/huddle/components/HuddleIndicator.tsx @@ -1,3 +1,4 @@ +import { listen } from "@tauri-apps/api/event"; import { Headphones } from "lucide-react"; import * as React from "react"; import { useQueryClient } from "@tanstack/react-query"; @@ -164,6 +165,29 @@ export function HuddleIndicator({ }; }, [channelId]); + // When the local user ends/leaves a huddle, the backend transitions to idle + // and emits huddle-state-changed. Clear the indicator immediately rather than + // waiting for the relay's 48103 event (which may arrive late or not at all + // if the relay connection tears down first). + React.useEffect(() => { + let unlisten: (() => void) | null = null; + let cancelled = false; + + listen<{ phase: string }>("huddle-state-changed", (event) => { + if (!cancelled && event.payload.phase === "idle") { + setActiveHuddle(null); + } + }).then((fn) => { + if (cancelled) fn(); + else unlisten = fn; + }); + + return () => { + cancelled = true; + unlisten?.(); + }; + }, []); + // No active huddle — render the start button (if onStart provided). if (!activeHuddle) { if (!onStart) return null; diff --git a/desktop/src/features/huddle/components/MicControls.tsx b/desktop/src/features/huddle/components/MicControls.tsx new file mode 100644 index 000000000..5730144f5 --- /dev/null +++ b/desktop/src/features/huddle/components/MicControls.tsx @@ -0,0 +1,225 @@ +import { Check, ChevronUp, Mic, MicOff, Volume2, VolumeX } from "lucide-react"; + +import { cn } from "@/shared/lib/cn"; +import { Button } from "@/shared/ui/button"; +import { Popover, PopoverContent, PopoverTrigger } from "@/shared/ui/popover"; + +type MicControlsProps = { + isMuted: boolean; + onToggleMute: () => void; + isPttMode: boolean; + pttActive: boolean; + micConnected: boolean; + audioDevices: MediaDeviceInfo[]; + selectedDeviceId: string; + onSelectDevice: (id: string) => void; + micGain: number; + onGainChange: (value: number) => void; +}; + +export function MicControls({ + isMuted, + onToggleMute, + isPttMode, + pttActive, + micConnected, + audioDevices, + selectedDeviceId, + onSelectDevice, + micGain, + onGainChange, +}: MicControlsProps) { + return ( + +
+ + + + +
+ +
+ ({ + id: d.deviceId, + label: d.label || `Mic ${d.deviceId.slice(0, 8)}`, + }))} + selectedId={selectedDeviceId} + onSelect={onSelectDevice} + showChangeHint={!!selectedDeviceId && micConnected} + /> +
+ +
+ onGainChange(Number(e.target.value))} + className="h-1.5 w-full cursor-pointer appearance-none rounded-full bg-muted accent-foreground" + /> + + {Math.round(micGain * 100)}% + +
+
+
+ + + ); +} + +type SpeakerControlsProps = { + ttsEnabled: boolean; + onToggleTts: () => void; + outputDevices: { name: string; is_default: boolean }[]; + selectedOutputDevice: string; + onSelectOutputDevice: (name: string) => void; +}; + +export function SpeakerControls({ + ttsEnabled, + onToggleTts, + outputDevices, + selectedOutputDevice, + onSelectOutputDevice, +}: SpeakerControlsProps) { + return ( + +
+ + + + +
+ + ({ id: d.name, label: d.name }))} + selectedId={selectedOutputDevice} + onSelect={onSelectOutputDevice} + showChangeHint={!!selectedOutputDevice} + /> + + + ); +} + +export function DeviceList({ + label, + devices, + selectedId, + onSelect, + showChangeHint, +}: { + label: string; + devices: { id: string; label: string }[]; + selectedId: string; + onSelect: (id: string) => void; + showChangeHint: boolean; +}) { + return ( +
+ {label} +
    +
  • + +
    • + {devices.map((d) => { + const isSelected = selectedId === d.id; + return ( +
  • + +
    • + ); + })} +
+ {showChangeHint && ( +

+ Change takes effect on next huddle +

+ )} +
+ ); +} diff --git a/desktop/src/features/huddle/components/ParticipantList.tsx b/desktop/src/features/huddle/components/ParticipantList.tsx index f53666c85..84d965129 100644 --- a/desktop/src/features/huddle/components/ParticipantList.tsx +++ b/desktop/src/features/huddle/components/ParticipantList.tsx @@ -1,3 +1,4 @@ +import { X } from "lucide-react"; import * as React from "react"; import { cn } from "@/shared/lib/cn"; @@ -10,6 +11,8 @@ type ParticipantListProps = { activeSpeakers?: string[]; /** Pubkeys of agent participants — rendered with a bot badge */ agentPubkeys?: string[]; + /** Called when the user clicks the remove button on an agent avatar */ + onRemoveAgent?: (pubkey: string) => void; className?: string; }; @@ -17,6 +20,7 @@ export function ParticipantList({ participants, activeSpeakers, agentPubkeys, + onRemoveAgent, className, }: ParticipantListProps) { const { data } = useUsersBatchQuery(participants); @@ -38,7 +42,7 @@ export function ParticipantList({ const ariaLabel = `${profile?.displayName || `Participant ${pubkey.slice(0, 8)}`}${isAgent ? " (agent)" : ""}`; return ( -
  • +
  • {hasProfile ? (
    )} - {isAgent && ( - - )} + {isAgent && + (onRemoveAgent ? ( + + ) : ( + + ))}
    • ); })} diff --git a/desktop/src/features/huddle/lib/audioWorklet.ts b/desktop/src/features/huddle/lib/audioWorklet.ts index 4022e8b85..62fd17c00 100644 --- a/desktop/src/features/huddle/lib/audioWorklet.ts +++ b/desktop/src/features/huddle/lib/audioWorklet.ts @@ -24,6 +24,8 @@ export type AudioWorkletHandle = { /** Switch voice input mode. In VAD mode, always transmitting (PTT events ignored). * In PTT mode, gated by Ctrl+Space. */ setMode: (mode: "push_to_talk" | "voice_activity") => void; + /** Set mic input gain (0–1). Adjusts the GainNode between source and worklet. */ + setGain: (value: number) => void; }; /** @@ -65,11 +67,15 @@ export async function setupAudioWorklet( new MediaStream([audioTrack]), ); + // Create gain node for volume control + const gainNode = audioContext.createGain(); + // Create worklet node const workletNode = new AudioWorkletNode(audioContext, "stt-tap-processor"); - // Connect: mic → worklet (tap only — no playback) - source.connect(workletNode); + // Connect: mic → gain → worklet (tap only — no playback) + source.connect(gainNode); + gainNode.connect(workletNode); // Set initial PTT state (worklet defaults to transmitting=true). // In PTT mode, immediately gate audio until the user presses the key. @@ -122,6 +128,7 @@ export async function setupAudioWorklet( workletNode.port.onmessage = null; pttUnlisten?.(); source.disconnect(); + gainNode.disconnect(); workletNode.disconnect(); void audioContext.close(); }, @@ -137,5 +144,8 @@ export async function setupAudioWorklet( active: mode === "voice_activity", }); }, + setGain: (value: number) => { + gainNode.gain.value = value; + }, }; } diff --git a/desktop/src/features/huddle/lib/useAudioDevices.ts b/desktop/src/features/huddle/lib/useAudioDevices.ts new file mode 100644 index 000000000..b7adc6789 --- /dev/null +++ b/desktop/src/features/huddle/lib/useAudioDevices.ts @@ -0,0 +1,56 @@ +import * as React from "react"; + +import type { AudioWorkletHandle } from "./audioWorklet"; + +/** + * Manages audio input device enumeration, device selection, and mic gain. + * Extracted from HuddleContext to keep file sizes manageable. + */ +export function useAudioDevices( + workletRef: React.RefObject, +) { + const [audioDevices, setAudioDevices] = React.useState([]); + const [selectedDeviceId, setSelectedDeviceId] = React.useState(""); + const [micGain, setMicGainState] = React.useState(1); + const micGainRef = React.useRef(1); + + // Enumerate audio input devices on mount and when devices change. + React.useEffect(() => { + function refreshDevices() { + navigator.mediaDevices + .enumerateDevices() + .then((devices) => + setAudioDevices(devices.filter((d) => d.kind === "audioinput")), + ) + .catch(() => { + /* best-effort */ + }); + } + refreshDevices(); + navigator.mediaDevices.addEventListener("devicechange", refreshDevices); + return () => { + navigator.mediaDevices.removeEventListener( + "devicechange", + refreshDevices, + ); + }; + }, []); + + const setMicGain = React.useCallback( + (value: number) => { + const clamped = Math.max(0, Math.min(1, value)); + micGainRef.current = clamped; + setMicGainState(clamped); + workletRef.current?.setGain(clamped); + }, + [workletRef], + ); + + return { + audioDevices, + selectedDeviceId, + setSelectedDeviceId, + micGain, + setMicGain, + }; +} diff --git a/desktop/src/features/huddle/lib/useTtsSubscription.ts b/desktop/src/features/huddle/lib/useTtsSubscription.ts new file mode 100644 index 000000000..bd09ce82c --- /dev/null +++ b/desktop/src/features/huddle/lib/useTtsSubscription.ts @@ -0,0 +1,111 @@ +import { invoke } from "@tauri-apps/api/core"; +import * as React from "react"; + +import { relayClient } from "@/shared/api/relayClient"; + +/** + * Subscribe to agent TTS messages on the ephemeral huddle channel. + * Pipes agent kind:9 messages to `speak_agent_message` on the Rust backend. + * + * Extracted from HuddleContext to keep file sizes manageable. + */ +export function useTtsSubscription( + ephemeralChannelId: string | null, + selfPubkeyRef: React.RefObject, +) { + React.useEffect(() => { + if (!ephemeralChannelId) return; + + let disposed = false; + let cleanup: (() => void) | null = null; + + // ── Agent identity (authoritative, fail-closed) ─────────────────────── + // + // Fetch the ephemeral channel's member list from the relay REST API and + // identify agents by their "bot" role. This is authoritative — it works + // for both creators and joiners, and reflects mid-huddle agent additions. + // + // FAIL-CLOSED: agentsLoaded starts false. Until the fetch succeeds and + // populates agentPubkeys, NO messages are spoken. An empty set after a + // successful fetch means "no agents in the huddle" → still mute. + let agentsLoaded = false; + const agentPubkeys = new Set(); + + async function loadAgentPubkeys() { + try { + const pubkeys = await invoke("get_huddle_agent_pubkeys"); + agentPubkeys.clear(); + for (const pk of pubkeys) agentPubkeys.add(pk); + agentsLoaded = true; + } catch (e) { + // Fail-closed on ALL failures, including refresh after prior success. + // Clear the set and mark as not loaded — TTS goes mute until the + // next successful refresh. Stale membership must never authorize speech. + agentPubkeys.clear(); + agentsLoaded = false; + console.error("[huddle] Failed to load agent pubkeys:", e); + } + } + + // Initial load + periodic refresh (catches mid-huddle agent additions). + void loadAgentPubkeys(); + const agentRefreshId = window.setInterval(() => { + void loadAgentPubkeys(); + }, 10_000); + + // ── Live-only subscription ─────────────────────────────────────────── + // subscribeToChannelLive uses `since: now` — the relay never sends + // historical backlog. Every event delivered is a live message. + // Event-ID dedup handles reconnect replay (same event arriving twice). + const seenEventIds = new Set(); + const seenOrder: string[] = []; + const MAX_SEEN_EVENTS = 5000; + + relayClient + .subscribeToChannelLive(ephemeralChannelId, (event) => { + if (disposed) return; + // Defense-in-depth: subscription already filters to kind:9 only. + if (event.kind !== 9) return; + + // Dedup by event ID (covers reconnect replay). + if (seenEventIds.has(event.id)) return; + seenEventIds.add(event.id); + seenOrder.push(event.id); + if (seenOrder.length > MAX_SEEN_EVENTS) { + const oldest = seenOrder.shift(); + if (oldest !== undefined) seenEventIds.delete(oldest); + } + + // Fail-closed: don't speak until agent list is loaded. + if (!agentsLoaded) return; + // Only speak agent messages — skip human STT transcripts. + if (!agentPubkeys.has(event.pubkey)) return; + if (event.pubkey === selfPubkeyRef.current) return; + if (event.content.trim().length <= 1) return; + // Legacy: skip [System]-prefixed messages from before kind:48106. + if (event.content.startsWith("[System]")) return; + invoke("speak_agent_message", { text: event.content }).catch((err) => { + console.warn( + "[huddle] TTS speak failed (backpressure or pipeline unavailable):", + err, + ); + }); + }) + .then((dispose) => { + if (disposed) { + void dispose(); + return; + } + cleanup = () => void dispose(); + }) + .catch((err) => { + console.error("[huddle] TTS subscription failed:", err); + }); + + return () => { + disposed = true; + cleanup?.(); + window.clearInterval(agentRefreshId); + }; + }, [ephemeralChannelId, selfPubkeyRef]); +} diff --git a/desktop/src/shared/ui/sidebar.tsx b/desktop/src/shared/ui/sidebar.tsx index 477eb1ec0..0545ea574 100644 --- a/desktop/src/shared/ui/sidebar.tsx +++ b/desktop/src/shared/ui/sidebar.tsx @@ -145,7 +145,7 @@ const SidebarProvider = React.forwardRef< } as React.CSSProperties } className={cn( - "group/sidebar-wrapper flex min-h-svh w-full has-[[data-variant=inset]]:bg-sidebar", + "group/sidebar-wrapper flex h-full min-h-0 w-full has-[[data-variant=inset]]:bg-sidebar", className, )} ref={ref} @@ -223,7 +223,7 @@ const Sidebar = React.forwardRef< return (