refactor(voice): extract reusable Pocket primitives + add Pocket voice settings

Combined landing of #2467 (extract buzz-voice crate) and #3208 (Pocket
voice settings), replayed onto main after #3266 and #3180 merged.

The original stacked PRs could not land as-is: the repo is squash-only
with delete-branch-on-merge, so #2467 was auto-closed when its base
branch was deleted, and squash merges severed the ancestry GitHub uses
for conflict detection.

Content is byte-identical to the blessed jt/buzz-voice-refactor branch
(93029c577, tree 6729e0eff) except for the three-file interaction with
 #3180 (huddle/mod.rs, huddle/state.rs, e2eBridge.ts), resolved here as
the union of both sides:

- huddle/mod.rs: keep #3180's pipeline re-exports (check_pipeline_hotstart,
  start_auto_enabled_transcription, PostConnectOutcome) alongside #3208's
  agent_tts_routing imports and await_inflight_tts_start.
- huddle/state.rs: reset_preserving_generation preserves both #3180's
  huddle_generation and #3208's tts_enabled; both test sets merged into
  the single tests module (fixes E0428 from textual union).
- e2eBridge.ts: both switch arms kept — #3180's huddle mock commands and
  #3208's TTS settings/voice-registry commands; no duplicate case labels.

Verified at this tree: cargo test 2047+3/0 (14 ignored), clippy -D
warnings clean, cargo fmt clean, tsc clean, pnpm test 3885/0, lint clean.

Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
Signed-off-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
This commit is contained in:
npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d
2026-07-31 07:30:52 -04:00
parent 4632c55041
commit cf32dacd71
63 changed files with 5019 additions and 698 deletions
Generated
+538 -64
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -26,6 +26,7 @@ members = [
"crates/buzz-pair-relay",
"crates/buzz-relay-mesh",
"crates/buzz-dev-mcp",
"crates/buzz-voice",
"examples/countdown-bot",
]
exclude = ["desktop/src-tauri"]
+1
View File
@@ -276,6 +276,7 @@ test-unit:
#!/usr/bin/env bash
if command -v cargo-nextest &>/dev/null; then
cargo nextest run -p buzz-core -p buzz-auth --lib
cargo nextest run -p buzz-voice --lib
cargo nextest run -p buzz-cli
# buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra).
# They guard the embedded-migrator invariant (exactly the consolidated
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "buzz-voice"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
description = "Reusable local voice primitives for Buzz"
[dependencies]
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] }
ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] }
rand = "0.10"
sentencepiece-model = "0.1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sherpa-onnx = "1.12"
tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] }
+22
View File
@@ -0,0 +1,22 @@
//! Reusable local voice primitives for Buzz.
pub mod pocket;
pub use pocket::{
april_model_info, load_text_to_speech, load_voice_style, PocketModelInfo, PocketTts,
VoiceStyle, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT,
};
/// One immutable artifact required by the April Pocket bundle.
///
/// `filename` is the bundle-relative file name, `sha256` pins its contents,
/// `size_bytes` supports download progress and validation, and `quantized`
/// identifies the INT8 components.
pub type PocketModelArtifact = pocket::PocketModelArtifact;
/// Language bundle selected from the pinned export.
pub const APRIL_BUNDLE_ID: &str = pocket::APRIL_BUNDLE_ID;
/// Pinned upstream export repository.
pub const APRIL_MODEL_ID: &str = pocket::APRIL_MODEL_ID;
/// Pinned revision containing the April bundle.
pub const APRIL_MODEL_REVISION: &str = pocket::APRIL_MODEL_REVISION;
+167
View File
@@ -0,0 +1,167 @@
//! April 2026 Pocket TTS engine for Buzz Desktop.
//!
//! The `english_2026-04` bundle uses SentencePiece tokenization, a learned
//! voice BOS embedding, recurrent FlowLM state, and stateful Mimi decoding.
//! Buzz selects the upstream three-graph INT8 variant while retaining the
//! full-precision Mimi encoder and text conditioner specified by that variant.
//!
//! ## Attribution
//!
//! - Pocket TTS and Mimi: Kyutai, CC-BY-4.0.
//! - ONNX export: KevinAHM/pocket-tts-onnx, CC-BY-4.0.
//! - Reference voice: Kyutai's Mary preset (VCTK p333), CC-BY-4.0.
//!
//! `huddle::models` writes the complete attribution beside the cached bytes.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use sherpa_onnx::Wave;
#[path = "pocket_april.rs"]
mod pocket_april;
#[path = "pocket_models.rs"]
mod pocket_models;
use pocket_april::{prepare_april_prompt, AprilPocketTts};
pub use pocket_models::{
april_model_info, PocketModelArtifact, PocketModelInfo, APRIL_BUNDLE_ID, APRIL_MODEL_ID,
APRIL_MODEL_REVISION,
};
/// Pocket TTS emits 24 kHz mono PCM.
pub const SAMPLE_RATE: u32 = 24_000;
/// Bundled reference voice name without its extension.
pub const DEFAULT_VOICE: &str = "reference_sample";
/// Pocket voice files are reference WAVs.
pub const VOICE_FILE_EXT: &str = "wav";
const TTS_NUM_THREADS: usize = 1;
/// Loaded reference voice samples and their original sample rate.
#[derive(Debug, Clone)]
pub struct VoiceStyle {
samples: Vec<f32>,
sample_rate: i32,
}
/// Load a Pocket reference voice WAV from disk.
pub fn load_voice_style(path: &Path) -> Result<VoiceStyle, String> {
let path_str = path
.to_str()
.ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?;
let wave = Wave::read(path_str)
.ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?;
let samples = wave.samples().to_vec();
if samples.is_empty() {
return Err(format!("voice WAV is empty: {}", path.display()));
}
Ok(VoiceStyle {
samples,
sample_rate: wave.sample_rate(),
})
}
/// Resident April INT8 Pocket TTS engine.
pub struct PocketTts {
inner: Mutex<AprilPocketTts>,
}
/// Load Buzz Desktop's pinned April INT8 model.
pub fn load_text_to_speech(model_dir: &str) -> Result<PocketTts, String> {
let dir = PathBuf::from(model_dir);
for artifact in april_model_info().artifacts {
let path = dir.join(artifact.filename);
if !path.is_file() {
return Err(format!(
"incomplete Pocket TTS {} INT8 bundle: missing {}",
APRIL_BUNDLE_ID,
path.display()
));
}
}
Ok(PocketTts {
inner: Mutex::new(AprilPocketTts::load(&dir, TTS_NUM_THREADS)?),
})
}
impl PocketTts {
/// Split text into synthesis units that satisfy the bundle's exact
/// 50-token input limit.
pub fn split_text_into_chunks(&self, text: &str) -> Result<Vec<String>, String> {
let Some(prepared) = prepare_april_prompt(text) else {
return Ok(Vec::new());
};
self.inner
.lock()
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?
.split_prompt(&prepared)
}
/// Synthesize text with the supplied reference voice.
///
/// Pocket detects language from text and this model uses one synthesis
/// step, so `_lang` and `_steps` intentionally do not affect output.
pub fn synth_chunk(
&self,
text: &str,
_lang: &str,
style: &VoiceStyle,
_steps: usize,
) -> Result<Vec<f32>, String> {
let Some(prepared) = prepare_april_prompt(text) else {
return Ok(Vec::new());
};
let mut engine = self
.inner
.lock()
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?;
let chunks = engine.split_prompt(&prepared)?;
let mut samples = Vec::new();
for chunk in chunks {
let prepared = prepare_april_prompt(&chunk)
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
samples.extend(engine.synth_chunk(&prepared, style)?);
}
Ok(samples)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn desktop_model_is_april_int8_only() {
let info = april_model_info();
assert_eq!(info.max_token_per_chunk, 50);
assert_eq!(info.sample_rate, SAMPLE_RATE);
assert!(info
.artifacts
.iter()
.any(|artifact| artifact.filename == "flow_lm_main_int8.onnx"));
assert!(!info
.artifacts
.iter()
.any(|artifact| artifact.filename == "flow_lm_main.onnx"));
}
#[test]
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
fn production_api_emits_non_silent_april_int8_pcm() {
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
.expect("load reference voice");
let samples = engine
.synth_chunk("Bright birds begin beside the bay.", "en", &style, 1)
.expect("synthesize through the production API");
assert!(!samples.is_empty());
assert!(samples.iter().all(|sample| sample.is_finite()));
assert!(samples.iter().any(|sample| sample.abs() > 1.0e-6));
}
}
@@ -24,12 +24,19 @@ pub struct PocketModelArtifact {
/// Capabilities of Buzz Desktop's sole Pocket model.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PocketModelInfo {
/// Language bundle selected from the pinned export.
pub bundle_id: &'static str,
/// Upstream model repository.
pub source_model_id: &'static str,
/// Pinned upstream model revision.
pub revision: &'static str,
/// PCM output sample rate.
pub sample_rate: u32,
/// Maximum input size declared by the bundle.
pub max_token_per_chunk: usize,
/// Immutable files required by the runtime.
pub artifacts: &'static [PocketModelArtifact],
/// Components quantized in the selected bundle.
pub quantized_components: &'static [&'static str],
}
+1
View File
@@ -49,6 +49,7 @@ export default defineConfig({
"**/activity-scope-label-screenshots.spec.ts",
"**/welcome-agent-modal-screenshots.spec.ts",
"**/local-archive-screenshots.spec.ts",
"**/voice-settings.spec.ts",
"**/agent-readiness-screenshots.spec.ts",
"**/agent-error-state-screenshots.spec.ts",
"**/edit-agent.spec.ts",
+15 -5
View File
@@ -1049,6 +1049,7 @@ dependencies = [
"buzz-media",
"buzz-persona",
"buzz-sdk",
"buzz-voice",
"bytes",
"bzip2 0.6.1",
"chrono",
@@ -1078,11 +1079,8 @@ dependencies = [
"objc2-app-kit",
"objc2-foundation",
"opus",
"ort",
"ort-sys",
"plist",
"png 0.18.1",
"rand 0.10.2",
"regex",
"reqwest 0.13.4",
"rodio",
@@ -1090,7 +1088,6 @@ dependencies = [
"rusqlite",
"rustls",
"security-framework 3.7.0",
"sentencepiece-model",
"serde",
"serde_json",
"serde_yaml",
@@ -1110,7 +1107,6 @@ dependencies = [
"tauri-plugin-updater",
"tauri-plugin-window-state",
"tempfile",
"tokenizers",
"tokio",
"tokio-tungstenite 0.29.0",
"tokio-util",
@@ -1178,6 +1174,20 @@ dependencies = [
"uuid",
]
[[package]]
name = "buzz-voice"
version = "0.1.0"
dependencies = [
"ort",
"ort-sys",
"rand 0.10.2",
"sentencepiece-model",
"serde",
"serde_json",
"sherpa-onnx",
"tokenizers",
]
[[package]]
name = "by_address"
version = "1.2.1"
+1 -5
View File
@@ -82,9 +82,6 @@ bytes = "1"
futures-util = "0.3"
opus = "0.3"
neteq = { version = "0.8", default-features = false }
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] }
ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] }
rand = "0.10"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
@@ -101,6 +98,7 @@ buzz_core_pkg = { package = "buzz-core", path = "../../crates/buzz-core" }
buzz_persona_pkg = { package = "buzz-persona", path = "../../crates/buzz-persona" }
buzz_sdk_pkg = { package = "buzz-sdk", path = "../../crates/buzz-sdk" }
buzz_agent_pkg = { package = "buzz-agent", path = "../../crates/buzz-agent" }
buzz_voice_pkg = { package = "buzz-voice", path = "../../crates/buzz-voice" }
iroh = { version = "1.0.2", optional = true }
mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.74.0", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"], optional = true }
mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.74.0", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"], optional = true }
@@ -128,7 +126,6 @@ image = { version = "0.25", default-features = false, features = ["jpeg", "png",
zip = "8"
flate2 = "1"
sherpa-onnx = "1.12"
sentencepiece-model = "0.1"
regex = "1"
rusqlite = { version = "0.37", features = ["bundled"] }
axum = "0.8"
@@ -139,7 +136,6 @@ audioadapter-buffers = "3.0"
tempfile = "3"
strip-ansi-escapes = "0.2"
tracing = "0.1"
tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] }
[dev-dependencies]
# `test-util` enables tokio's paused-clock (`start_paused`) so the relay
@@ -0,0 +1,35 @@
# Pocket TTS English VCTK presets
Buzz exposes Kyutai's twelve official English VCTK Pocket presets. The WAV
bytes are unchanged from `kyutai/tts-voices` revision
`323332d33f997de8394f24a193e1a76df720e01a`; only local filenames differ.
| Voice | Upstream asset | SHA-256 |
| --- | --- | --- |
| Anna | `vctk/p228_023_enhanced.wav` | `0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856` |
| Vera | `vctk/p229_023_enhanced.wav` | `309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b` |
| Fantine | `vctk/p244_023_enhanced.wav` | `5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b` |
| Charles | `vctk/p254_023_enhanced.wav` | `6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756` |
| Paul | `vctk/p259_023_enhanced.wav` | `7aba504fe0b3b16478b69eb27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b` |
| Eponine | `vctk/p262_023_enhanced.wav` | `a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b` |
| Azelma | `vctk/p303_023_enhanced.wav` | `60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026` |
| George | `vctk/p315_023_enhanced.wav` | `29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae` |
| Mary | `vctk/p333_023_enhanced.wav` | `a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f` |
| Jane | `vctk/p339_023_enhanced.wav` | `2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a` |
| Michael | `vctk/p360_023_enhanced.wav` | `b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad` |
| Eve | `vctk/p361_023_enhanced.wav` | `396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd` |
Mary is already installed as the Pocket model's `reference_sample.wav`, so it
is not duplicated in this resource directory.
Source repository:
https://huggingface.co/kyutai/tts-voices/tree/323332d33f997de8394f24a193e1a76df720e01a/vctk
The original recordings are from the Voice Cloning Toolkit (VCTK) corpus,
licensed CC BY 4.0:
https://datashare.ed.ac.uk/handle/10283/3443
The recordings were enhanced by ai-coustics:
https://ai-coustics.com/
Neither Kyutai, the VCTK speakers, nor ai-coustics endorses Buzz.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+2 -4
View File
@@ -53,15 +53,13 @@ pub struct AppState {
pub channel_templates_store_lock: Mutex<()>,
pub managed_agent_processes: Mutex<HashMap<ManagedAgentRuntimeKey, ManagedAgentPairRuntime>>,
pub huddle_state: Mutex<HuddleState>,
pub huddle_audio: crate::huddle::tts_settings::HuddleAudioSettingsState,
/// Tauri app handle — stored after setup so huddle commands can emit
/// `huddle-state-changed` events without needing the handle threaded
/// through every call site.
///
/// Set once during `setup()` in `lib.rs`; never cleared.
pub app_handle: Mutex<Option<AppHandle>>,
/// Selected audio output device name. `None` = system default.
/// Used by `connect_audio_relay` and TTS pipeline when opening sinks.
pub audio_output_device: Mutex<Option<String>>,
/// Port of the localhost media streaming proxy (set during setup).
pub media_proxy_port: AtomicU16,
/// Set when identity resolution detected a "keyring-locked" state: the
@@ -219,8 +217,8 @@ pub fn build_app_state() -> AppState {
managed_agent_processes: Mutex::new(HashMap::new()),
session_config_cache: Mutex::new(HashMap::new()),
huddle_state: Mutex::new(HuddleState::default()),
huddle_audio: Default::default(),
app_handle: Mutex::new(None),
audio_output_device: Mutex::new(None),
media_proxy_port: AtomicU16::new(0),
prevent_sleep: Arc::new(Mutex::new(
crate::prevent_sleep::PreventSleepState::default(),
@@ -0,0 +1,56 @@
use super::HuddlePhase;
#[derive(Debug, PartialEq, Eq)]
pub(super) enum AgentTtsRuntimeGate {
Disabled,
Inactive,
NeedsPipeline,
Ready,
}
pub(super) fn classify_agent_tts_runtime(
enabled: bool,
phase: &HuddlePhase,
has_pipeline: bool,
) -> AgentTtsRuntimeGate {
if !enabled {
AgentTtsRuntimeGate::Disabled
} else if !matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) {
AgentTtsRuntimeGate::Inactive
} else if has_pipeline {
AgentTtsRuntimeGate::Ready
} else {
AgentTtsRuntimeGate::NeedsPipeline
}
}
/// Maximum text length accepted for TTS synthesis.
/// ~2000 chars is 12 minutes of speech. Longer messages are truncated.
pub(super) const MAX_TTS_TEXT_LEN: usize = 2000;
pub(super) fn normalize_agent_tts_text(text: String) -> String {
if text.chars().count() > MAX_TTS_TEXT_LEN {
let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect();
truncated.push_str("... message truncated.");
truncated
} else {
text
}
}
pub(super) async fn enqueue_agent_tts_text<F>(
route_id: u64,
text: String,
enqueue: F,
) -> Result<(), String>
where
F: FnOnce(u64, String) -> Result<(), String> + Send + 'static,
{
tokio::task::spawn_blocking(move || enqueue(route_id, text))
.await
.map_err(|error| format!("TTS enqueue task failed: {error}"))?
}
#[cfg(test)]
#[path = "agent_tts_routing_tests.rs"]
mod tests;
@@ -0,0 +1,57 @@
use super::{
classify_agent_tts_runtime, enqueue_agent_tts_text, normalize_agent_tts_text,
AgentTtsRuntimeGate, MAX_TTS_TEXT_LEN,
};
use crate::huddle::HuddlePhase;
#[tokio::test]
async fn assistant_plain_text_routes_unchanged_into_voice_pipeline_boundary() {
let (sender, receiver) = std::sync::mpsc::channel();
let text = "A newly submitted assistant reply.".to_string();
let route_id = 42;
enqueue_agent_tts_text(route_id, text.clone(), move |route_id, queued| {
sender
.send((route_id, queued))
.map_err(|error| error.to_string())
})
.await
.expect("route assistant text");
assert_eq!(
receiver.recv().expect("queued text"),
(route_id, text),
"route correlation must survive the native queue boundary"
);
}
#[test]
fn disabled_is_the_only_intentional_runtime_no_op() {
assert_eq!(
classify_agent_tts_runtime(false, &HuddlePhase::Connected, false),
AgentTtsRuntimeGate::Disabled
);
assert_eq!(
classify_agent_tts_runtime(true, &HuddlePhase::Idle, false),
AgentTtsRuntimeGate::Inactive
);
assert_eq!(
classify_agent_tts_runtime(true, &HuddlePhase::Connected, false),
AgentTtsRuntimeGate::NeedsPipeline
);
assert_eq!(
classify_agent_tts_runtime(true, &HuddlePhase::Connected, true),
AgentTtsRuntimeGate::Ready
);
}
#[test]
fn assistant_text_truncation_is_unicode_safe_before_voice_routing() {
let input = "🦀".repeat(MAX_TTS_TEXT_LEN + 1);
let output = normalize_agent_tts_text(input);
assert_eq!(
output.chars().count(),
MAX_TTS_TEXT_LEN + "... message truncated.".chars().count()
);
assert!(output.ends_with("... message truncated."));
}
+4 -2
View File
@@ -39,7 +39,8 @@ fn list_audio_output_devices_blocking() -> Result<Vec<AudioOutputDevice>, String
#[tauri::command]
pub fn set_audio_output_device(name: String, state: State<'_, AppState>) -> Result<(), String> {
let mut guard = state
.audio_output_device
.huddle_audio
.output_device
.lock()
.map_err(|e| e.to_string())?;
*guard = if name.is_empty() { None } else { Some(name) };
@@ -50,7 +51,8 @@ pub fn set_audio_output_device(name: String, state: State<'_, AppState>) -> Resu
#[tauri::command]
pub fn get_audio_output_device(state: State<'_, AppState>) -> Result<String, String> {
let guard = state
.audio_output_device
.huddle_audio
.output_device
.lock()
.map_err(|e| e.to_string())?;
Ok(guard.clone().unwrap_or_default())
+79 -71
View File
@@ -23,6 +23,7 @@
//! takes `stt_pipeline`/`tts_pipeline` out of the lock, then calls `shutdown()`
//! and drops them outside the lock (thread joins can block ~200ms).
mod agent_tts_routing;
pub mod agents;
pub mod audio_output;
pub mod jitter;
@@ -37,6 +38,8 @@ pub mod state;
pub mod stt;
pub mod transcription;
pub mod tts;
pub mod tts_settings;
mod tts_voice_registry;
pub mod wire;
// ── Shared utilities ──────────────────────────────────────────────────────────
@@ -63,6 +66,7 @@ pub(super) fn drain_until_shutdown<T>(
pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode};
pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline};
pub use tts_settings::set_tts_enabled;
// ── Imports ───────────────────────────────────────────────────────────────────
@@ -71,10 +75,15 @@ use tauri::State;
use uuid::Uuid;
use crate::{app_state::AppState, events, relay::submit_event};
use agent_tts_routing::{
classify_agent_tts_runtime, enqueue_agent_tts_text, normalize_agent_tts_text,
AgentTtsRuntimeGate,
};
pub use pipeline::check_pipeline_hotstart;
use pipeline::{
maybe_start_stt_pipeline, maybe_start_tts_pipeline, post_connect_setup,
start_auto_enabled_transcription, PostConnectOutcome,
await_inflight_tts_start, maybe_start_stt_pipeline, maybe_start_tts_pipeline,
post_connect_setup, start_auto_enabled_transcription, PostConnectOutcome,
};
use relay_api::{
count_human_members, fetch_channel_members, parse_channel_uuid, validate_pubkey_hex,
@@ -745,91 +754,90 @@ pub fn get_model_status(_state: State<'_, AppState>) -> Result<models::VoiceMode
})
}
/// Enable or disable TTS output.
///
/// When disabled, the TTS pipeline is shut down and audio output stops.
/// When re-enabled, the pipeline is restarted if TTS models are available.
///
/// Takes the pipeline handle out of the lock before calling shutdown() — the
/// thread join in Drop can block for ~200 ms (ONNX inference) and we don't
/// want to hold the HuddleState mutex during that time.
#[tauri::command]
pub async fn set_tts_enabled(enabled: bool, state: State<'_, AppState>) -> Result<(), String> {
let old_pipeline = {
let mut hs = state.huddle()?;
hs.tts_enabled = enabled;
if !enabled {
hs.tts_pipeline.take() // Take out of lock.
} else {
None
}
};
// Shut down outside the lock — thread join happens here.
if let Some(ref pipeline) = old_pipeline {
pipeline.shutdown();
}
drop(old_pipeline);
if enabled {
// Re-start TTS pipeline if models are available and huddle is active.
let phase = {
let hs = state.huddle()?;
hs.phase.clone()
};
if matches!(phase, HuddlePhase::Connected | HuddlePhase::Active) {
if let Err(e) = maybe_start_tts_pipeline(&state).await {
eprintln!("buzz-desktop: TTS pipeline restart failed: {e}");
}
}
}
Ok(())
}
/// Speak an agent message via TTS.
///
/// Maximum text length accepted for TTS synthesis.
/// ~2000 chars ≈ 12 minutes of speech. Longer messages are truncated.
const MAX_TTS_TEXT_LEN: usize = 2000;
/// Called by the WebView when it receives an incoming agent kind:9 message.
/// Called by the WebView when it receives an eligible live agent message.
/// Lazily starts the TTS pipeline if models are ready but the pipeline hasn't
/// been created yet (e.g. models finished downloading after huddle started).
///
/// No-op if TTS is disabled or models aren't ready.
/// Disabled is the only intentional no-op. Enabled-but-unavailable speech
/// returns an error so the caller cannot mistake a dropped message for success.
#[tauri::command]
pub async fn speak_agent_message(text: String, state: State<'_, AppState>) -> Result<(), String> {
pub async fn speak_agent_message(
text: String,
route_id: u64,
state: State<'_, AppState>,
) -> Result<(), String> {
eprintln!("buzz-desktop: tts stage=invoke status=started route_id={route_id}");
// Truncate oversized messages — agents shouldn't monologue in a voice huddle.
// Use char count (not byte length) to avoid panicking on multi-byte UTF-8.
let text = if text.chars().count() > MAX_TTS_TEXT_LEN {
let mut truncated: String = text.chars().take(MAX_TTS_TEXT_LEN).collect();
truncated.push_str("... message truncated.");
truncated
} else {
text
};
let text = normalize_agent_tts_text(text);
let needs_pipeline = {
let hs = state.huddle()?;
hs.tts_enabled
&& hs.tts_pipeline.is_none()
&& matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active)
let mut hs = state.huddle()?;
if hs
.tts_pipeline
.as_ref()
.is_some_and(|pipeline| pipeline.is_finished())
{
hs.tts_pipeline = None;
}
match classify_agent_tts_runtime(hs.tts_enabled, &hs.phase, hs.tts_pipeline.is_some()) {
AgentTtsRuntimeGate::Disabled => {
eprintln!(
"buzz-desktop: tts stage=invoke status=no_op reason=disabled route_id={route_id}"
);
return Ok(());
}
AgentTtsRuntimeGate::Inactive => {
eprintln!(
"buzz-desktop: tts stage=invoke status=failed reason=inactive_huddle route_id={route_id}"
);
return Err(
"Agent text to speech is unavailable outside an active huddle".to_string(),
);
}
AgentTtsRuntimeGate::NeedsPipeline => true,
AgentTtsRuntimeGate::Ready => false,
}
};
// Lazy-start: models may have finished downloading after the huddle began.
if needs_pipeline {
if let Err(e) = maybe_start_tts_pipeline(&state).await {
eprintln!("buzz-desktop: TTS lazy-start failed: {e}");
}
maybe_start_tts_pipeline(&state).await.inspect_err(|_| {
eprintln!(
"buzz-desktop: tts stage=invoke status=failed reason=startup_failed route_id={route_id}"
);
})?;
await_inflight_tts_start(&state).await.inspect_err(|_| {
eprintln!(
"buzz-desktop: tts stage=invoke status=failed reason=startup_timeout route_id={route_id}"
);
})?;
}
let hs = state.huddle()?;
if hs.tts_enabled {
if let Some(ref pipeline) = hs.tts_pipeline {
pipeline.speak(text)?;
}
}
Ok(())
let sender = {
let hs = state.huddle()?;
hs.tts_pipeline
.as_ref()
.map(|pipeline| pipeline.text_sender())
};
let Some(sender) = sender else {
eprintln!(
"buzz-desktop: tts stage=invoke status=failed reason=unavailable route_id={route_id}"
);
return Err("Agent text to speech is enabled but its audio pipeline is unavailable".into());
};
enqueue_agent_tts_text(route_id, text, move |route_id, text| {
sender
.send(route_id, text)
.map_err(|error| format!("TTS queue closed while waiting to enqueue: {error}"))
})
.await
.inspect(|_| eprintln!("buzz-desktop: tts stage=queue status=accepted route_id={route_id}"))
.inspect_err(|_| {
eprintln!("buzz-desktop: tts stage=queue status=failed reason=closed route_id={route_id}")
})
}
/// Add an agent to the active huddle.
+35 -39
View File
@@ -27,6 +27,10 @@ use sha2::{Digest, Sha256};
use super::pocket::{
april_model_info, PocketModelArtifact, APRIL_BUNDLE_ID, APRIL_MODEL_ID, APRIL_MODEL_REVISION,
};
use super::tts_voice_registry::POCKET_VOICES;
#[path = "models_voice_upgrade.rs"]
mod voice_upgrade;
// ── Integrity verification ────────────────────────────────────────────────────
//
@@ -91,8 +95,8 @@ const TTS_REFERENCE_ARTIFACT: PocketModelArtifact = PocketModelArtifact {
/// honest (each version tag identifies one specific set of model bytes).
const STT_MODEL_VERSION: &str = "2";
/// Identifies the exact April INT8 asset set expected by readiness checks.
const TTS_MODEL_VERSION: &str = "4";
/// Identifies the April INT8 asset set plus the official VCTK presets.
const TTS_MODEL_VERSION: &str = "5";
/// Filename for the version manifest written alongside model files.
const MANIFEST_FILENAME: &str = ".buzz-model-manifest";
@@ -160,39 +164,6 @@ const TTS_MODEL_DIR_NAME: &str = "pocket-tts";
/// Attribution sidecar written next to the Pocket TTS model files.
const TTS_LICENSE_FILE_NAME: &str = "MODEL_LICENSE.txt";
/// CC-BY-4.0 §3(a)(1) attribution block for Pocket TTS, its ONNX packaging,
/// and the bundled reference voice WAV.
const TTS_LICENSE_TEXT: &str = "\
Pocket TTS
© Kyutai.
Licensed under the Creative Commons Attribution 4.0 International License
(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/
Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts
Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926).
Mimi neural codec by Kyutai is bundled as part of the model.
April 2026 ONNX export by KevinAHM:
https://huggingface.co/KevinAHM/pocket-tts-onnx
Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c
Bundled reference voice (reference_sample.wav):
\"Mary (f, conversation)\" preset from the Kyutai TTS demo voice catalogue
(https://kyutai.org/tts), distributed via
https://huggingface.co/kyutai/tts-voices as `vctk/p333_023_enhanced.wav`.
Original recording from the Voice Cloning Toolkit (VCTK) corpus, speaker p333:
https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0).
Recording enhancement (denoise/dereverb) by ai-coustics:
https://ai-coustics.com/
Buzz ships all ONNX/model artifacts and the reference voice WAV unmodified,
renamed only by placement in the local model directory.
Provided \"AS IS\", without warranty of any kind, express or implied. See the
license text for full warranty disclaimer.
";
/// All files that must be present for Pocket TTS to be considered ready.
const TTS_EXPECTED_FILES: &[&str] = &[
"bundle.json",
@@ -205,6 +176,17 @@ const TTS_EXPECTED_FILES: &[&str] = &[
"tokenizer.model",
"LICENSE",
"reference_sample.wav",
"anna.wav",
"vera.wav",
"fantine.wav",
"charles.wav",
"paul.wav",
"eponine.wav",
"azelma.wav",
"george.wav",
"jane.wav",
"michael.wav",
"eve.wav",
TTS_LICENSE_FILE_NAME,
];
@@ -705,6 +687,9 @@ impl ModelManager {
/// Start a background Pocket TTS download. No-op if already ready or downloading.
pub fn start_tts_download(&self, http_client: reqwest::Client) {
if let Err(error) = voice_upgrade::install_vctk_presets_into_v4_model(&self.models_dir) {
eprintln!("buzz-desktop: could not upgrade existing Pocket voices in place: {error}");
}
let manager = self.clone();
self.tts.start_download(
&self.models_dir,
@@ -822,7 +807,7 @@ impl ModelManager {
/// - five ONNX sessions selected by the April INT8 bundle
/// - bundle metadata, SentencePiece tokenizer, and learned voice BOS
/// - upstream `LICENSE` plus Buzz's `MODEL_LICENSE.txt` attribution sidecar
/// - `reference_sample.wav` as the bundled default voice
/// - `reference_sample.wav` plus the embedded official VCTK presets
///
/// Files are written to a temp directory first, then moved atomically.
async fn download_tts_model(&self, http_client: reqwest::Client) -> Result<(), String> {
@@ -904,9 +889,20 @@ impl ModelManager {
});
}
tokio::fs::write(temp_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT)
.await
.map_err(|e| format!("write TTS model license sidecar: {e}"))?;
tokio::fs::write(
temp_dir.join(TTS_LICENSE_FILE_NAME),
voice_upgrade::TTS_LICENSE_TEXT,
)
.await
.map_err(|e| format!("write TTS model license sidecar: {e}"))?;
for voice in POCKET_VOICES {
let Some(bytes) = voice.bytes else {
continue;
};
tokio::fs::write(temp_dir.join(voice.reference_file), bytes)
.await
.map_err(|e| format!("install bundled {} voice: {e}", voice.display_name))?;
}
self.tts.set_status(ModelStatus::Downloading {
progress_percent: 90,
+3 -5
View File
@@ -22,11 +22,8 @@ fn expected_files_match_april_int8_metadata() {
.artifacts
.iter()
.map(|artifact| artifact.filename)
.chain([
TTS_LICENSE_ARTIFACT.filename,
TTS_REFERENCE_ARTIFACT.filename,
TTS_LICENSE_FILE_NAME,
])
.chain([TTS_LICENSE_ARTIFACT.filename, TTS_LICENSE_FILE_NAME])
.chain(POCKET_VOICES.iter().map(|voice| voice.reference_file))
.collect::<Vec<_>>();
expected.sort_unstable();
let mut actual = TTS_EXPECTED_FILES.to_vec();
@@ -36,6 +33,7 @@ fn expected_files_match_april_int8_metadata() {
assert!(!actual.contains(&"flow_lm_main.onnx"));
assert!(!actual.contains(&"flow_lm_flow.onnx"));
assert!(!actual.contains(&"mimi_decoder.onnx"));
assert!(!actual.contains(&"marius.wav"));
}
#[test]
@@ -0,0 +1,128 @@
use super::*;
use crate::huddle::tts_voice_registry::POCKET_VOICES;
const PRESET_VOICE_TTS_MODEL_VERSION: &str = "4";
/// Attribution written beside every installed Pocket model and voice asset.
pub(super) const TTS_LICENSE_TEXT: &str = "\
Pocket TTS
© Kyutai.
Licensed under the Creative Commons Attribution 4.0 International License
(CC-BY-4.0). License text: https://creativecommons.org/licenses/by/4.0/
Original model by Kyutai: https://huggingface.co/kyutai/pocket-tts
Paper: Charles, Roebel, et al., Pocket TTS (arXiv:2509.06926).
Mimi neural codec by Kyutai is bundled as part of the model.
April 2026 ONNX export by KevinAHM:
https://huggingface.co/KevinAHM/pocket-tts-onnx
Pinned revision: 58a6d00cf13d239b6748cb0769f35c580a8f606c
Bundled English VCTK presets: Anna (p228), Vera (p229), Fantine (p244),
Charles (p254), Paul (p259), Eponine (p262), Azelma (p303), George (p315),
Mary (p333), Jane (p339), Michael (p360), and Eve (p361). These exact,
ai-coustics-enhanced WAVs come from Kyutai's tts-voices repository at revision
323332d33f997de8394f24a193e1a76df720e01a.
Source: https://huggingface.co/kyutai/tts-voices/tree/323332d33f997de8394f24a193e1a76df720e01a/vctk
Original recordings: Voice Cloning Toolkit (VCTK) corpus,
https://datashare.ed.ac.uk/handle/10283/3443 (CC-BY-4.0).
Enhancement (denoise/dereverb): ai-coustics, https://ai-coustics.com/
Buzz ships the ONNX/model artifacts and voice WAVs unmodified, renamed only
by placement in the local model directory.
Provided \"AS IS\", without warranty of any kind, express or implied. See the
license text for full warranty disclaimer.
";
fn is_embedded_voice_file(filename: &str) -> bool {
POCKET_VOICES
.iter()
.any(|voice| voice.bytes.is_some() && voice.reference_file == filename)
}
/// Add the official VCTK presets to an otherwise-ready v4 install.
///
/// Model artifacts and Mary already exist in v4. The manifest is written last,
/// so interruption leaves v4 intact and the next launch retries.
pub(super) fn install_vctk_presets_into_v4_model(models_dir: &Path) -> Result<(), String> {
let model_dir = models_dir.join(TTS_MODEL_DIR_NAME);
let manifest_path = model_dir.join(MANIFEST_FILENAME);
let version = match std::fs::read_to_string(&manifest_path) {
Ok(version) => version,
Err(_) => return Ok(()),
};
if version.trim() != PRESET_VOICE_TTS_MODEL_VERSION {
return Ok(());
}
if !TTS_EXPECTED_FILES
.iter()
.filter(|filename| !is_embedded_voice_file(filename))
.all(|filename| model_dir.join(filename).is_file())
{
return Ok(());
}
for voice in POCKET_VOICES {
let Some(bytes) = voice.bytes else {
continue;
};
std::fs::write(model_dir.join(voice.reference_file), bytes)
.map_err(|error| format!("write bundled {} voice: {error}", voice.display_name))?;
}
let retired_marius = model_dir.join("marius.wav");
if retired_marius.is_file() {
std::fs::remove_file(retired_marius)
.map_err(|error| format!("remove retired Marius voice: {error}"))?;
}
std::fs::write(model_dir.join(TTS_LICENSE_FILE_NAME), TTS_LICENSE_TEXT)
.map_err(|error| format!("update Pocket voice notice: {error}"))?;
std::fs::write(manifest_path, TTS_MODEL_VERSION)
.map_err(|error| format!("update Pocket model manifest: {error}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn v4_install_adds_presets_without_redownloading_models() {
let temp = tempfile::tempdir().expect("tempdir");
let model_dir = temp.path().join(TTS_MODEL_DIR_NAME);
std::fs::create_dir_all(&model_dir).expect("create model dir");
for file in TTS_EXPECTED_FILES
.iter()
.filter(|filename| !is_embedded_voice_file(filename))
{
std::fs::write(model_dir.join(file), b"existing").expect("write prior file");
}
std::fs::write(
model_dir.join(MANIFEST_FILENAME),
PRESET_VOICE_TTS_MODEL_VERSION,
)
.expect("write prior manifest");
std::fs::write(model_dir.join("marius.wav"), b"retired").expect("write retired voice");
install_vctk_presets_into_v4_model(temp.path()).expect("in-place upgrade");
for voice in POCKET_VOICES {
if let Some(bytes) = voice.bytes {
assert_eq!(
std::fs::read(model_dir.join(voice.reference_file))
.expect("bundled voice installed"),
bytes
);
}
}
assert_eq!(
std::fs::read_to_string(model_dir.join(MANIFEST_FILENAME)).expect("updated manifest"),
TTS_MODEL_VERSION
);
assert!(!model_dir.join("marius.wav").exists());
assert!(
ModelSlot::new(TTS_MODEL_DIR_NAME, TTS_EXPECTED_FILES, TTS_MODEL_VERSION)
.is_ready(temp.path())
);
}
}
+246 -20
View File
@@ -3,9 +3,12 @@
//! Handles starting, hot-starting, and spawning transcription tasks for
//! the voice pipelines. Extracted from mod.rs to keep the command layer thin.
use std::sync::{
atomic::{AtomicU64, Ordering},
Arc, Mutex,
use std::{
sync::{
atomic::{AtomicU64, Ordering},
Arc, Mutex,
},
time::Duration,
};
use nostr::JsonUtil;
@@ -17,7 +20,7 @@ use crate::events;
use super::models;
use super::relay_api::{self, fetch_channel_members, parse_channel_uuid};
use super::state::{HuddlePhase, VoiceInputMode};
use super::state::{HuddlePhase, HuddleState, VoiceInputMode};
use super::stt;
use super::tts;
@@ -392,7 +395,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
// Atomically check preconditions and claim the construction slot.
// The sentinel prevents a second caller from starting construction
// while we're building outside the lock.
let (tts_active, tts_cancel) = {
let (tts_active, tts_cancel, tts_starting) = {
let hs = state.huddle()?;
if hs.tts_pipeline.is_some() {
return Ok(false);
@@ -403,18 +406,39 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
if hs.tts_starting.swap(true, Ordering::AcqRel) {
return Ok(false); // Another caller is already constructing.
}
(Arc::clone(&hs.tts_active), Arc::clone(&hs.tts_cancel))
(
Arc::clone(&hs.tts_active),
Arc::clone(&hs.tts_cancel),
Arc::clone(&hs.tts_starting),
)
};
let _starting_guard = TtsStartingGuard(tts_starting);
// Construct outside the lock — this spawns the TTS worker thread and
// loads ONNX sessions (~200ms). If this fails, clear the sentinel.
let output_device = state
.audio_output_device
.huddle_audio
.output_device
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
let initial_voice = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| {
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
})?;
let constructed_voice = initial_voice.clone();
let constructed = tokio::task::spawn_blocking(move || {
tts::TtsPipeline::new(model_dir, tts_active, tts_cancel, output_device)
tts::TtsPipeline::new_with_voice(
model_dir,
tts_active,
tts_cancel,
&initial_voice,
output_device,
)
})
.await;
let pipeline = match constructed {
@@ -431,23 +455,80 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
}
};
{
let mut hs = state.huddle()?;
hs.tts_starting.store(false, Ordering::Release);
// Phase check: huddle may have been torn down during construction.
if !matches!(hs.phase, HuddlePhase::Connected | HuddlePhase::Active) {
return Ok(false);
finalize_tts_pipeline_start(state, move |voice, huddle| {
if should_reselect_constructed_voice(&constructed_voice, voice) {
pipeline.select_voice_before_publish(voice);
}
// Final check: another path may have created a pipeline while we were constructing.
if hs.tts_pipeline.is_some() {
return Ok(false);
}
hs.tts_pipeline = Some(pipeline);
}
huddle.tts_pipeline = Some(pipeline);
})
}
/// Wait for a concurrent TTS constructor to publish or fail.
///
/// `maybe_start_tts_pipeline` deliberately lets only one caller construct the
/// engine. A live message that loses that race must wait for the owner instead
/// of observing the temporary empty slot and being dropped.
pub(crate) async fn await_inflight_tts_start(state: &AppState) -> Result<(), String> {
let starting = {
let huddle = state.huddle()?;
Arc::clone(&huddle.tts_starting)
};
tokio::time::timeout(Duration::from_secs(15), async {
while starting.load(Ordering::Acquire) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.map_err(|_| "TTS pipeline startup did not finish before timeout".to_string())?;
// The owner clears the sentinel while holding the huddle lock, before it
// publishes. Reacquiring that lock ensures publication is visible before
// the losing caller looks up the sender.
drop(state.huddle()?);
Ok(())
}
struct TtsStartingGuard(Arc<std::sync::atomic::AtomicBool>);
impl Drop for TtsStartingGuard {
fn drop(&mut self) {
self.0.store(false, Ordering::Release);
}
}
/// Publish a constructed TTS pipeline against the latest settings.
///
/// Construction happens outside locks and can overlap a voice change or OFF
/// transition. Holding the huddle lock while re-reading settings gives either
/// transition a safe ordering: it updates the installed pipeline afterward,
/// or this finalizer observes the new setting before publishing.
fn finalize_tts_pipeline_start(
state: &AppState,
publish: impl FnOnce(&str, &mut HuddleState),
) -> Result<bool, String> {
let mut huddle = state.huddle()?;
huddle.tts_starting.store(false, Ordering::Release);
if !huddle.tts_enabled
|| !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active)
|| huddle.tts_pipeline.is_some()
{
return Ok(false);
}
let voice = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| {
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
})?;
publish(&voice, &mut huddle);
Ok(true)
}
fn should_reselect_constructed_voice(constructed_voice: &str, latest_voice: &str) -> bool {
constructed_voice != latest_voice
}
/// Sign an STT transcript event and produce the guarded POST body.
///
/// Factored out of the transcription loop so egress boundary 5 (huddle STT)
@@ -570,3 +651,148 @@ pub(crate) fn spawn_transcription_task(
}
});
}
#[cfg(test)]
mod tts_start_race_tests {
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc, Barrier, Mutex,
};
use std::time::Duration;
use crate::app_state::build_app_state;
use super::{
await_inflight_tts_start, finalize_tts_pipeline_start, should_reselect_constructed_voice,
HuddlePhase,
};
#[tokio::test]
async fn a_losing_starter_observes_publication_before_resuming() {
let state = Arc::new(build_app_state());
{
let mut huddle = state.huddle().expect("huddle state");
huddle.phase = HuddlePhase::Active;
huddle.tts_enabled = true;
huddle.tts_starting.store(true, Ordering::Release);
}
let published = Arc::new(AtomicBool::new(false));
let owner_state = Arc::clone(&state);
let owner_published = Arc::clone(&published);
let owner = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(20));
finalize_tts_pipeline_start(&owner_state, |_, _| {
owner_published.store(true, Ordering::Release);
})
});
await_inflight_tts_start(&state)
.await
.expect("wait for pipeline owner");
assert!(published.load(Ordering::Acquire));
assert!(owner.join().expect("pipeline owner").expect("finalize"));
}
#[test]
fn constructor_fallback_survives_unchanged_preference_at_publication() {
let selected_voice = Mutex::new(super::super::pocket::DEFAULT_VOICE.to_string());
let constructed_voice = "eve";
let latest_voice = "eve";
if should_reselect_constructed_voice(constructed_voice, latest_voice) {
*selected_voice.lock().expect("selected voice") = latest_voice.to_string();
}
assert_eq!(
selected_voice.lock().expect("selected voice").as_str(),
super::super::pocket::DEFAULT_VOICE
);
}
#[test]
fn construction_reconciles_a_voice_selected_while_starting() {
let state = Arc::new(build_app_state());
{
let mut huddle = state.huddle().expect("huddle state");
huddle.phase = HuddlePhase::Active;
huddle.tts_enabled = true;
huddle.tts_starting.store(true, Ordering::Release);
}
let constructed = Arc::new(Barrier::new(2));
let publish = Arc::new(Barrier::new(2));
let selected_voice = Arc::new(Mutex::new(None));
let worker_state = Arc::clone(&state);
let worker_constructed = Arc::clone(&constructed);
let worker_publish = Arc::clone(&publish);
let worker_voice = Arc::clone(&selected_voice);
let worker = std::thread::spawn(move || {
worker_constructed.wait();
worker_publish.wait();
finalize_tts_pipeline_start(&worker_state, |voice, _| {
*worker_voice.lock().expect("selected voice") = Some(voice.to_string());
})
});
constructed.wait();
assert!(state
.huddle()
.expect("huddle state")
.tts_starting
.load(Ordering::Acquire));
state
.huddle_audio
.tts
.lock()
.expect("text-to-speech settings")
.voice_preferences = vec!["pocket:eve".to_string()];
publish.wait();
assert!(worker.join().expect("starter thread").expect("finalize"));
assert_eq!(
*selected_voice.lock().expect("selected voice"),
Some("eve".to_string())
);
}
#[test]
fn construction_is_discarded_when_disabled_while_starting() {
let state = Arc::new(build_app_state());
{
let mut huddle = state.huddle().expect("huddle state");
huddle.phase = HuddlePhase::Active;
huddle.tts_enabled = true;
huddle.tts_starting.store(true, Ordering::Release);
}
let constructed = Arc::new(Barrier::new(2));
let publish = Arc::new(Barrier::new(2));
let did_publish = Arc::new(Mutex::new(false));
let worker_state = Arc::clone(&state);
let worker_constructed = Arc::clone(&constructed);
let worker_publish = Arc::clone(&publish);
let worker_did_publish = Arc::clone(&did_publish);
let worker = std::thread::spawn(move || {
worker_constructed.wait();
worker_publish.wait();
finalize_tts_pipeline_start(&worker_state, |_, _| {
*worker_did_publish.lock().expect("publish flag") = true;
})
});
constructed.wait();
{
let mut huddle = state.huddle().expect("huddle state");
huddle.tts_enabled = false;
}
publish.wait();
assert!(!worker.join().expect("starter thread").expect("finalize"));
assert!(!*did_publish.lock().expect("publish flag"));
assert!(!state
.huddle()
.expect("huddle state")
.tts_starting
.load(Ordering::Acquire));
}
}
+2 -164
View File
@@ -1,166 +1,4 @@
//! April 2026 Pocket TTS engine for Buzz Desktop.
//!
//! The `english_2026-04` bundle uses SentencePiece tokenization, a learned
//! voice BOS embedding, recurrent FlowLM state, and stateful Mimi decoding.
//! Buzz selects the upstream three-graph INT8 variant while retaining the
//! full-precision Mimi encoder and text conditioner specified by that variant.
//!
//! ## Attribution
//!
//! - Pocket TTS and Mimi: Kyutai, CC-BY-4.0.
//! - ONNX export: KevinAHM/pocket-tts-onnx, CC-BY-4.0.
//! - Reference voice: Kyutai's Mary preset (VCTK p333), CC-BY-4.0.
//!
//! `huddle::models` writes the complete attribution beside the cached bytes.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use sherpa_onnx::Wave;
#[path = "pocket_april.rs"]
mod pocket_april;
#[path = "pocket_models.rs"]
mod pocket_models;
use pocket_april::{prepare_april_prompt, AprilPocketTts};
pub(crate) use pocket_models::{
pub use buzz_voice_pkg::pocket::*;
pub(crate) use buzz_voice_pkg::{
april_model_info, PocketModelArtifact, APRIL_BUNDLE_ID, APRIL_MODEL_ID, APRIL_MODEL_REVISION,
};
/// Pocket TTS emits 24 kHz mono PCM.
pub const SAMPLE_RATE: u32 = 24_000;
/// Bundled reference voice name without its extension.
pub const DEFAULT_VOICE: &str = "reference_sample";
/// Pocket voice files are reference WAVs.
pub const VOICE_FILE_EXT: &str = "wav";
const TTS_NUM_THREADS: usize = 1;
/// Loaded reference voice samples and their original sample rate.
#[derive(Debug, Clone)]
pub struct VoiceStyle {
samples: Vec<f32>,
sample_rate: i32,
}
/// Load a Pocket reference voice WAV from disk.
pub fn load_voice_style(path: &Path) -> Result<VoiceStyle, String> {
let path_str = path
.to_str()
.ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?;
let wave = Wave::read(path_str)
.ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?;
let samples = wave.samples().to_vec();
if samples.is_empty() {
return Err(format!("voice WAV is empty: {}", path.display()));
}
Ok(VoiceStyle {
samples,
sample_rate: wave.sample_rate(),
})
}
/// Resident April INT8 Pocket TTS engine.
pub struct PocketTts {
inner: Mutex<AprilPocketTts>,
}
/// Load Buzz Desktop's pinned April INT8 model.
pub fn load_text_to_speech(model_dir: &str) -> Result<PocketTts, String> {
let dir = PathBuf::from(model_dir);
for artifact in april_model_info().artifacts {
let path = dir.join(artifact.filename);
if !path.is_file() {
return Err(format!(
"incomplete Pocket TTS {} INT8 bundle: missing {}",
APRIL_BUNDLE_ID,
path.display()
));
}
}
Ok(PocketTts {
inner: Mutex::new(AprilPocketTts::load(&dir, TTS_NUM_THREADS)?),
})
}
impl PocketTts {
/// Split text into synthesis units that satisfy the bundle's exact
/// 50-token input limit.
pub fn split_text_into_chunks(&self, text: &str) -> Result<Vec<String>, String> {
let Some(prepared) = prepare_april_prompt(text) else {
return Ok(Vec::new());
};
self.inner
.lock()
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?
.split_prompt(&prepared)
}
/// Synthesize text with the supplied reference voice.
///
/// Pocket detects language from text and this model uses one synthesis
/// step, so `_lang` and `_steps` intentionally do not affect output.
pub fn synth_chunk(
&self,
text: &str,
_lang: &str,
style: &VoiceStyle,
_steps: usize,
) -> Result<Vec<f32>, String> {
let Some(prepared) = prepare_april_prompt(text) else {
return Ok(Vec::new());
};
let mut engine = self
.inner
.lock()
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?;
let chunks = engine.split_prompt(&prepared)?;
let mut samples = Vec::new();
for chunk in chunks {
let prepared = prepare_april_prompt(&chunk)
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
samples.extend(engine.synth_chunk(&prepared, style)?);
}
Ok(samples)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn desktop_model_is_april_int8_only() {
let info = april_model_info();
assert_eq!(info.max_token_per_chunk, 50);
assert_eq!(info.sample_rate, SAMPLE_RATE);
assert!(info
.artifacts
.iter()
.any(|artifact| artifact.filename == "flow_lm_main_int8.onnx"));
assert!(!info
.artifacts
.iter()
.any(|artifact| artifact.filename == "flow_lm_main.onnx"));
}
#[test]
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
fn production_api_emits_non_silent_april_int8_pcm() {
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
.expect("load reference voice");
let samples = engine
.synth_chunk("Bright birds begin beside the bay.", "en", &style, 1)
.expect("synthesize through the production API");
assert!(!samples.is_empty());
assert!(samples.iter().all(|sample| sample.is_finite()));
assert!(samples.iter().any(|sample| sample.abs() > 1.0e-6));
}
}
+2 -1
View File
@@ -164,7 +164,8 @@ pub(crate) async fn connect_audio_relay(
let cancel_clone = cancel.clone();
let (pcm_tx, pcm_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(50);
let output_device_name = state
.audio_output_device
.huddle_audio
.output_device
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
+14
View File
@@ -288,9 +288,11 @@ impl HuddleState {
pub(crate) fn reset_preserving_generation(&mut self) {
let gen = Arc::clone(&self.session_generation);
let huddle_generation = self.huddle_generation;
let tts_enabled = self.tts_enabled;
*self = Self::default();
self.session_generation = gen;
self.huddle_generation = huddle_generation;
self.tts_enabled = tts_enabled;
}
}
@@ -430,6 +432,18 @@ mod tests {
assert!(state.owns_huddle_lifetime(replacement_generation, super::HuddlePhase::Connecting));
}
#[test]
fn teardown_preserves_installation_global_tts_preference() {
let mut state = HuddleState {
tts_enabled: false,
phase: super::HuddlePhase::Active,
..HuddleState::default()
};
state.reset_preserving_generation();
assert!(!state.tts_enabled);
assert_eq!(state.phase, super::HuddlePhase::Idle);
}
#[test]
fn stale_constructor_cannot_clear_replacement_sentinel() {
let mut state = HuddleState::default();
+392 -281
View File
@@ -35,10 +35,11 @@
//! can gate microphone input while the agent is speaking.
use std::{
collections::VecDeque,
num::NonZero,
path::PathBuf,
sync::{
atomic::{AtomicBool, Ordering},
atomic::{AtomicBool, AtomicU64, Ordering},
mpsc::{self, SyncSender},
Arc, Mutex, MutexGuard, PoisonError,
},
@@ -46,9 +47,21 @@ use std::{
time::Duration,
};
use super::pocket::{load_text_to_speech, load_voice_style, SAMPLE_RATE, VOICE_FILE_EXT};
use super::pocket::{
load_text_to_speech, load_voice_style, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT,
};
use super::preprocessing::{preprocess_for_tts, split_sentences};
#[path = "tts_voice_transition.rs"]
mod voice_transition;
use voice_transition::*;
#[path = "tts_startup.rs"]
mod startup;
use startup::await_worker_startup;
#[path = "tts_audio.rs"]
mod audio;
use audio::*;
// ── Constants ─────────────────────────────────────────────────────────────────
/// Maximum number of queued text items.
@@ -56,15 +69,15 @@ use super::preprocessing::{preprocess_for_tts, split_sentences};
/// TTS can play it. Excess items are dropped with a warning.
const TEXT_QUEUE_DEPTH: usize = 8;
/// How long the worker waits on the text channel before checking the shutdown flag.
/// How long the worker waits before checking the shutdown flag.
const RECV_TIMEOUT: Duration = Duration::from_millis(100);
/// Poll interval of the barge-in monitor thread. Bounds flag-to-silence
/// latency: a cancel is noticed within one tick, and rodio's internal
/// `periodic_access` wrapper stops the in-flight source within a further
/// ~5 ms — so playing audio dies ~15 ms after the flag is set, even while
/// the worker is blocked inside `synth_chunk`.
const MONITOR_TICK: Duration = Duration::from_millis(10);
const AUDIO_PRIME_TIMEOUT: Duration = Duration::from_secs(2);
/// Pocket TTS is a one-step consistency model, not diffusion. Kept for API compat.
const SYNTH_STEPS: usize = 1;
@@ -109,6 +122,8 @@ const MAX_CHUNK_CHARS: usize = 200;
/// Injected as a silent buffer between each synthesized sentence chunk.
const INTER_SENTENCE_SILENCE: f32 = 0.1;
type WorkerControlState = (Arc<AtomicBool>, Arc<AtomicBool>, WorkerCancelSignals);
// ── Public pipeline handle ────────────────────────────────────────────────────
/// Handle to the running TTS pipeline.
@@ -117,7 +132,7 @@ const INTER_SENTENCE_SILENCE: f32 = 0.1;
#[derive(Debug)]
pub struct TtsPipeline {
/// Send preprocessed text into the pipeline.
text_tx: SyncSender<String>,
text_tx: SyncSender<QueuedText>,
/// `true` while the agent is speaking. Shared with the STT pipeline for gating.
#[allow(dead_code)]
pub tts_active: Arc<AtomicBool>,
@@ -127,38 +142,25 @@ pub struct TtsPipeline {
/// Kept alive here so the Arc isn't dropped — the worker holds a clone.
#[allow(dead_code)]
cancel: Arc<AtomicBool>,
/// Voice name (e.g. "reference_sample"). Stored for future voice-switching support.
#[allow(dead_code)]
voice: String,
/// Internal cancellation used only for voice changes. Kept separate so a
/// concurrent human barge-in always clears every queued message.
voice_cancel: Arc<AtomicBool>,
/// Selected manifest voice. The worker reloads only the lightweight style
/// when this changes; the warmed Pocket engine and audio player stay alive.
voice: Arc<Mutex<String>>,
/// Tags messages so a voice change drops only pre-change queue entries.
voice_generation: Arc<AtomicU64>,
/// Completed after the worker drains pre-change text and installs the new style.
voice_change_ack: VoiceChangeAck,
/// Worker thread handle — taken on drop to join cleanly.
thread: Option<thread::JoinHandle<()>>,
}
impl TtsPipeline {
/// Spawn the TTS pipeline thread using the default voice.
/// Spawn the TTS pipeline thread with a manifest-backed voice name.
///
/// `model_dir` must contain the Pocket TTS files declared by `huddle::models`
/// (the five ONNX sessions, the two JSON tables, and `<voice>.wav`).
///
/// `tts_active` is set to `true` while audio is playing and `false` when idle.
/// Pass the same `Arc` to the STT pipeline to gate microphone input.
///
/// `cancel` is the shared barge-in flag from `HuddleState.tts_cancel`. Pass the
/// same `Arc` to the STT pipeline so both sides reference the same flag for the
/// entire huddle session — no stale references after pipeline restarts.
pub fn new(
model_dir: PathBuf,
tts_active: Arc<AtomicBool>,
cancel: Arc<AtomicBool>,
output_device: Option<String>,
) -> Result<Self, String> {
use super::pocket::DEFAULT_VOICE;
Self::new_with_voice(model_dir, tts_active, cancel, DEFAULT_VOICE, output_device)
}
/// Spawn the TTS pipeline thread with a specific voice name. Today only the
/// bundled default voice (see `pocket::DEFAULT_VOICE`) is shipped; other
/// names will surface a clear error from `load_voice_style`.
/// `cancel` is shared with STT for barge-in. The same handle survives voice
/// changes so the warmed Pocket engine is retained.
pub fn new_with_voice(
model_dir: PathBuf,
tts_active: Arc<AtomicBool>,
@@ -166,37 +168,56 @@ impl TtsPipeline {
voice: &str,
output_device: Option<String>,
) -> Result<Self, String> {
let (text_tx, text_rx) = mpsc::sync_channel::<String>(TEXT_QUEUE_DEPTH);
let (text_tx, text_rx) = mpsc::sync_channel::<QueuedText>(TEXT_QUEUE_DEPTH);
let shutdown = Arc::new(AtomicBool::new(false));
// cancel is passed in from HuddleState.tts_cancel — shared with STT for barge-in.
let shutdown_worker = Arc::clone(&shutdown);
let cancel_worker = Arc::clone(&cancel);
let voice_cancel = Arc::new(AtomicBool::new(false));
let worker_voice_cancel = Arc::clone(&voice_cancel);
let tts_active_worker = Arc::clone(&tts_active);
let voice_name = voice.to_string();
let voice = Arc::new(Mutex::new(voice.to_string()));
let voice_worker = Arc::clone(&voice);
let voice_generation = Arc::new(AtomicU64::new(1));
let worker_voice_generation = Arc::clone(&voice_generation);
let voice_change_ack = Arc::new(Mutex::new(None));
let worker_voice_change_ack = Arc::clone(&voice_change_ack);
let model_dir_worker = model_dir.clone();
let (startup_tx, startup_rx) = mpsc::sync_channel(1);
let handle = thread::Builder::new()
.name("tts-worker".into())
.spawn(move || {
tts_worker(
model_dir_worker,
voice_name,
(
voice_worker,
worker_voice_generation,
worker_voice_change_ack,
),
text_rx,
tts_active_worker,
shutdown_worker,
cancel_worker,
(
tts_active_worker,
shutdown_worker,
(cancel_worker, worker_voice_cancel),
),
output_device,
startup_tx,
)
})
.map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?;
let handle = await_worker_startup(handle, startup_rx)?;
Ok(Self {
text_tx,
tts_active,
shutdown,
cancel,
voice: voice.to_string(),
voice_cancel,
voice,
voice_generation,
voice_change_ack,
thread: Some(handle),
})
}
@@ -206,14 +227,59 @@ impl TtsPipeline {
/// Non-blocking. Returns `Err` if the queue is full (bounded at
/// `TEXT_QUEUE_DEPTH`) — caller may log and discard.
pub fn speak(&self, text: String) -> Result<(), String> {
self.text_tx.try_send(text).map_err(|e| {
eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}");
format!("TTS queue full, dropping: {e}")
})
self.text_tx
.try_send(QueuedText {
generation: self.voice_generation.load(Ordering::Acquire),
route_id: 0,
text,
})
.map_err(|e| {
eprintln!("buzz-desktop: TTS queue saturated, dropping message: {e}");
format!("TTS queue full, dropping: {e}")
})
}
/// Clone the bounded queue sender so callers can apply backpressure without
/// holding the huddle mutex. Disabling TTS drops the receiver and unblocks
/// any waiting sender while the shared cancellation flag stops playback.
pub(crate) fn text_sender(&self) -> TtsTextSender {
TtsTextSender {
text_tx: self.text_tx.clone(),
generation: self.voice_generation.load(Ordering::Acquire),
}
}
/// Select a bundled Pocket voice for subsequent speech.
///
/// Current playback and queued text are cancelled immediately so content
/// cannot continue in the old voice. The worker keeps its warmed inference
/// engine and reloads only the reference style before the next utterance.
pub fn select_voice(&self, voice: &str) -> Option<tokio::sync::oneshot::Receiver<()>> {
let acknowledged = begin_voice_change(
&self.voice,
&self.voice_generation,
&self.voice_cancel,
&self.voice_change_ack,
voice,
);
if acknowledged.is_some() {
eprintln!("buzz-desktop: tts stage=cancellation reason=voice_switch route_id=0");
}
acknowledged
}
/// Reconcile the voice of a pipeline that has not been published yet.
///
/// No caller can enqueue text before publication, so raising the shared
/// cancellation flag here would create a race that could discard the first
/// message queued immediately after installation.
pub(crate) fn select_voice_before_publish(&self, voice: &str) {
*self.voice.lock().unwrap_or_else(|error| error.into_inner()) = voice.to_string();
}
/// Signal the worker thread to stop.
pub fn shutdown(&self) {
eprintln!("buzz-desktop: tts stage=cancellation reason=shutdown route_id=0");
self.shutdown.store(true, Ordering::Release);
}
@@ -239,40 +305,52 @@ impl Drop for TtsPipeline {
fn tts_worker(
model_dir: PathBuf,
voice_name: String,
text_rx: mpsc::Receiver<String>,
tts_active: Arc<AtomicBool>,
shutdown: Arc<AtomicBool>,
cancel: Arc<AtomicBool>,
voice_state: WorkerVoiceState,
text_rx: mpsc::Receiver<QueuedText>,
control_state: WorkerControlState,
output_device: Option<String>,
startup_tx: mpsc::SyncSender<Result<(), String>>,
) {
let (selected_voice, voice_generation, voice_change_ack) = voice_state;
let (tts_active, shutdown, cancel_signals) = control_state;
let (cancel, voice_cancel) = cancel_signals;
// ── 1. Initialise TTS engine ──────────────────────────────────────────────
let model_dir_str = model_dir.to_string_lossy().to_string();
let engine = match load_text_to_speech(&model_dir_str) {
Ok(e) => e,
Err(e) => {
eprintln!(
"buzz-desktop: TTS engine init failed (model_dir={}): {e}. TTS disabled.",
model_dir.display()
);
drain_until_shutdown(text_rx, &shutdown);
let error = format!("TTS engine initialization failed: {e}");
eprintln!("buzz-desktop: tts stage=startup status=failed reason=engine_load");
let _ = startup_tx.send(Err(error));
return;
}
};
// ── 2. Load voice style ───────────────────────────────────────────────────
let voice_path = model_dir.join(format!("{voice_name}.{VOICE_FILE_EXT}"));
let style = match load_voice_style(&voice_path) {
let requested_voice = selected_voice
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
let mut voice_name = DEFAULT_VOICE.to_string();
let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"));
let mut style = match load_voice_style(&fallback_path) {
Ok(s) => s,
Err(e) => {
eprintln!(
"buzz-desktop: TTS voice style load failed ({voice_name}): {e}. TTS disabled."
);
drain_until_shutdown(text_rx, &shutdown);
let error = format!("TTS voice style initialization failed: {e}");
eprintln!("buzz-desktop: tts stage=startup status=failed reason=fallback_voice_style");
let _ = startup_tx.send(Err(error));
return;
}
};
if requested_voice != DEFAULT_VOICE
&& !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style)
{
let _ = startup_tx.send(Err(
"TTS selected voice and Mary fallback are unavailable".to_string()
));
return;
}
// ── 2b. Warmup inference ─────────────────────────────────────────────────
// The first ONNX inference on any session is significantly slower than
@@ -280,15 +358,10 @@ fn tts_worker(
// pool allocation, and graph-specific caches. Run a short dummy synthesis
// and discard the output so the first real utterance runs at warm-session speed.
{
let t = std::time::Instant::now();
match engine.synth_chunk("warmup", "en", &style, SYNTH_STEPS) {
Ok(_) => eprintln!(
"buzz-desktop: TTS warmup completed in {:.0}ms",
t.elapsed().as_millis()
),
Err(e) => eprintln!(
"buzz-desktop: TTS warmup failed after {:.0}ms: {e} — first utterance may be slow",
t.elapsed().as_millis()
Ok(_) => eprintln!("buzz-desktop: tts stage=warmup status=ready"),
Err(_) => eprintln!(
"buzz-desktop: tts stage=warmup status=failed reason=inference first_utterance_may_be_slow=true"
),
}
}
@@ -301,8 +374,9 @@ fn tts_worker(
{
Ok(h) => h,
Err(e) => {
eprintln!("buzz-desktop: TTS audio output failed: {e}. TTS disabled.");
drain_until_shutdown(text_rx, &shutdown);
let error = format!("TTS audio output initialization failed: {e}");
eprintln!("buzz-desktop: tts stage=startup status=failed reason=output_open");
let _ = startup_tx.send(Err(error));
return;
}
};
@@ -310,14 +384,14 @@ fn tts_worker(
let channels = match NonZero::new(1u16) {
Some(c) => c,
None => {
eprintln!("buzz-desktop: TTS channel count invariant violated");
let _ = startup_tx.send(Err("TTS channel count invariant violated".to_string()));
return;
}
};
let rate = match NonZero::new(SAMPLE_RATE) {
Some(r) => r,
None => {
eprintln!("buzz-desktop: TTS sample rate invariant violated");
let _ = startup_tx.send(Err("TTS sample rate invariant violated".to_string()));
return;
}
};
@@ -341,10 +415,22 @@ fn tts_worker(
player.append(SamplesBuffer::new(channels, rate, silence));
// Wait for the silent buffer to drain — this ensures the output stream
// is fully initialized before the first real utterance.
let deadline = std::time::Instant::now() + AUDIO_PRIME_TIMEOUT;
while !player.empty() {
if std::time::Instant::now() >= deadline {
eprintln!("buzz-desktop: tts stage=startup status=failed reason=output_prime");
let _ = startup_tx.send(Err(
"TTS audio output did not become ready before timeout".to_string(),
));
return;
}
thread::sleep(Duration::from_millis(10));
}
}
if startup_tx.send(Ok(())).is_err() {
return;
}
eprintln!("buzz-desktop: tts stage=startup status=ready");
// ── 3b. Barge-in monitor thread ───────────────────────────────────────────
//
@@ -372,6 +458,7 @@ fn tts_worker(
let monitor = {
let player = Arc::clone(&player);
let cancel = Arc::clone(&cancel);
let voice_cancel = Arc::clone(&voice_cancel);
let tts_active = Arc::clone(&tts_active);
let stop = Arc::clone(&monitor_stop);
let player_ops = Arc::clone(&player_ops);
@@ -379,12 +466,12 @@ fn tts_worker(
.name("tts-barge-in-monitor".into())
.spawn(move || {
while !stop.load(Ordering::Acquire) {
if cancel.load(Ordering::Acquire) {
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
let _ops = lock_player_ops(&player_ops);
// Re-check under the lock: the worker may have
// consumed this cancel (and appended fresh audio)
// between the load above and the lock acquisition.
if cancel.load(Ordering::Acquire) {
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
// clear() pauses the persistent player; play()
// un-pauses (see handle_cancel_or_shutdown).
// Idempotent — safe to repeat every tick until
@@ -418,13 +505,45 @@ fn tts_worker(
// idle branch below uses it to decide when to drop `tts_active` and to
// arm a fresh lead-in cushion for the next utterance.
let mut first_append = true;
let mut last_route_id = 0;
let mut deferred_text = VecDeque::new();
let append_audio = |prepared: PreparedModelAudio, route_id: u64| {
let _ops = lock_player_ops(&player_ops);
if cancel.load(Ordering::Acquire)
|| voice_cancel.load(Ordering::Acquire)
|| shutdown.load(Ordering::Acquire)
{
let reason = if shutdown.load(Ordering::Acquire) {
"shutdown"
} else if cancel.load(Ordering::Acquire) {
"barge_in"
} else {
"voice_switch"
};
eprintln!(
"buzz-desktop: tts stage=synthesis status=cancelled reason={reason} route_id={route_id}"
);
return false;
}
player.append(SamplesBuffer::new(channels, rate, prepared.buffer));
eprintln!(
"buzz-desktop: tts stage=player status=append_accepted route_id={route_id} chunk_index={} sample_count={}",
prepared.chunk_index, prepared.sample_count
);
// Set this only after append so STT remains open during synthesis.
tts_active.store(true, Ordering::Release);
true
};
loop {
let mut no_current_text = None;
if handle_cancel_or_shutdown(
&cancel,
(&cancel, &voice_cancel),
&shutdown,
&tts_active,
&text_rx,
(&text_rx, &mut deferred_text, &mut no_current_text),
&voice_change_ack,
None,
Some((&player, &player_ops)),
) {
if shutdown.load(Ordering::Acquire) {
@@ -436,28 +555,47 @@ fn tts_worker(
continue;
}
let raw_text = match text_rx.recv_timeout(RECV_TIMEOUT) {
Ok(t) => t,
Err(mpsc::RecvTimeoutError::Timeout) => {
// Nothing queued. If playback has also finished, the agent
// has gone quiet — release the mic gate and reset the
// lead-in so the next utterance gets a fresh cushion.
if player.empty() && !first_append {
tts_active.store(false, Ordering::Release);
first_append = true;
// Voice changes cancel the old utterance/queue and are observed here,
// before receiving subsequent text. A bad bundled asset falls back to
// Mary without discarding the already-warmed Pocket engine.
let voice_ready =
reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style);
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
if !voice_ready {
continue;
}
let mut queued_text = Some(match deferred_text.pop_front() {
Some(text) => text,
None => match text_rx.recv_timeout(RECV_TIMEOUT) {
Ok(text) => text,
Err(mpsc::RecvTimeoutError::Timeout) => {
// Nothing queued. If playback has also finished, the agent
// has gone quiet — release the mic gate and reset the
// lead-in so the next utterance gets a fresh cushion.
if player.empty() && !first_append {
tts_active.store(false, Ordering::Release);
eprintln!(
"buzz-desktop: tts stage=player status=drained route_id={last_route_id}"
);
first_append = true;
}
continue;
}
continue;
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
};
Err(mpsc::RecvTimeoutError::Disconnected) => break,
},
});
// Check cancel again after unblocking — a cancel may have arrived
// while we were waiting.
let pending_route_id = queued_text.as_ref().map(|queued| queued.route_id);
if handle_cancel_or_shutdown(
&cancel,
(&cancel, &voice_cancel),
&shutdown,
&tts_active,
&text_rx,
(&text_rx, &mut deferred_text, &mut queued_text),
&voice_change_ack,
pending_route_id,
Some((&player, &player_ops)),
) {
if shutdown.load(Ordering::Acquire) {
@@ -466,6 +604,30 @@ fn tts_worker(
first_append = true;
continue;
}
let Some(queued_text) = queued_text else {
continue;
};
if queued_text.generation < voice_generation.load(Ordering::Acquire) {
eprintln!(
"buzz-desktop: tts stage=queue status=dropped reason=voice_switch route_id={}",
queued_text.route_id
);
continue;
}
let raw_text = queued_text.text;
let route_id = queued_text.route_id;
eprintln!("buzz-desktop: tts stage=synthesis status=started route_id={route_id}");
// The selected voice can change while this worker is blocked in
// recv_timeout. Reconcile again after receipt so the first message
// queued after an unpublished pipeline is installed cannot use the
// voice captured when construction began.
if !reconcile_selected_voice(&model_dir, &selected_voice, &mut voice_name, &mut style) {
eprintln!(
"buzz-desktop: tts stage=synthesis status=failed reason=voice_unavailable route_id={route_id}"
);
continue;
}
// If playback already drained while we were waiting for this item,
// the agent is silent — release the mic gate BEFORE preprocessing/
@@ -477,12 +639,16 @@ fn tts_worker(
// stays set across items.)
if player.empty() && !first_append {
tts_active.store(false, Ordering::Release);
eprintln!("buzz-desktop: tts stage=player status=drained route_id={last_route_id}");
first_append = true;
}
// Preprocess text.
let text = preprocess_for_tts(&raw_text);
if text.is_empty() {
eprintln!(
"buzz-desktop: tts stage=synthesis status=empty reason=preprocess route_id={route_id}"
);
continue;
}
@@ -497,16 +663,29 @@ fn tts_worker(
.filter(|s| !s.trim().is_empty())
.collect();
let chunks = group_sentences_into_chunks(&sentences, MAX_CHUNK_CHARS);
if chunks.is_empty() {
eprintln!(
"buzz-desktop: tts stage=synthesis status=empty reason=no_chunks route_id={route_id}"
);
continue;
}
let mut synthesis_outcome = "completed";
let mut appended_audio = false;
let mut model_unit_index = 0_usize;
'playback_chunks: for chunk in &chunks {
let mut no_current_text = None;
if handle_cancel_or_shutdown(
&cancel,
(&cancel, &voice_cancel),
&shutdown,
&tts_active,
&text_rx,
(&text_rx, &mut deferred_text, &mut no_current_text),
&voice_change_ack,
Some(route_id),
Some((&player, &player_ops)),
) {
first_append = true;
synthesis_outcome = "cancelled";
break;
}
@@ -517,76 +696,111 @@ fn tts_worker(
let model_chunks = match engine.split_text_into_chunks(text) {
Ok(model_chunks) => model_chunks,
Err(error) => {
eprintln!("buzz-desktop: TTS chunking failed: {error}");
break;
Err(_) => {
eprintln!(
"buzz-desktop: tts stage=synthesis status=failed reason=chunking route_id={route_id}"
);
synthesis_outcome = "failed";
break 'playback_chunks;
}
};
let model_chunk_count = model_chunks.len();
for (model_chunk_index, model_chunk) in model_chunks.iter().enumerate() {
if model_chunks.is_empty() {
eprintln!(
"buzz-desktop: tts stage=synthesis status=empty reason=no_chunks route_id={route_id}"
);
continue;
}
let mut playback_audio = PlaybackChunkAudio::new();
for model_chunk in &model_chunks {
let chunk_index = model_unit_index;
model_unit_index += 1;
let mut no_current_text = None;
if handle_cancel_or_shutdown(
&cancel,
(&cancel, &voice_cancel),
&shutdown,
&tts_active,
&text_rx,
(&text_rx, &mut deferred_text, &mut no_current_text),
&voice_change_ack,
Some(route_id),
Some((&player, &player_ops)),
) {
first_append = true;
synthesis_outcome = "cancelled";
break 'playback_chunks;
}
let ends_playback_chunk = model_chunk_index + 1 == model_chunk_count;
match engine.synth_chunk(model_chunk, "en", &style, SYNTH_STEPS) {
let synthesis = engine.synth_chunk(model_chunk, "en", &style, SYNTH_STEPS);
if cancel.load(Ordering::Acquire)
|| voice_cancel.load(Ordering::Acquire)
|| shutdown.load(Ordering::Acquire)
{
let reason = if shutdown.load(Ordering::Acquire) {
"shutdown"
} else if cancel.load(Ordering::Acquire) {
"barge_in"
} else {
"voice_switch"
};
eprintln!(
"buzz-desktop: tts stage=synthesis status=cancelled reason={reason} route_id={route_id}"
);
// The monitor already stopped any queued playback. Discard
// synthesis that completed after cancellation so stale audio
// never reaches the player, while keeping buzz-voice's
// extracted April engine API unchanged.
first_append = true;
synthesis_outcome = "cancelled";
break 'playback_chunks;
}
match synthesis {
Ok(samples) if !samples.is_empty() => {
let mut audio = clamp_to_full_scale(samples);
if ends_playback_chunk {
// Fade only at the playback-chunk boundary. Applying
// it at the model's internal token boundary would
// create an audible dip between contiguous units.
apply_fade_out(&mut audio);
}
let buf = build_sentence_append_buffer(
if let Some(prepared) = playback_audio.push(
samples,
chunk_index,
&mut first_append,
audio,
silence_buf_len,
model_chunk_index == 0 || player.empty(),
ends_playback_chunk,
);
// Check-and-append under `player_ops`, serialized with
// the monitor: a barge-in may have arrived during
// synthesis (the blocking window the monitor thread
// exists for). Don't append the now-stale sentence — the
// human interrupted; speaking it anyway would talk over
// them. Holding the lock for the check + append means the
// monitor can never clear between our check passing and
// the buffer landing. The flag is deliberately NOT
// consumed here: the loop-top handle_cancel_or_shutdown
// does the full consume (drain queue, reset lead-in) on
// the next iteration.
let _ops = lock_player_ops(&player_ops);
if cancel.load(Ordering::Acquire) {
// Nothing appended; the loop-top consume re-arms
// `first_append` (the flag is still set — the worker
// is its only consumer).
break;
player.empty(),
) {
if !append_audio(prepared, route_id) {
first_append = true;
synthesis_outcome = "cancelled";
break 'playback_chunks;
}
appended_audio = true;
last_route_id = route_id;
}
player.append(SamplesBuffer::new(channels, rate, buf));
// NOTE: tts_active is set AFTER player.append(), not
// before. Setting it before synthesis would cause STT to
// discard user speech during the synthesis window as
// "echo" even though no audio is actually playing yet.
// See crossfire review C3.
tts_active.store(true, Ordering::Release);
}
Ok(_) => {}
Err(e) => {
eprintln!("buzz-desktop: TTS synth failed: {e}");
Ok(_) => {
eprintln!(
"buzz-desktop: tts stage=synthesis status=empty route_id={route_id} chunk_index={chunk_index}"
);
}
Err(_) => {
eprintln!(
"buzz-desktop: tts stage=synthesis status=failed reason=inference route_id={route_id} chunk_index={chunk_index}"
);
synthesis_outcome = "failed";
break;
}
}
}
if let Some(prepared) =
playback_audio.finish(&mut first_append, silence_buf_len, player.empty())
{
if !append_audio(prepared, route_id) {
first_append = true;
synthesis_outcome = "cancelled";
break 'playback_chunks;
}
appended_audio = true;
last_route_id = route_id;
}
if synthesis_outcome == "failed" {
break 'playback_chunks;
}
}
if synthesis_outcome == "completed" && appended_audio {
eprintln!("buzz-desktop: tts stage=synthesis status=completed route_id={route_id}");
}
if shutdown.load(Ordering::Acquire) {
@@ -601,6 +815,7 @@ fn tts_worker(
let _ = handle.join();
}
finish_voice_change_ack(&voice_change_ack);
tts_active.store(false, Ordering::Release);
}
@@ -614,13 +829,21 @@ fn tts_worker(
/// it is serialized with the monitor's stale-branch re-check (see the monitor
/// block in `tts_worker`).
fn handle_cancel_or_shutdown(
cancel: &AtomicBool,
cancel_signals: CancelSignals<'_>,
shutdown: &AtomicBool,
tts_active: &AtomicBool,
text_rx: &mpsc::Receiver<String>,
text_state: CancelTextState<'_>,
voice_change_ack: &VoiceChangeAck,
active_route_id: Option<u64>,
player: Option<(&rodio::Player, &Mutex<()>)>,
) -> bool {
let (cancel, voice_cancel) = cancel_signals;
let (text_rx, deferred_text, current_text) = text_state;
if shutdown.load(Ordering::Acquire) {
eprintln!(
"buzz-desktop: tts stage=cancellation reason=shutdown route_id={}",
active_route_id.unwrap_or(0)
);
if let Some((p, ops)) = player {
let _ops = lock_player_ops(ops);
p.clear();
@@ -628,7 +851,29 @@ fn handle_cancel_or_shutdown(
tts_active.store(false, Ordering::Release);
return true;
}
if cancel.load(Ordering::Acquire) {
if cancel.load(Ordering::Acquire) || voice_cancel.load(Ordering::Acquire) {
// Serialize with begin_voice_change so the generation boundary and
// cancel consumption are observed as one transition.
let pending_voice_change = voice_change_ack
.lock()
.unwrap_or_else(|error| error.into_inner());
// Consume at the serialization point. A later barge-in remains true
// for the next pass instead of being overwritten after queue cleanup.
let barge_in = cancel.swap(false, Ordering::AcqRel);
voice_cancel.store(false, Ordering::Release);
eprintln!(
"buzz-desktop: tts stage=cancellation reason={} route_id={}",
if barge_in { "barge_in" } else { "voice_switch" },
active_route_id.unwrap_or(0)
);
let preserve_generation = (!barge_in)
.then(|| {
pending_voice_change
.as_ref()
.map(|pending| pending.generation)
})
.flatten();
retain_cancelled_text(deferred_text, current_text, text_rx, preserve_generation);
if let Some((p, ops)) = player {
let _ops = lock_player_ops(ops);
// `Player::clear()` removes queued sources AND pauses the player
@@ -641,11 +886,6 @@ fn handle_cancel_or_shutdown(
// Consume the flag under the lock: once released with
// `cancel == false`, the monitor's stale branch no-ops instead
// of clearing the fresh post-cancel utterance.
while text_rx.try_recv().is_ok() {}
cancel.store(false, Ordering::Release);
} else {
while text_rx.try_recv().is_ok() {}
cancel.store(false, Ordering::Release);
}
tts_active.store(false, Ordering::Release);
return true;
@@ -663,140 +903,11 @@ fn lock_player_ops(ops: &Mutex<()>) -> MutexGuard<'_, ()> {
ops.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Hard-clamp samples to ±1.0 full scale.
///
/// No gain is applied because Pocket TTS already emits speech-level audio and
/// the reference pipeline applies no output scaling. Normalizing each sentence
/// would cause level pumping between chunks. The clamp remains only as a safety
/// net against outlier transients.
fn clamp_to_full_scale(samples: Vec<f32>) -> Vec<f32> {
samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect()
}
/// Apply a short linear fade-out at the *end* of `samples`.
///
/// Uses `FADE_OUT_SAMPLES` (8 ms) or half the buffer length, whichever is
/// smaller. Eliminates the click that occurs when a non-zero waveform
/// terminates abruptly at a sentence boundary.
///
/// # Why no fade-in
///
/// A symmetric fade-in would attenuate the leading consonant attack because
/// Pocket TTS produces real audio energy inside the first millisecond. A
/// linear 0→1 ramp over 192 samples scales those onset samples by ≤50% for the
/// first ~4 ms, which can make the first phoneme sound clipped.
///
/// The first sample of Pocket output measures ≈ 0.0018 (≈ 54 dBFS) — well
/// below the threshold at which a DC-jump would be audible as a click — so
/// no fade-in is needed. The OS audio device gets its quiet ramp-up window
/// from `SENTENCE_LEAD_IN_SAMPLES` instead, inserted as pure silence before
/// each sentence buffer.
fn apply_fade_out(samples: &mut [f32]) {
let len = samples.len();
let fade = FADE_OUT_SAMPLES.min(len / 2);
for i in 0..fade {
samples[len - 1 - i] *= i as f32 / fade as f32;
}
}
/// Build one buffer appended to the rodio `Player` for a synthesis unit.
///
/// Every playback boundary gets a short lead-in pad immediately before its
/// audio. This matters for chunks that start with soft first phonemes (`I'm`,
/// `I've`): the synthesized buffer can begin with speech within the first
/// millisecond, so the playback layer must provide the device/mixer cushion.
/// To keep the audible gap unchanged, the trailing silence after this chunk is
/// shortened by the same amount (`silence_buf_len - SENTENCE_LEAD_IN_SAMPLES`):
/// sentence N contributes 80 ms of post-speech silence and sentence N+1
/// contributes the remaining 20 ms of pre-speech cushion.
///
/// The lead-in, audio, and trailing silence are concatenated into one
/// `SamplesBuffer` before appending. This keeps rodio's queue shape at one
/// tracked source per synthesized sentence, avoiding source-boundary/drain
/// regressions from enqueueing the lead-in, audio, and tail as separate sounds.
///
/// A playback chunk may contain several model-sized synthesis units. Only the
/// first unit receives the onset cushion and only the last receives the
/// remaining gap. If playback underruns while the next unit is synthesized,
/// that unit becomes a new playback boundary and receives a fresh cushion.
///
/// `first_append` is flipped on the first call after the player goes idle.
/// The worker uses it in the idle branch of the main loop to distinguish
/// "never queued anything since last drain" from "drained after speaking",
/// which controls when `tts_active` is released and the lead-in re-armed.
fn build_sentence_append_buffer(
first_append: &mut bool,
audio: Vec<f32>,
silence_buf_len: usize,
starts_playback_chunk: bool,
ends_playback_chunk: bool,
) -> Vec<f32> {
if *first_append {
*first_append = false;
}
let lead_in_len = if starts_playback_chunk {
SENTENCE_LEAD_IN_SAMPLES
} else {
0
};
let trailing_silence_len = if ends_playback_chunk {
silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES)
} else {
0
};
let mut buf = Vec::with_capacity(lead_in_len + audio.len() + trailing_silence_len);
buf.extend(std::iter::repeat_n(0.0_f32, lead_in_len));
buf.extend(audio);
buf.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len));
buf
}
/// Group sentences into synthesis chunks.
///
/// The first sentence always stands alone — it is what the listener hears
/// first, and synthesizing it by itself keeps time-to-first-audio at the
/// single-sentence cost. Subsequent sentences pack greedily: a sentence
/// joins the current chunk while the combined length stays within
/// `max_chars`; otherwise it starts a new chunk. A single sentence longer
/// than `max_chars` becomes its own chunk here, then the Pocket engine splits
/// it at the April bundle's exact token limit before synthesis.
///
/// Sentences within a chunk are joined with a single space; sentence-ending
/// punctuation is preserved by `split_sentences`, so the model sees natural
/// multi-sentence prose — the same shape upstream's ~50-token chunker feeds it.
fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec<String> {
let mut chunks: Vec<String> = Vec::new();
for (i, sentence) in sentences.iter().enumerate() {
let sentence = sentence.trim();
if sentence.is_empty() {
continue;
}
if i == 0 || chunks.is_empty() {
chunks.push(sentence.to_string());
continue;
}
// Never merge into the first chunk — it's the latency-critical one.
let can_merge = chunks.len() > 1
&& chunks
.last()
.is_some_and(|c| c.len() + 1 + sentence.len() <= max_chars);
if can_merge {
let last = chunks.last_mut().expect("non-empty checked above");
last.push(' ');
last.push_str(sentence);
} else {
chunks.push(sentence.to_string());
}
}
chunks
}
// drain_until_shutdown lives in super (huddle/mod.rs) — shared with stt.rs.
use super::drain_until_shutdown;
// ── Tests ─────────────────────────────────────────────────────────────────────
#[cfg(test)]
#[path = "tts_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "tts_voice_selection_tests.rs"]
mod voice_selection_tests;
+235
View File
@@ -0,0 +1,235 @@
use super::{FADE_OUT_SAMPLES, SENTENCE_LEAD_IN_SAMPLES};
pub(super) struct PreparedModelAudio {
pub(super) buffer: Vec<f32>,
pub(super) sample_count: usize,
pub(super) chunk_index: usize,
}
/// Holds one synthesized model unit so playback-boundary decoration is based
/// on the first and last unit that actually produced audio.
pub(super) struct PlaybackChunkAudio {
pending: Option<(Vec<f32>, usize)>,
appended: bool,
}
impl PlaybackChunkAudio {
pub(super) fn new() -> Self {
Self {
pending: None,
appended: false,
}
}
pub(super) fn push(
&mut self,
samples: Vec<f32>,
chunk_index: usize,
first_append: &mut bool,
silence_buf_len: usize,
playback_idle: bool,
) -> Option<PreparedModelAudio> {
if samples.is_empty() {
return None;
}
let previous = self.pending.replace((samples, chunk_index))?;
let prepared = prepare_model_audio(
previous,
first_append,
silence_buf_len,
!self.appended || playback_idle,
false,
);
self.appended = true;
Some(prepared)
}
pub(super) fn finish(
&mut self,
first_append: &mut bool,
silence_buf_len: usize,
playback_idle: bool,
) -> Option<PreparedModelAudio> {
let pending = self.pending.take()?;
Some(prepare_model_audio(
pending,
first_append,
silence_buf_len,
!self.appended || playback_idle,
true,
))
}
}
fn prepare_model_audio(
(samples, chunk_index): (Vec<f32>, usize),
first_append: &mut bool,
silence_buf_len: usize,
starts_playback_chunk: bool,
ends_playback_chunk: bool,
) -> PreparedModelAudio {
let sample_count = samples.len();
let mut audio = clamp_to_full_scale(samples);
if ends_playback_chunk {
apply_fade_out(&mut audio);
}
PreparedModelAudio {
buffer: build_sentence_append_buffer(
first_append,
audio,
silence_buf_len,
starts_playback_chunk,
ends_playback_chunk,
),
sample_count,
chunk_index,
}
}
/// Hard-clamp samples to ±1.0 full scale.
pub(super) fn clamp_to_full_scale(samples: Vec<f32>) -> Vec<f32> {
samples.into_iter().map(|s| s.clamp(-1.0, 1.0)).collect()
}
/// Apply a short linear fade-out to avoid a discontinuity at playback boundaries.
pub(super) fn apply_fade_out(samples: &mut [f32]) {
let len = samples.len();
let fade = FADE_OUT_SAMPLES.min(len / 2);
for i in 0..fade {
samples[len - 1 - i] *= i as f32 / fade as f32;
}
}
pub(super) fn build_sentence_append_buffer(
first_append: &mut bool,
audio: Vec<f32>,
silence_buf_len: usize,
starts_playback_chunk: bool,
ends_playback_chunk: bool,
) -> Vec<f32> {
if *first_append {
*first_append = false;
}
let lead_in_len = if starts_playback_chunk {
SENTENCE_LEAD_IN_SAMPLES
} else {
0
};
let trailing_silence_len = if ends_playback_chunk {
silence_buf_len.saturating_sub(SENTENCE_LEAD_IN_SAMPLES)
} else {
0
};
let mut buffer = Vec::with_capacity(lead_in_len + audio.len() + trailing_silence_len);
buffer.extend(std::iter::repeat_n(0.0_f32, lead_in_len));
buffer.extend(audio);
buffer.extend(std::iter::repeat_n(0.0_f32, trailing_silence_len));
buffer
}
pub(super) fn group_sentences_into_chunks(sentences: &[String], max_chars: usize) -> Vec<String> {
let mut chunks: Vec<String> = Vec::new();
for (index, sentence) in sentences.iter().enumerate() {
let sentence = sentence.trim();
if sentence.is_empty() {
continue;
}
if index == 0 || chunks.is_empty() {
chunks.push(sentence.to_string());
continue;
}
let can_merge = chunks.len() > 1
&& chunks
.last()
.is_some_and(|chunk| chunk.len() + 1 + sentence.len() <= max_chars);
if can_merge {
if let Some(last) = chunks.last_mut() {
last.push(' ');
last.push_str(sentence);
}
} else {
chunks.push(sentence.to_string());
}
}
chunks
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn multi_unit_audio_decorates_only_outer_playback_boundaries() {
let mut chunk = PlaybackChunkAudio::new();
let mut first_append = true;
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
assert!(chunk
.push(vec![0.4; 16], 0, &mut first_append, silence, false)
.is_none());
let first = chunk
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
.expect("first ready model unit");
assert_eq!(first.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
assert!(first.buffer[..SENTENCE_LEAD_IN_SAMPLES]
.iter()
.all(|sample| *sample == 0.0));
assert_eq!(first.buffer[SENTENCE_LEAD_IN_SAMPLES], 0.4);
let last = chunk
.finish(&mut first_append, silence, false)
.expect("last ready model unit");
assert_eq!(last.buffer.len(), 16 + 100);
assert_eq!(last.buffer.last(), Some(&0.0));
}
#[test]
fn empty_edge_units_do_not_steal_lead_in_or_trailing_boundary() {
let mut chunk = PlaybackChunkAudio::new();
let mut first_append = true;
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
assert!(chunk
.push(Vec::new(), 0, &mut first_append, silence, false)
.is_none());
assert!(chunk
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
.is_none());
assert!(chunk
.push(Vec::new(), 2, &mut first_append, silence, false)
.is_none());
let only = chunk
.finish(&mut first_append, silence, false)
.expect("only audible model unit");
assert_eq!(only.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16 + 100);
assert!(only.buffer[..SENTENCE_LEAD_IN_SAMPLES]
.iter()
.all(|sample| *sample == 0.0));
assert_eq!(only.buffer.last(), Some(&0.0));
}
#[test]
fn playback_underrun_rearms_the_onset_cushion() {
let mut chunk = PlaybackChunkAudio::new();
let mut first_append = true;
let silence = SENTENCE_LEAD_IN_SAMPLES + 100;
assert!(chunk
.push(vec![0.4; 16], 0, &mut first_append, silence, false)
.is_none());
let first = chunk
.push(vec![0.5; 16], 1, &mut first_append, silence, false)
.expect("first model unit");
assert_eq!(first.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
let after_underrun = chunk
.push(vec![0.6; 16], 2, &mut first_append, silence, true)
.expect("model unit after underrun");
assert_eq!(after_underrun.buffer.len(), SENTENCE_LEAD_IN_SAMPLES + 16);
assert!(after_underrun.buffer[..SENTENCE_LEAD_IN_SAMPLES]
.iter()
.all(|sample| *sample == 0.0));
}
}
@@ -0,0 +1,831 @@
//! Installation-global text-to-speech preferences and the local voice registry.
//!
//! Voice keys are backend-qualified (`pocket:mary`, `siri:aaron`) and
//! preferences are ordered. A client resolves the first compatible entry for
//! its one active playback backend. The same [`VoicePreferences`] value can be
//! embedded in installation-global settings or future agent identity without a
//! schema change. Availability is intentionally client-local.
use std::{
path::{Path, PathBuf},
sync::{Arc, Mutex},
time::Duration,
};
use serde::{Deserialize, Serialize};
use tauri::{AppHandle, Manager, State};
use crate::{app_state::AppState, managed_agents::storage::atomic_write_json_restricted};
use super::{
models,
pocket::DEFAULT_VOICE,
tts_voice_registry::{source_url, MARY_VOICE_KEY, POCKET_VOICES},
HuddlePhase, HuddleState,
};
const SETTINGS_FILE: &str = "tts-settings.json";
const CURRENT_VERSION: u32 = 1;
const VOICE_CHANGE_ACK_TIMEOUT: Duration = Duration::from_secs(5);
pub const POCKET_BACKEND_ID: &str = "pocket";
type VoiceChangeWait = (
Arc<super::tts::TtsPipeline>,
tokio::sync::oneshot::Receiver<()>,
);
const VOICE_AVAILABILITY_BUNDLED: &str = "bundled";
const VOICE_AVAILABILITY_INSTALLED: &str = "installed";
/// Installation-global huddle audio and speech preferences.
#[derive(Default)]
pub struct HuddleAudioSettingsState {
pub tts: Mutex<TtsSettings>,
pub tts_load_error: Mutex<Option<String>>,
pub tts_transition: tokio::sync::Mutex<()>,
/// Selected huddle output device. `None` uses the system default.
pub output_device: Mutex<Option<String>>,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct VoiceRegistryEntry {
/// Stable identity, never derived from or merged by the display name.
///
/// Built-ins use `backend:slug`. Future imports use
/// `pocket:imported:<audio-content-sha256>` so two clips with the same
/// editable label remain distinct.
pub key: String,
pub display_name: String,
pub backend: String,
pub backend_name: String,
/// Client-local state: bundled, installed, downloadable, or unavailable.
pub availability: String,
pub fallback_key: Option<String>,
pub reference_file: Option<String>,
pub provenance: VoiceProvenance,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct VoiceProvenance {
pub source: String,
pub content_hash: Option<String>,
pub license: Option<String>,
pub source_url: Option<String>,
}
/// Ordered, backend-qualified preferences shared by global and agent settings.
///
/// Unknown but well-formed keys remain persisted because a different client
/// may have that backend installed. Resolution is always local.
pub type VoicePreferences = Vec<String>;
/// Cross-backend registry for voices known to this client.
///
/// V1 contains Pocket entries only. Siri, Kokoro, imported voices, and
/// per-agent assignment can add entries or reuse the preference type without
/// changing the registry/settings boundary.
pub fn voice_registry() -> Vec<VoiceRegistryEntry> {
POCKET_VOICES
.iter()
.map(|voice| VoiceRegistryEntry {
key: voice.key.to_string(),
display_name: voice.display_name.to_string(),
backend: POCKET_BACKEND_ID.to_string(),
backend_name: "Pocket TTS".to_string(),
availability: VOICE_AVAILABILITY_BUNDLED.to_string(),
fallback_key: (voice.key != MARY_VOICE_KEY).then(|| MARY_VOICE_KEY.to_string()),
reference_file: Some(voice.reference_file.to_string()),
provenance: VoiceProvenance {
source: "bundled".to_string(),
content_hash: Some(voice.sha256.to_string()),
license: Some("CC-BY-4.0".to_string()),
source_url: Some(source_url(voice)),
},
})
.collect()
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct TtsSettings {
pub version: u32,
pub agent_text_to_speech: bool,
pub voice_preferences: VoicePreferences,
}
impl Default for TtsSettings {
fn default() -> Self {
Self {
version: CURRENT_VERSION,
agent_text_to_speech: true,
voice_preferences: vec![MARY_VOICE_KEY.to_string()],
}
}
}
pub fn voice_by_key(key: &str) -> Option<VoiceRegistryEntry> {
voice_registry().into_iter().find(|voice| voice.key == key)
}
fn is_qualified_voice_key(key: &str) -> bool {
key.split_once(':')
.is_some_and(|(backend, voice)| !backend.is_empty() && !voice.is_empty())
}
fn is_locally_available(availability: &str) -> bool {
matches!(
availability,
VOICE_AVAILABILITY_BUNDLED | VOICE_AVAILABILITY_INSTALLED
)
}
pub fn resolve_voice_for_backend(
preferences: &[String],
backend: &str,
) -> Result<VoiceRegistryEntry, String> {
let registry = voice_registry();
preferences
.iter()
.filter_map(|key| registry.iter().find(|voice| voice.key == *key))
.find(|voice| voice.backend == backend && is_locally_available(voice.availability.as_str()))
.or_else(|| {
registry.iter().find(|voice| {
voice.backend == backend
&& voice.fallback_key.is_none()
&& is_locally_available(voice.availability.as_str())
})
})
.cloned()
.ok_or_else(|| format!("No locally available fallback voice for backend {backend}"))
}
pub fn pocket_voice_name(preferences: &[String]) -> String {
resolve_voice_for_backend(preferences, POCKET_BACKEND_ID)
.ok()
.and_then(|voice| voice.reference_file)
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
.unwrap_or_else(|| DEFAULT_VOICE.to_string())
}
pub(crate) fn settings_path(app: &AppHandle) -> Result<PathBuf, String> {
app.path()
.app_data_dir()
.map(|dir| dir.join(SETTINGS_FILE))
.map_err(|error| format!("could not locate Buzz settings storage: {error}"))
}
pub(crate) fn load_from_path(path: &Path) -> Result<TtsSettings, String> {
if !path.exists() {
return Ok(TtsSettings::default());
}
let bytes = std::fs::read(path)
.map_err(|error| format!("could not read text-to-speech settings: {error}"))?;
let value: serde_json::Value = serde_json::from_slice(&bytes)
.map_err(|error| format!("text-to-speech settings are not valid JSON: {error}"))?;
// Unversioned settings are incompatible with the V1 schema. Use
// deterministic V1 defaults rather than interpreting ambiguous fields.
if value.get("version").is_none() {
return Ok(TtsSettings::default());
}
let version = value
.get("version")
.and_then(serde_json::Value::as_u64)
.ok_or("text-to-speech settings version is invalid")?;
if version > u64::from(CURRENT_VERSION) {
return Err(format!(
"text-to-speech settings version {version} is newer than this Buzz build supports"
));
}
// Legacy V1 settings may contain one bare Pocket `voiceId`. Preserve the
// toggle and qualify it into the ordered cross-backend preference schema.
if value.get("voicePreferences").is_none() {
let legacy_voice = value
.get("voiceId")
.or_else(|| value.get("voice_id"))
.and_then(serde_json::Value::as_str)
.unwrap_or("mary");
let voice_key = if is_qualified_voice_key(legacy_voice) {
legacy_voice.to_string()
} else {
format!("{POCKET_BACKEND_ID}:{legacy_voice}")
};
return Ok(TtsSettings {
version: CURRENT_VERSION,
agent_text_to_speech: value
.get("agentTextToSpeech")
.and_then(serde_json::Value::as_bool)
.unwrap_or(true),
voice_preferences: vec![voice_key],
});
}
let mut settings: TtsSettings = serde_json::from_value(value)
.map_err(|error| format!("text-to-speech settings are invalid: {error}"))?;
settings.version = CURRENT_VERSION;
if settings.voice_preferences.is_empty()
|| settings
.voice_preferences
.iter()
.any(|key| !is_qualified_voice_key(key))
{
settings.voice_preferences = TtsSettings::default().voice_preferences;
}
Ok(settings)
}
pub(crate) fn save_to_path(path: &Path, settings: &TtsSettings) -> Result<(), String> {
if settings.voice_preferences.is_empty() {
return Err("At least one voice preference is required".to_string());
}
if let Some(key) = settings
.voice_preferences
.iter()
.find(|key| !is_qualified_voice_key(key))
{
return Err(format!(
"Voice preference keys must be backend-qualified: {key}"
));
}
let payload = serde_json::to_vec_pretty(settings)
.map_err(|error| format!("could not encode text-to-speech settings: {error}"))?;
atomic_write_json_restricted(path, &payload)
.map_err(|error| format!("could not save text-to-speech settings: {error}"))
}
pub fn load_for_app(app: &AppHandle) -> (TtsSettings, Option<String>) {
let result = settings_path(app).and_then(|path| load_from_path(&path));
match result {
Ok(settings) => (settings, None),
Err(error) => {
eprintln!("buzz-desktop: {error}; preserving the file and using Mary for this session");
(TtsSettings::default(), Some(error))
}
}
}
#[tauri::command]
pub fn get_tts_settings(state: State<'_, AppState>) -> Result<TtsSettings, String> {
if let Some(error) = state
.huddle_audio
.tts_load_error
.lock()
.map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))?
.clone()
{
return Err(format!(
"Voice settings could not be loaded and were left unchanged: {error}"
));
}
state
.huddle_audio
.tts
.lock()
.map(|settings| settings.clone())
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
}
#[tauri::command]
pub fn list_voice_registry() -> Vec<VoiceRegistryEntry> {
voice_registry()
}
fn ensure_settings_writable(state: &AppState) -> Result<(), String> {
if let Some(error) = state
.huddle_audio
.tts_load_error
.lock()
.map_err(|lock_error| format!("text-to-speech settings lock poisoned: {lock_error}"))?
.as_ref()
{
return Err(format!(
"Voice settings were not saved because the existing file could not be loaded: {error}"
));
}
Ok(())
}
fn cancel_huddle_speech(
huddle: &mut super::HuddleState,
) -> Option<std::sync::Arc<super::tts::TtsPipeline>> {
huddle.tts_enabled = false;
huddle
.tts_cancel
.store(true, std::sync::atomic::Ordering::Release);
huddle.tts_pipeline.take()
}
fn disable_tts_runtime(state: &AppState) -> Result<(), String> {
let old_pipeline = {
let mut huddle = state.huddle()?;
cancel_huddle_speech(&mut huddle)
};
if let Some(ref pipeline) = old_pipeline {
pipeline.shutdown();
}
drop(old_pipeline);
state.emit_huddle_state_changed();
Ok(())
}
fn commit_effective_off(state: &AppState) -> Result<(), String> {
state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
.agent_text_to_speech = false;
Ok(())
}
fn enable_tts_runtime(huddle: &mut HuddleState, voice: &str) -> Option<VoiceChangeWait> {
huddle.tts_enabled = true;
// OFF removes the pipeline. Clear a prior cancellation only when enabling
// a fresh pipeline; an idempotent ON write must not erase a voice
// transition that the existing worker still needs to drain.
prepare_enable_cancel(&huddle.tts_cancel, huddle.tts_pipeline.is_some());
huddle.tts_pipeline.as_ref().and_then(|pipeline| {
pipeline
.select_voice(voice)
.map(|acknowledged| (Arc::clone(pipeline), acknowledged))
})
}
fn prepare_enable_cancel(cancel: &std::sync::atomic::AtomicBool, has_pipeline: bool) {
if !has_pipeline {
cancel.store(false, std::sync::atomic::Ordering::Release);
}
}
async fn apply_tts_settings(
settings: TtsSettings,
app: &AppHandle,
state: &AppState,
) -> Result<Option<VoiceChangeWait>, String> {
if settings.version != CURRENT_VERSION {
return Err(format!(
"Unsupported text-to-speech settings version: {}",
settings.version
));
}
// OFF is safety-sensitive: stop current and queued speech before any disk
// I/O, and never resume it merely because persistence fails.
if !settings.agent_text_to_speech {
disable_tts_runtime(state)?;
commit_effective_off(state)?;
}
ensure_settings_writable(state)?;
save_to_path(&settings_path(app)?, &settings)?;
*state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))? =
settings.clone();
let mut voice_change_wait = None;
if settings.agent_text_to_speech {
let (active, voice_change_ack) = {
let mut huddle = state.huddle()?;
let voice_change_ack =
enable_tts_runtime(&mut huddle, &pocket_voice_name(&settings.voice_preferences));
(
matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active),
voice_change_ack,
)
};
voice_change_wait = voice_change_ack;
if active {
if let Err(error) = super::pipeline::maybe_start_tts_pipeline(state).await {
eprintln!("buzz-desktop: could not hot-start text to speech: {error}");
}
}
state.emit_huddle_state_changed();
}
Ok(voice_change_wait)
}
fn current_settings(state: &AppState) -> Result<TtsSettings, String> {
state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| settings.clone())
}
async fn finish_voice_change(voice_change: Option<VoiceChangeWait>) -> Result<(), String> {
let Some((pipeline, acknowledged)) = voice_change else {
return Ok(());
};
wait_for_voice_change_ack(acknowledged, VOICE_CHANGE_ACK_TIMEOUT, || {
pipeline.is_finished()
})
.await
}
async fn wait_for_voice_change_ack(
mut acknowledged: tokio::sync::oneshot::Receiver<()>,
timeout: Duration,
mut worker_is_finished: impl FnMut() -> bool,
) -> Result<(), String> {
let deadline = tokio::time::sleep(timeout);
tokio::pin!(deadline);
loop {
tokio::select! {
_ = &mut acknowledged => return Ok(()),
_ = &mut deadline => {
return Err(
"Pocket TTS is still finishing the previous voice. Turn Agent text to speech off and try again."
.to_string(),
);
}
_ = tokio::time::sleep(Duration::from_millis(25)) => {
if worker_is_finished() {
return Ok(());
}
}
}
}
}
/// Compatibility command for the huddle speaker button. It updates the same
/// installation-global preference as Settings; there is no per-huddle override.
#[tauri::command]
pub async fn set_tts_enabled(
enabled: bool,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<TtsSettings, String> {
let transition = state.huddle_audio.tts_transition.lock().await;
let mut settings = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
.clone();
settings.agent_text_to_speech = enabled;
let voice_change = apply_tts_settings(settings, &app, &state).await?;
drop(transition);
finish_voice_change(voice_change).await?;
current_settings(&state)
}
fn settings_with_pocket_voice(
mut settings: TtsSettings,
voice_key: &str,
) -> Result<TtsSettings, String> {
let voice = voice_by_key(voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
if voice.backend != POCKET_BACKEND_ID || !is_locally_available(&voice.availability) {
return Err("The selected Pocket voice is not available on this device".to_string());
}
let first_pocket_index = settings
.voice_preferences
.iter()
.position(|key| key.starts_with("pocket:"));
settings
.voice_preferences
.retain(|key| !key.starts_with("pocket:"));
let insert_at = first_pocket_index
.unwrap_or(settings.voice_preferences.len())
.min(settings.voice_preferences.len());
settings
.voice_preferences
.insert(insert_at, voice_key.to_string());
Ok(settings)
}
#[tauri::command]
pub async fn set_pocket_voice(
voice_key: String,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<TtsSettings, String> {
let transition = state.huddle_audio.tts_transition.lock().await;
let settings = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
.clone();
let settings = settings_with_pocket_voice(settings, &voice_key)?;
let voice_change = apply_tts_settings(settings, &app, &state).await?;
drop(transition);
if let Err(error) = finish_voice_change(voice_change).await {
// The preference is already durable. Report the delayed live
// transition diagnostically without telling the UI that saving failed;
// the next pipeline start resolves the persisted voice normally.
eprintln!(
"buzz-desktop: tts stage=voice_switch status=delayed reason=ack_timeout error={error}"
);
}
current_settings(&state)
}
#[tauri::command]
pub async fn preview_pocket_voice(
voice_key: String,
state: State<'_, AppState>,
) -> Result<(), String> {
let voice = voice_by_key(&voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
if voice.backend != POCKET_BACKEND_ID {
return Err("Only Pocket voices can be previewed in this build".to_string());
}
if !models::is_tts_ready() {
return Err("Voice files are still downloading. Try preview again shortly.".to_string());
}
let model_dir = models::tts_model_dir().ok_or("Pocket voice files are unavailable")?;
let output_device = state
.huddle_audio
.output_device
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
let voice_name = voice
.reference_file
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
.ok_or_else(|| format!("Voice {voice_key} has no local Pocket reference file"))?;
tokio::task::spawn_blocking(move || {
let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let pipeline = super::tts::TtsPipeline::new_with_voice(
model_dir,
active.clone(),
cancel,
&voice_name,
output_device,
)?;
pipeline.speak("Hello! This is how Ill read agent responses.".to_string())?;
let started = std::time::Instant::now();
let mut heard_audio = false;
while started.elapsed() < std::time::Duration::from_secs(30) {
let is_active = active.load(std::sync::atomic::Ordering::Acquire);
heard_audio |= is_active;
if heard_audio && !is_active {
return Ok(());
}
std::thread::sleep(std::time::Duration::from_millis(25));
}
Err("Voice preview timed out. Check your audio output and try again.".to_string())
})
.await
.map_err(|error| format!("Voice preview task failed: {error}"))?
}
#[cfg(test)]
mod tests {
use super::*;
const EVE_VOICE_KEY: &str = "pocket:eve";
#[tokio::test]
async fn stalled_voice_change_returns_an_actionable_error() {
let (_keep_pending, acknowledged) = tokio::sync::oneshot::channel();
let error = wait_for_voice_change_ack(acknowledged, Duration::from_millis(1), || false)
.await
.expect_err("stalled worker should time out");
assert!(error.contains("Turn Agent text to speech off"));
}
#[test]
fn idempotent_enable_preserves_an_existing_pipeline_cancel() {
let cancel = std::sync::atomic::AtomicBool::new(true);
prepare_enable_cancel(&cancel, true);
assert!(cancel.load(std::sync::atomic::Ordering::Acquire));
prepare_enable_cancel(&cancel, false);
assert!(!cancel.load(std::sync::atomic::Ordering::Acquire));
}
#[test]
fn defaults_are_backwards_compatible_and_use_mary() {
assert_eq!(
TtsSettings::default(),
TtsSettings {
version: 1,
agent_text_to_speech: true,
voice_preferences: vec!["pocket:mary".to_string()],
}
);
}
#[test]
fn registry_has_all_official_english_vctk_presets() {
assert_eq!(
voice_registry()
.iter()
.map(|voice| {
(
voice.key.as_str(),
voice.display_name.as_str(),
voice.reference_file.as_deref(),
)
})
.collect::<Vec<_>>(),
vec![
("pocket:anna", "Anna", Some("anna.wav")),
("pocket:vera", "Vera", Some("vera.wav")),
("pocket:fantine", "Fantine", Some("fantine.wav")),
("pocket:charles", "Charles", Some("charles.wav")),
("pocket:paul", "Paul", Some("paul.wav")),
("pocket:eponine", "Eponine", Some("eponine.wav")),
("pocket:azelma", "Azelma", Some("azelma.wav")),
("pocket:george", "George", Some("george.wav")),
("pocket:mary", "Mary", Some("reference_sample.wav")),
("pocket:jane", "Jane", Some("jane.wav")),
("pocket:michael", "Michael", Some("michael.wav")),
("pocket:eve", "Eve", Some("eve.wav")),
]
);
}
#[test]
fn local_backend_resolution_uses_first_compatible_preference() {
let preferences = vec![
"siri:aaron".to_string(),
EVE_VOICE_KEY.to_string(),
MARY_VOICE_KEY.to_string(),
"kokoro:af_heart".to_string(),
];
assert_eq!(
resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID)
.expect("Pocket fallback")
.key,
EVE_VOICE_KEY
);
}
#[test]
fn unsupported_or_missing_preferences_fall_back_to_backend_default() {
let preferences = vec![
"siri:aaron".to_string(),
"pocket:imported:deadbeef".to_string(),
];
assert_eq!(
resolve_voice_for_backend(&preferences, POCKET_BACKEND_ID)
.expect("Pocket fallback")
.key,
MARY_VOICE_KEY
);
}
#[test]
fn identity_is_qualified_key_not_display_label() {
assert!(is_qualified_voice_key("pocket:imported:audio-content-hash"));
assert_ne!(MARY_VOICE_KEY, EVE_VOICE_KEY);
let mut registry = voice_registry();
registry[0].display_name = "Jim".to_string();
registry[1].display_name = "Jim".to_string();
assert_eq!(registry[0].display_name, registry[1].display_name);
assert_ne!(registry[0].key, registry[1].key);
assert_eq!(
registry
.iter()
.map(|voice| voice.key.as_str())
.collect::<std::collections::HashSet<_>>()
.len(),
registry.len()
);
}
#[test]
fn bundled_vctk_assets_match_the_registry_manifest() {
for voice in POCKET_VOICES {
let Some(bytes) = voice.bytes else {
continue;
};
assert_eq!(&bytes[0..4], b"RIFF", "{}", voice.display_name);
assert_eq!(&bytes[8..12], b"WAVE", "{}", voice.display_name);
assert_eq!(
hex::encode(<sha2::Sha256 as sha2::Digest>::digest(bytes)),
voice.sha256,
"{}",
voice.display_name
);
}
}
#[test]
fn migrates_unversioned_experiment_settings_to_v1_defaults() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join(SETTINGS_FILE);
std::fs::write(&path, r#"{"voice":"legacy-experiment"}"#).expect("fixture write");
assert_eq!(
load_from_path(&path).expect("migration"),
TtsSettings::default()
);
}
#[test]
fn migrates_bare_pocket_voice_id_to_qualified_preferences() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join(SETTINGS_FILE);
std::fs::write(
&path,
r#"{"version":1,"agentTextToSpeech":false,"voiceId":"eve"}"#,
)
.expect("fixture write");
assert_eq!(
load_from_path(&path).expect("migration"),
TtsSettings {
version: 1,
agent_text_to_speech: false,
voice_preferences: vec![EVE_VOICE_KEY.to_string()],
}
);
}
#[test]
fn unknown_qualified_preferences_are_preserved_for_other_clients() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join(SETTINGS_FILE);
std::fs::write(
&path,
r#"{"version":1,"agentTextToSpeech":false,"voicePreferences":["siri:aaron","pocket:imported:abc123"]}"#,
)
.expect("fixture write");
let settings = load_from_path(&path).expect("load");
assert!(!settings.agent_text_to_speech);
assert_eq!(
settings.voice_preferences,
vec!["siri:aaron", "pocket:imported:abc123"]
);
}
#[test]
fn rejects_future_schema_versions_clearly() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join(SETTINGS_FILE);
std::fs::write(
&path,
r#"{"version":99,"agentTextToSpeech":true,"voicePreferences":["pocket:mary"]}"#,
)
.expect("fixture write");
assert!(load_from_path(&path)
.expect_err("future version should fail")
.contains("newer than this Buzz build supports"));
}
#[test]
fn disabling_cancels_runtime_before_persistence_can_fail() {
let mut huddle = super::super::HuddleState {
tts_enabled: true,
..super::super::HuddleState::default()
};
assert!(!huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire));
assert!(cancel_huddle_speech(&mut huddle).is_none());
assert!(!huddle.tts_enabled);
assert!(huddle.tts_cancel.load(std::sync::atomic::Ordering::Acquire));
}
#[test]
fn pocket_voice_update_preserves_the_latest_toggle_and_other_backends() {
let current = TtsSettings {
agent_text_to_speech: false,
voice_preferences: vec!["siri:aaron".to_string(), MARY_VOICE_KEY.to_string()],
..TtsSettings::default()
};
let updated = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
assert!(!updated.agent_text_to_speech);
assert_eq!(updated.voice_preferences, vec!["siri:aaron", EVE_VOICE_KEY]);
}
#[test]
fn failed_off_persistence_cannot_be_undone_by_a_later_voice_update() {
let state = crate::app_state::build_app_state();
commit_effective_off(&state).expect("commit effective OFF state");
// This models the next command after the OFF save fails: it must merge
// from effective memory state, not the stale last-persisted ON value.
let current = state.huddle_audio.tts.lock().expect("settings").clone();
let voice_update =
settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
assert!(!voice_update.agent_text_to_speech);
}
#[test]
fn failed_disabled_voice_save_does_not_change_the_remembered_voice() {
let state = crate::app_state::build_app_state();
state
.huddle_audio
.tts
.lock()
.expect("settings")
.agent_text_to_speech = false;
let current = state.huddle_audio.tts.lock().expect("settings").clone();
let unsaved = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
// This is the only pre-persistence mutation for an OFF candidate.
commit_effective_off(&state).expect("commit effective OFF state");
let remembered = state.huddle_audio.tts.lock().expect("settings").clone();
assert_eq!(remembered.voice_preferences, vec![MARY_VOICE_KEY]);
assert_eq!(unsaved.voice_preferences, vec![EVE_VOICE_KEY]);
}
}
@@ -0,0 +1,24 @@
use std::{sync::mpsc, thread};
pub(super) fn await_worker_startup(
handle: thread::JoinHandle<()>,
startup_rx: mpsc::Receiver<Result<(), String>>,
) -> Result<thread::JoinHandle<()>, String> {
match startup_rx.recv() {
Ok(Ok(())) => Ok(handle),
Ok(Err(error)) => {
let _ = handle.join();
Err(error)
}
Err(error) => {
let _ = handle.join();
Err(format!(
"TTS worker exited before reporting readiness: {error}"
))
}
}
}
#[cfg(test)]
#[path = "tts_startup_tests.rs"]
mod tests;
@@ -0,0 +1,44 @@
use super::*;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
#[test]
fn startup_failure_is_returned_after_worker_exit() {
let (tx, rx) = mpsc::sync_channel(1);
let exited = Arc::new(AtomicBool::new(false));
let exited_worker = Arc::clone(&exited);
let handle = std::thread::spawn(move || {
tx.send(Err("output unavailable".to_string()))
.expect("startup receiver");
exited_worker.store(true, Ordering::Release);
});
assert_eq!(
await_worker_startup(handle, rx).expect_err("startup must fail"),
"output unavailable"
);
assert!(exited.load(Ordering::Acquire));
}
#[test]
fn worker_exit_before_readiness_is_a_startup_error() {
let (tx, rx) = mpsc::sync_channel::<Result<(), String>>(1);
let handle = std::thread::spawn(move || drop(tx));
assert!(await_worker_startup(handle, rx)
.expect_err("closed startup channel must fail")
.contains("before reporting readiness"));
}
#[test]
fn ready_ack_precedes_pipeline_publication_boundary() {
let (tx, rx) = mpsc::sync_channel(1);
let handle = std::thread::spawn(move || {
tx.send(Ok(())).expect("startup receiver");
});
let handle = await_worker_startup(handle, rx).expect("ready worker");
handle.join().expect("worker exits");
}
@@ -0,0 +1,129 @@
//! Built-in Pocket voice identities and immutable asset metadata.
//!
//! Stable keys identify audio, not display labels. Future imported voices use
//! `pocket:imported:<audio-content-sha256>` and may share editable labels.
pub(super) const MARY_VOICE_KEY: &str = "pocket:mary";
pub(super) const VCTK_REVISION: &str = "323332d33f997de8394f24a193e1a76df720e01a";
pub(super) struct PocketVoiceSpec {
pub key: &'static str,
pub display_name: &'static str,
pub reference_file: &'static str,
pub upstream_file: &'static str,
pub sha256: &'static str,
pub bytes: Option<&'static [u8]>,
}
macro_rules! bundled_voice {
($key:literal, $name:literal, $file:literal, $upstream:literal, $hash:literal) => {
PocketVoiceSpec {
key: $key,
display_name: $name,
reference_file: concat!($file, ".wav"),
upstream_file: concat!("vctk/", $upstream),
sha256: $hash,
bytes: Some(include_bytes!(concat!(
"../../resources/pocket-voices/",
$file,
".wav"
))),
}
};
}
/// Official English Pocket presets, in the order published by Kyutai.
pub(super) static POCKET_VOICES: &[PocketVoiceSpec] = &[
bundled_voice!(
"pocket:anna",
"Anna",
"anna",
"p228_023_enhanced.wav",
"0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856"
),
bundled_voice!(
"pocket:vera",
"Vera",
"vera",
"p229_023_enhanced.wav",
"309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b"
),
bundled_voice!(
"pocket:fantine",
"Fantine",
"fantine",
"p244_023_enhanced.wav",
"5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b"
),
bundled_voice!(
"pocket:charles",
"Charles",
"charles",
"p254_023_enhanced.wav",
"6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756"
),
bundled_voice!(
"pocket:paul",
"Paul",
"paul",
"p259_023_enhanced.wav",
"7aba504fe0b3b16478b69eb27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b"
),
bundled_voice!(
"pocket:eponine",
"Eponine",
"eponine",
"p262_023_enhanced.wav",
"a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b"
),
bundled_voice!(
"pocket:azelma",
"Azelma",
"azelma",
"p303_023_enhanced.wav",
"60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026"
),
bundled_voice!(
"pocket:george",
"George",
"george",
"p315_023_enhanced.wav",
"29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae"
),
PocketVoiceSpec {
key: MARY_VOICE_KEY,
display_name: "Mary",
reference_file: "reference_sample.wav",
upstream_file: "vctk/p333_023_enhanced.wav",
sha256: "a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f",
bytes: None,
},
bundled_voice!(
"pocket:jane",
"Jane",
"jane",
"p339_023_enhanced.wav",
"2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a"
),
bundled_voice!(
"pocket:michael",
"Michael",
"michael",
"p360_023_enhanced.wav",
"b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad"
),
bundled_voice!(
"pocket:eve",
"Eve",
"eve",
"p361_023_enhanced.wav",
"396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd"
),
];
pub(super) fn source_url(voice: &PocketVoiceSpec) -> String {
format!(
"https://huggingface.co/kyutai/tts-voices/blob/{VCTK_REVISION}/{}",
voice.upstream_file
)
}
@@ -0,0 +1,385 @@
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
fn inert_pipeline(cancel: Arc<AtomicBool>) -> TtsPipeline {
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(TEXT_QUEUE_DEPTH);
let shutdown = Arc::new(AtomicBool::new(false));
let worker_shutdown = Arc::clone(&shutdown);
let thread = std::thread::spawn(move || {
while !worker_shutdown.load(Ordering::Acquire) {
let _ = text_rx.recv_timeout(RECV_TIMEOUT);
}
});
TtsPipeline {
text_tx,
tts_active: Arc::new(AtomicBool::new(false)),
shutdown,
cancel,
voice_cancel: Arc::new(AtomicBool::new(false)),
voice: Arc::new(std::sync::Mutex::new("reference_sample".to_string())),
voice_generation: Arc::new(AtomicU64::new(1)),
voice_change_ack: Arc::new(std::sync::Mutex::new(None)),
thread: Some(thread),
}
}
#[test]
fn selecting_a_voice_raises_only_the_internal_cancel_and_retains_the_engine_handle() {
let cancel = Arc::new(AtomicBool::new(false));
let pipeline = inert_pipeline(Arc::clone(&cancel));
let _acknowledged = pipeline.select_voice("eve");
assert!(!cancel.load(Ordering::Acquire));
assert!(pipeline.voice_cancel.load(Ordering::Acquire));
assert_eq!(
pipeline
.voice
.lock()
.unwrap_or_else(|error| error.into_inner())
.as_str(),
"eve"
);
}
#[test]
fn reconciling_an_unpublished_pipeline_does_not_cancel_its_first_message() {
let cancel = Arc::new(AtomicBool::new(false));
let pipeline = inert_pipeline(Arc::clone(&cancel));
pipeline.select_voice_before_publish("eve");
assert!(!cancel.load(Ordering::Acquire));
assert_eq!(
pipeline
.voice
.lock()
.unwrap_or_else(|error| error.into_inner())
.as_str(),
"eve"
);
}
#[test]
fn received_text_reconciles_a_voice_changed_while_the_worker_was_waiting() {
let model_dir = tempfile::tempdir().expect("temp model dir");
let bundled_voice =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/eve.wav");
std::fs::copy(
&bundled_voice,
model_dir.path().join("reference_sample.wav"),
)
.expect("Mary test voice");
std::fs::copy(&bundled_voice, model_dir.path().join("eve.wav")).expect("Eve test voice");
let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string()));
let mut style =
load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("initial style");
let waiting = Arc::new(std::sync::Barrier::new(2));
let (text_tx, text_rx) = std::sync::mpsc::channel();
let worker_voice = Arc::clone(&selected_voice);
let worker_waiting = Arc::clone(&waiting);
let worker_model_dir = model_dir.path().to_path_buf();
let worker = std::thread::spawn(move || {
let mut voice_name = "reference_sample".to_string();
worker_waiting.wait();
let text = text_rx.recv().expect("first queued text");
assert!(reconcile_selected_voice(
&worker_model_dir,
&worker_voice,
&mut voice_name,
&mut style,
));
(text, voice_name)
});
waiting.wait();
*selected_voice.lock().expect("selected voice") = "eve".to_string();
text_tx
.send("first message".to_string())
.expect("queue first message");
assert_eq!(
worker.join().expect("worker"),
("first message".to_string(), "eve".to_string())
);
}
#[test]
fn corrupt_selected_voice_falls_back_to_mary() {
let model_dir = tempfile::tempdir().expect("temp model dir");
let bundled_voice =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("resources/pocket-voices/eve.wav");
std::fs::copy(bundled_voice, model_dir.path().join("reference_sample.wav"))
.expect("Mary test voice");
std::fs::write(model_dir.path().join("eve.wav"), b"not a wave")
.expect("corrupt selected voice");
let selected_voice = std::sync::Mutex::new("eve".to_string());
let mut voice_name = "reference_sample".to_string();
let mut style =
load_voice_style(&model_dir.path().join("reference_sample.wav")).expect("Mary style");
assert!(reconcile_selected_voice(
model_dir.path(),
&selected_voice,
&mut voice_name,
&mut style,
));
assert_eq!(voice_name, DEFAULT_VOICE);
assert_eq!(
selected_voice
.lock()
.unwrap_or_else(|error| error.into_inner())
.as_str(),
DEFAULT_VOICE
);
}
#[test]
fn an_in_hand_post_change_message_survives_cancellation() {
let selected_voice = Arc::new(std::sync::Mutex::new("reference_sample".to_string()));
let voice_generation = AtomicU64::new(1);
let barge_in = AtomicBool::new(false);
let voice_cancel = Arc::new(AtomicBool::new(false));
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1);
let mut acknowledged = begin_voice_change(
&selected_voice,
&voice_generation,
&voice_cancel,
&voice_change_ack,
"eve",
)
.expect("voice changed");
assert!(voice_cancel.load(Ordering::Acquire));
assert!(matches!(
acknowledged.try_recv(),
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
));
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
assert!(matches!(
acknowledged.try_recv(),
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
));
text_tx
.send(QueuedText {
generation: voice_generation.load(Ordering::Acquire),
route_id: 1,
text: "new message".to_string(),
})
.expect("new message");
let mut current_text = Some(text_rx.recv().expect("in-hand new message"));
let shutdown = AtomicBool::new(false);
let active = AtomicBool::new(true);
let mut deferred_text = VecDeque::from([
QueuedText {
generation: 1,
route_id: 2,
text: "old message".to_string(),
},
QueuedText {
generation: voice_generation.load(Ordering::Acquire),
route_id: 3,
text: "later new message".to_string(),
},
]);
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
acknowledged.blocking_recv().expect("voice change ack");
assert_eq!(
deferred_text
.pop_front()
.expect("preserved post-change message")
.text,
"new message"
);
assert_eq!(
deferred_text
.pop_front()
.expect("later post-change message")
.text,
"later new message"
);
assert!(text_rx.try_recv().is_err());
}
#[test]
fn superseding_voice_change_removes_earlier_deferred_messages() {
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
let voice_generation = AtomicU64::new(1);
let barge_in = AtomicBool::new(false);
let voice_cancel = AtomicBool::new(false);
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
let (_text_tx, text_rx) = std::sync::mpsc::channel();
let shutdown = AtomicBool::new(false);
let active = AtomicBool::new(true);
let mut deferred_text = VecDeque::new();
let mut current_text = None;
let first = begin_voice_change(
&selected_voice,
&voice_generation,
&voice_cancel,
&voice_change_ack,
"eve",
)
.expect("first voice change");
deferred_text.push_back(QueuedText {
generation: voice_generation.load(Ordering::Acquire),
route_id: 4,
text: "message for Eve".to_string(),
});
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
acknowledge_voice_change(&voice_change_ack, &voice_cancel);
first.blocking_recv().expect("first acknowledgement");
let _second = begin_voice_change(
&selected_voice,
&voice_generation,
&voice_cancel,
&voice_change_ack,
"reference_sample",
)
.expect("second voice change");
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
assert!(deferred_text.is_empty());
}
#[test]
fn barge_in_clears_deferred_voice_change_messages() {
let barge_in = AtomicBool::new(true);
let voice_cancel = AtomicBool::new(false);
let shutdown = AtomicBool::new(false);
let active = AtomicBool::new(true);
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
let (_text_tx, text_rx) = std::sync::mpsc::channel();
let mut deferred_text = VecDeque::from([QueuedText {
generation: 2,
route_id: 5,
text: "deferred message".to_string(),
}]);
let mut current_text = None;
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
assert!(deferred_text.is_empty());
}
#[test]
fn barge_in_during_a_voice_change_clears_post_change_messages() {
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
let voice_generation = AtomicU64::new(1);
let barge_in = AtomicBool::new(false);
let voice_cancel = AtomicBool::new(false);
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
let (_text_tx, text_rx) = std::sync::mpsc::channel();
let shutdown = AtomicBool::new(false);
let active = AtomicBool::new(true);
let mut deferred_text = VecDeque::new();
let mut current_text = None;
let _acknowledged = begin_voice_change(
&selected_voice,
&voice_generation,
&voice_cancel,
&voice_change_ack,
"eve",
)
.expect("voice change");
deferred_text.push_back(QueuedText {
generation: voice_generation.load(Ordering::Acquire),
route_id: 6,
text: "post-change message".to_string(),
});
barge_in.store(true, Ordering::Release);
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
assert!(deferred_text.is_empty());
}
#[test]
fn a_sender_captured_before_voice_change_is_stale_even_if_it_sends_after_drain() {
let selected_voice = std::sync::Mutex::new("reference_sample".to_string());
let voice_generation = Arc::new(AtomicU64::new(1));
let barge_in = AtomicBool::new(false);
let voice_cancel = AtomicBool::new(false);
let voice_change_ack = Arc::new(std::sync::Mutex::new(None));
let (text_tx, text_rx) = std::sync::mpsc::sync_channel(1);
let old_sender = TtsTextSender {
text_tx,
generation: voice_generation.load(Ordering::Acquire),
};
let shutdown = AtomicBool::new(false);
let active = AtomicBool::new(true);
let mut deferred_text = VecDeque::new();
let mut current_text = None;
let _acknowledged = begin_voice_change(
&selected_voice,
&voice_generation,
&voice_cancel,
&voice_change_ack,
"eve",
)
.expect("voice change");
assert!(handle_cancel_or_shutdown(
(&barge_in, &voice_cancel),
&shutdown,
&active,
(&text_rx, &mut deferred_text, &mut current_text),
&voice_change_ack,
None,
None,
));
old_sender
.send(7, "late old message".to_string())
.expect("late send");
let late = text_rx.recv().expect("late queued text");
assert!(late.generation < voice_generation.load(Ordering::Acquire));
}
@@ -0,0 +1,197 @@
use std::{
collections::VecDeque,
path::Path,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
mpsc::{self, SyncSender},
Arc, Mutex,
},
};
use crate::huddle::pocket::{load_voice_style, VoiceStyle, DEFAULT_VOICE, VOICE_FILE_EXT};
#[derive(Debug)]
pub(super) struct PendingVoiceChange {
pub(super) generation: u64,
acknowledged: tokio::sync::oneshot::Sender<()>,
}
pub(super) type VoiceChangeAck = Arc<Mutex<Option<PendingVoiceChange>>>;
pub(super) type WorkerVoiceState = (Arc<Mutex<String>>, Arc<AtomicU64>, VoiceChangeAck);
pub(super) type WorkerCancelSignals = (Arc<AtomicBool>, Arc<AtomicBool>);
pub(super) type CancelTextState<'a> = (
&'a mpsc::Receiver<QueuedText>,
&'a mut VecDeque<QueuedText>,
&'a mut Option<QueuedText>,
);
pub(super) type CancelSignals<'a> = (&'a AtomicBool, &'a AtomicBool);
#[derive(Debug)]
pub(super) struct QueuedText {
pub(super) generation: u64,
pub(super) route_id: u64,
pub(super) text: String,
}
#[derive(Clone, Debug)]
pub(crate) struct TtsTextSender {
pub(super) text_tx: SyncSender<QueuedText>,
pub(super) generation: u64,
}
impl TtsTextSender {
pub(crate) fn send(&self, route_id: u64, text: String) -> Result<(), String> {
self.text_tx
.send(QueuedText {
generation: self.generation,
route_id,
text,
})
.map_err(|error| error.to_string())
}
}
pub(super) fn begin_voice_change(
selected_voice: &Mutex<String>,
voice_generation: &AtomicU64,
voice_cancel: &AtomicBool,
voice_change_ack: &VoiceChangeAck,
voice: &str,
) -> Option<tokio::sync::oneshot::Receiver<()>> {
let mut pending_ack = voice_change_ack
.lock()
.unwrap_or_else(|error| error.into_inner());
let mut selected = selected_voice
.lock()
.unwrap_or_else(|error| error.into_inner());
if selected.as_str() == voice {
return None;
}
let (sender, receiver) = tokio::sync::oneshot::channel();
voice_cancel.store(true, Ordering::Release);
let generation = voice_generation.fetch_add(1, Ordering::AcqRel) + 1;
if let Some(superseded) = pending_ack.replace(PendingVoiceChange {
generation,
acknowledged: sender,
}) {
let _ = superseded.acknowledged.send(());
}
*selected = voice.to_string();
Some(receiver)
}
pub(super) fn acknowledge_voice_change(
voice_change_ack: &VoiceChangeAck,
voice_cancel: &AtomicBool,
) {
let mut pending_ack = voice_change_ack
.lock()
.unwrap_or_else(|error| error.into_inner());
if voice_cancel.load(Ordering::Acquire) {
return;
}
if let Some(pending) = pending_ack.take() {
let _ = pending.acknowledged.send(());
}
}
pub(super) fn finish_voice_change_ack(voice_change_ack: &VoiceChangeAck) {
if let Some(pending) = voice_change_ack
.lock()
.unwrap_or_else(|error| error.into_inner())
.take()
{
let _ = pending.acknowledged.send(());
}
}
pub(super) fn reconcile_selected_voice(
model_dir: &Path,
selected_voice: &Mutex<String>,
voice_name: &mut String,
style: &mut VoiceStyle,
) -> bool {
let requested_voice = selected_voice
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
if requested_voice == *voice_name {
return true;
}
let requested_path = model_dir.join(format!("{requested_voice}.{VOICE_FILE_EXT}"));
match load_voice_style(&requested_path) {
Ok(requested_style) => {
*style = requested_style;
*voice_name = requested_voice;
true
}
Err(_) => {
eprintln!("buzz-desktop: tts stage=voice_switch status=fallback reason=voice_style");
let fallback_path = model_dir.join(format!("{DEFAULT_VOICE}.{VOICE_FILE_EXT}"));
match load_voice_style(&fallback_path) {
Ok(fallback_style) => {
*style = fallback_style;
*voice_name = DEFAULT_VOICE.to_string();
*selected_voice
.lock()
.unwrap_or_else(|lock_error| lock_error.into_inner()) =
DEFAULT_VOICE.to_string();
true
}
Err(_) => {
eprintln!(
"buzz-desktop: tts stage=voice_switch status=failed reason=fallback_voice_style"
);
false
}
}
}
}
}
pub(super) fn retain_cancelled_text(
deferred_text: &mut VecDeque<QueuedText>,
current_text: &mut Option<QueuedText>,
text_rx: &mpsc::Receiver<QueuedText>,
preserve_generation: Option<u64>,
) {
if let Some(generation) = preserve_generation {
deferred_text.retain(|text| {
let preserve = text.generation >= generation;
if !preserve {
log_cancelled_route(text.route_id, "voice_switch");
}
preserve
});
if let Some(text) = current_text.take() {
if text.generation >= generation {
deferred_text.push_front(text);
} else {
log_cancelled_route(text.route_id, "voice_switch");
}
}
while let Ok(text) = text_rx.try_recv() {
if text.generation >= generation {
deferred_text.push_back(text);
} else {
log_cancelled_route(text.route_id, "voice_switch");
}
}
} else {
for text in deferred_text.drain(..) {
log_cancelled_route(text.route_id, "barge_in");
}
if let Some(text) = current_text.take() {
log_cancelled_route(text.route_id, "barge_in");
}
while let Ok(text) = text_rx.try_recv() {
log_cancelled_route(text.route_id, "barge_in");
}
}
}
fn log_cancelled_route(route_id: u64, reason: &str) {
eprintln!("buzz-desktop: tts stage=queue status=dropped reason={reason} route_id={route_id}");
}
+16
View File
@@ -467,6 +467,18 @@ pub fn run() {
*guard = Some(app_handle.clone());
}
let (tts_settings, tts_settings_load_error) =
huddle::tts_settings::load_for_app(&app_handle);
if let Ok(mut guard) = state.huddle_audio.tts.lock() {
*guard = tts_settings.clone();
}
if let Ok(mut guard) = state.huddle_audio.tts_load_error.lock() {
*guard = tts_settings_load_error;
}
if let Ok(mut huddle) = state.huddle_state.lock() {
huddle.tts_enabled = tts_settings.agent_text_to_speech;
}
// Bring up the runtime-owned shared-compute coordinator before
// saved agents are restored. Its lifetime is tied to the app, not
// a UI mount; it publishes discovery and reconciles membership for
@@ -877,6 +889,10 @@ pub fn run() {
download_voice_models,
get_model_status,
set_tts_enabled,
huddle::tts_settings::get_tts_settings,
huddle::tts_settings::list_voice_registry,
huddle::tts_settings::set_pocket_voice,
huddle::tts_settings::preview_pocket_voice,
speak_agent_message,
add_agent_to_huddle,
check_pipeline_hotstart,
@@ -0,0 +1,247 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
classifySpeakableAgentText,
createInitialMembershipGate,
createLatestStateGate,
createOrderedSpeaker,
routeLiveAgentText,
} from "./ttsLiveMessages.ts";
const agents = new Set(["agent"]);
const CHANNEL = "active-huddle";
const base = {
id: "1",
kind: 9,
pubkey: "agent",
content: "Hello there",
tags: [["h", CHANNEL]],
};
const speakableText = (event, selfPubkey = "human") =>
classifySpeakableAgentText(event, agents, selfPubkey, CHANNEL).text;
test("speaks only new agent-authored text message events", () => {
assert.equal(speakableText(base), "Hello there");
assert.equal(
speakableText({ ...base, kind: 40002 }),
"Hello there",
"managed stream-message-v2 replies are spoken",
);
assert.equal(
speakableText({ ...base, kind: 7 }),
null,
"reactions and other event kinds are excluded",
);
assert.equal(
speakableText({ ...base, kind: 10 }),
null,
"edits and status events are excluded",
);
assert.equal(
speakableText({ ...base, pubkey: "human" }),
null,
"human-authored messages are excluded",
);
assert.equal(
speakableText({ ...base, content: " " }),
null,
"empty and non-text content are excluded",
);
assert.equal(
speakableText({ ...base, content: "K" }),
"K",
"one-character agent text remains speakable",
);
assert.equal(
speakableText({ ...base, content: "[System] tool started" }),
null,
"legacy system rows are excluded",
);
assert.equal(
speakableText({ ...base, tags: [["h", "another-huddle"]] }),
null,
"messages for another huddle are excluded",
);
});
test("routes managed stream-message-v2 through membership and enabled ordering", async () => {
const invoked = [];
const speaker = createOrderedSpeaker(async (text, routeId) => {
invoked.push({ text, routeId });
}, assert.fail);
assert.equal(
routeLiveAgentText(
{ ...base, kind: 40002 },
agents,
"human",
CHANNEL,
77,
speaker.enqueue,
),
"queued",
);
assert.equal(
routeLiveAgentText(
{ ...base, kind: 7 },
agents,
"human",
CHANNEL,
78,
speaker.enqueue,
),
"unsupported_kind",
);
assert.equal(
routeLiveAgentText(
{ ...base, tags: [["h", "wrong"]] },
agents,
"human",
CHANNEL,
79,
speaker.enqueue,
),
"h_tag_mismatch",
);
assert.equal(
routeLiveAgentText(
{ ...base, pubkey: "human" },
agents,
"human",
CHANNEL,
80,
speaker.enqueue,
),
"author_not_agent",
);
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(invoked, [{ text: "Hello there", routeId: 77 }]);
});
test("strips attachment markup and skips attachment-only events", () => {
const url = "https://cdn.example/voice.png";
const tags = [...base.tags, ["imeta", `url ${url}`, "m image/png"]];
assert.equal(
speakableText({ ...base, content: `![image](${url})`, tags }),
null,
);
assert.equal(
speakableText({
...base,
content: `Here is the diagram.\n\n![image](${url})`,
tags,
}),
"Here is the diagram.",
);
assert.equal(
speakableText({ ...base, content: `||\n![image](${url})\n||`, tags }),
null,
);
});
test("queues agent messages in live thread arrival order", async () => {
const spoken = [];
let releaseFirst;
const firstBlocked = new Promise((resolve) => {
releaseFirst = resolve;
});
const speaker = createOrderedSpeaker(async (text, routeId) => {
if (text === "first") await firstBlocked;
spoken.push([text, routeId]);
}, assert.fail);
speaker.enqueue("first", 41);
speaker.enqueue("second", 42);
await Promise.resolve();
assert.deepEqual(spoken, []);
releaseFirst();
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(spoken, [
["first", 41],
["second", 42],
]);
});
test("disabling cancels queued speech and rejects new messages until enabled", async () => {
const invoked = [];
const dropped = [];
let releaseFirst;
const firstBlocked = new Promise((resolve) => {
releaseFirst = resolve;
});
const speaker = createOrderedSpeaker(
async (text) => {
invoked.push(text);
if (text === "first") await firstBlocked;
},
assert.fail,
true,
(routeId, reason) => dropped.push([routeId, reason]),
);
speaker.enqueue("first", 51);
speaker.enqueue("queued-before-off", 52);
await Promise.resolve();
speaker.setEnabled(false);
speaker.enqueue("while-off");
releaseFirst();
await new Promise((resolve) => setTimeout(resolve, 0));
speaker.setEnabled(true);
speaker.enqueue("after-on");
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(invoked, ["first", "after-on"]);
assert.deepEqual(dropped, [[52, "disabled"]]);
});
test("does not speak before the native enabled state is known", async () => {
const invoked = [];
const speaker = createOrderedSpeaker(
async (text) => invoked.push(text),
assert.fail,
false,
);
speaker.enqueue("before-state");
await Promise.resolve();
speaker.setEnabled(true);
speaker.enqueue("after-state");
await new Promise((resolve) => setTimeout(resolve, 0));
assert.deepEqual(invoked, ["after-state"]);
});
test("a live TTS state event supersedes a delayed bootstrap result", () => {
const applied = [];
const gate = createLatestStateGate((enabled) => applied.push(enabled));
const applyBootstrap = gate.beginSnapshot();
gate.applyEvent(false);
applyBootstrap(true);
assert.deepEqual(applied, [false]);
});
test("buffers initial live events until membership resolves in order", () => {
const delivered = [];
const gate = createInitialMembershipGate((event) => delivered.push(event));
gate.push("first");
gate.push("second");
assert.deepEqual(delivered, []);
gate.succeed();
gate.push("third");
assert.deepEqual(delivered, ["first", "second", "third"]);
});
test("drops the initial buffer fail-closed when membership lookup fails", () => {
const delivered = [];
const dropped = [];
const gate = createInitialMembershipGate(
(event) => delivered.push(event),
(event) => dropped.push(event),
);
gate.push("unverified");
gate.fail();
gate.push("after-failure");
assert.deepEqual(delivered, ["after-failure"]);
assert.deepEqual(dropped, ["unverified"]);
});
@@ -0,0 +1,185 @@
import {
KIND_STREAM_MESSAGE,
KIND_STREAM_MESSAGE_V2,
} from "../../../shared/constants/kinds.ts";
export type LiveTtsEvent = {
id: string;
kind: number;
pubkey: string;
content: string;
tags: string[][];
};
export type LiveTtsEligibility =
| { text: string; reason: null }
| {
text: null;
reason:
| "unsupported_kind"
| "h_tag_mismatch"
| "author_not_agent"
| "self_authored"
| "empty_or_system";
};
export type LiveTtsRouteResult =
| "queued"
| "disabled"
| Exclude<LiveTtsEligibility, { text: string }>["reason"];
function textWithoutAttachments(event: LiveTtsEvent): string {
const urls = new Set(
event.tags
.filter((tag) => tag[0] === "imeta")
.flatMap((tag) =>
tag
.slice(1)
.filter((field) => field.startsWith("url "))
.map((field) => field.slice(4)),
),
);
if (urls.size === 0) return event.content;
const withoutMedia = event.content
.split("\n")
.filter(
(line) => !Array.from(urls).some((url) => line.includes(`](${url})`)),
)
.join("\n");
return withoutMedia.replace(
/(^|\n)\s*\|\|\s*\n(?:\s*\n)*\s*\|\|\s*(?=\n|$)/gu,
"$1",
);
}
export function classifySpeakableAgentText(
event: LiveTtsEvent,
agentPubkeys: ReadonlySet<string>,
selfPubkey: string | null,
channelId: string,
): LiveTtsEligibility {
if (
event.kind !== KIND_STREAM_MESSAGE &&
event.kind !== KIND_STREAM_MESSAGE_V2
)
return { text: null, reason: "unsupported_kind" };
if (!event.tags.some((tag) => tag[0] === "h" && tag[1] === channelId))
return { text: null, reason: "h_tag_mismatch" };
if (!agentPubkeys.has(event.pubkey))
return { text: null, reason: "author_not_agent" };
if (event.pubkey === selfPubkey)
return { text: null, reason: "self_authored" };
const content = textWithoutAttachments(event).trim();
if (content.length === 0 || content.startsWith("[System]"))
return { text: null, reason: "empty_or_system" };
return { text: content, reason: null };
}
/** Classify and enqueue one live event through the production routing seam. */
export function routeLiveAgentText(
event: LiveTtsEvent,
agentPubkeys: ReadonlySet<string>,
selfPubkey: string | null,
channelId: string,
routeId: number,
enqueue: (text: string, routeId: number) => "queued" | "disabled",
): LiveTtsRouteResult {
const eligibility = classifySpeakableAgentText(
event,
agentPubkeys,
selfPubkey,
channelId,
);
if (eligibility.text === null) return eligibility.reason;
return enqueue(eligibility.text, routeId);
}
/**
* Serialize native speak calls so live messages enter the bounded Pocket queue
* in thread arrival order even when the bridge resolves calls asynchronously.
*/
export function createOrderedSpeaker(
speak: (text: string, routeId: number) => Promise<void>,
onError: (error: unknown) => void,
initiallyEnabled = true,
onDrop: (routeId: number, reason: "disabled") => void = () => {},
): {
enqueue: (text: string, routeId?: number) => "queued" | "disabled";
setEnabled: (enabled: boolean) => void;
} {
let tail = Promise.resolve();
let enabled = initiallyEnabled;
let generation = 0;
return {
enqueue(text, routeId = 0) {
if (!enabled) return "disabled";
const queuedGeneration = generation;
tail = tail
.then(() => {
if (!enabled || generation !== queuedGeneration) {
onDrop(routeId, "disabled");
return;
}
return speak(text, routeId);
})
.catch(onError);
return "queued";
},
setEnabled(nextEnabled) {
if (!nextEnabled) generation += 1;
enabled = nextEnabled;
},
};
}
/** Ensure a delayed bootstrap snapshot cannot overwrite a newer live event. */
export function createLatestStateGate<T>(apply: (value: T) => void): {
applyEvent: (value: T) => void;
beginSnapshot: () => (value: T) => void;
} {
let revision = 0;
return {
applyEvent(value) {
revision += 1;
apply(value);
},
beginSnapshot() {
const snapshotRevision = revision;
return (value) => {
if (revision === snapshotRevision) apply(value);
};
},
};
}
/** Hold live events until the first authoritative agent-membership lookup. */
export function createInitialMembershipGate<T>(
deliver: (event: T) => void,
drop: (event: T) => void = () => {},
): {
push: (event: T) => void;
succeed: () => void;
fail: () => void;
} {
let settled = false;
let pending: T[] = [];
return {
push(event) {
if (settled) deliver(event);
else pending.push(event);
},
succeed() {
if (settled) return;
settled = true;
const buffered = pending;
pending = [];
for (const event of buffered) deliver(event);
},
fail() {
settled = true;
const dropped = pending;
pending = [];
for (const event of dropped) drop(event);
},
};
}
@@ -1,13 +1,28 @@
import { invoke } from "@tauri-apps/api/core";
import { listen } from "@tauri-apps/api/event";
import * as React from "react";
import { buildHuddleTtsLiveFilter } from "@/shared/api/relayChannelFilters";
import { relayClient } from "@/shared/api/relayClient";
import {
createInitialMembershipGate,
createLatestStateGate,
createOrderedSpeaker,
routeLiveAgentText,
} from "./ttsLiveMessages";
const AGENT_PUBKEY_REFRESH_INTERVAL_MS = 30_000;
let nextTtsRouteId = 1;
function allocateTtsRouteId(): number {
const routeId = nextTtsRouteId;
nextTtsRouteId += 1;
return routeId;
}
/**
* Subscribe to agent TTS messages on the ephemeral huddle channel.
* Pipes agent kind:9 messages to `speak_agent_message` on the Rust backend.
* Pipes new agent message events to `speak_agent_message` on the Rust backend.
*
* Extracted from HuddleContext to keep file sizes manageable.
*/
@@ -20,6 +35,8 @@ export function useTtsSubscription(
let disposed = false;
let cleanup: (() => void) | null = null;
let unlistenHuddleState: (() => void) | null = null;
let ttsStateKnown = false;
// ── Agent identity (authoritative, fail-closed) ───────────────────────
//
@@ -33,43 +50,152 @@ export function useTtsSubscription(
let agentsLoaded = false;
const agentPubkeys = new Set<string>();
async function loadAgentPubkeys() {
const speakInOrder = createOrderedSpeaker(
async (text, routeId) => {
if (!disposed) {
console.debug(
`[huddle] tts stage=invoke status=attempted route_id=${routeId}`,
);
try {
await invoke("speak_agent_message", { text, routeId });
console.debug(
`[huddle] tts stage=invoke status=accepted route_id=${routeId}`,
);
} catch (error) {
console.warn(
`[huddle] tts stage=invoke status=failed reason=native_error route_id=${routeId}`,
);
throw error;
}
}
},
() => {},
false,
(routeId, reason) => {
console.debug(
`[huddle] tts stage=queue status=dropped reason=${reason} route_id=${routeId}`,
);
},
);
const deliver = ({
event,
routeId,
}: {
event: Parameters<typeof routeLiveAgentText>[0];
routeId: number;
}) => {
if (disposed) return;
if (!agentsLoaded) {
console.debug(
`[huddle] tts stage=eligibility status=rejected reason=membership_unavailable route_id=${routeId}`,
);
return;
}
const result = routeLiveAgentText(
event,
agentPubkeys,
selfPubkeyRef.current,
ephemeralChannelId,
routeId,
speakInOrder.enqueue,
);
if (result === "queued") {
console.debug(
`[huddle] tts stage=eligibility status=accepted route_id=${routeId}`,
);
} else {
const reason =
result === "disabled" && !ttsStateKnown
? "tts_state_unknown"
: result;
console.debug(
`[huddle] tts stage=eligibility status=rejected reason=${reason} route_id=${routeId}`,
);
}
};
const initialMembershipGate = createInitialMembershipGate(
deliver,
({ routeId }) => {
console.debug(
`[huddle] tts stage=eligibility status=rejected reason=membership_unavailable route_id=${routeId}`,
);
},
);
async function loadAgentPubkeys(initial = false) {
try {
const pubkeys = await invoke<string[]>("get_huddle_agent_pubkeys");
if (disposed) return;
agentPubkeys.clear();
for (const pk of pubkeys) agentPubkeys.add(pk);
agentsLoaded = true;
if (initial) {
initialMembershipGate.succeed();
}
} catch (e) {
// Fail-closed on ALL failures, including refresh after prior success.
// Clear the set and mark as not loaded — TTS goes mute until the
// next successful refresh. Stale membership must never authorize speech.
agentPubkeys.clear();
agentsLoaded = false;
if (initial) {
initialMembershipGate.fail();
}
console.error("[huddle] Failed to load agent pubkeys:", e);
}
}
// Initial load + periodic refresh (catches mid-huddle agent additions).
void loadAgentPubkeys();
void loadAgentPubkeys(true);
const agentRefreshId = window.setInterval(() => {
void loadAgentPubkeys();
}, AGENT_PUBKEY_REFRESH_INTERVAL_MS);
// Install the state listener before requesting a snapshot. If a newer
// event arrives while IPC is pending, it supersedes the stale snapshot.
const ttsStateGate = createLatestStateGate<{ tts_enabled: boolean }>(
(state) => {
if (!disposed) {
ttsStateKnown = true;
speakInOrder.setEnabled(state.tts_enabled);
}
},
);
void listen<{ tts_enabled: boolean }>("huddle-state-changed", (event) => {
if (!disposed) ttsStateGate.applyEvent(event.payload);
})
.then((unlisten) => {
if (disposed) {
unlisten();
return;
}
unlistenHuddleState = unlisten;
const applyBootstrap = ttsStateGate.beginSnapshot();
void invoke<{ tts_enabled: boolean }>("get_huddle_state")
.then((state) => {
if (!disposed) applyBootstrap(state);
})
.catch((err) => {
console.warn("[huddle] Failed to load TTS state:", err);
});
})
.catch((err) => {
speakInOrder.setEnabled(false);
console.warn("[huddle] Failed to listen for TTS state:", err);
});
// ── Live-only subscription ───────────────────────────────────────────
// subscribeToChannelLive uses `since: now` — the relay never sends
// historical backlog. Every event delivered is a live message.
// A limit:0 subscription receives future message fan-out while the relay
// returns no stored rows, including pre-join rows from the current second.
// Event-ID dedup handles reconnect replay (same event arriving twice).
const seenEventIds = new Set<string>();
const seenOrder: string[] = [];
const MAX_SEEN_EVENTS = 5000;
relayClient
.subscribeToChannelLive(ephemeralChannelId, (event) => {
.subscribeLive(buildHuddleTtsLiveFilter(ephemeralChannelId), (event) => {
if (disposed) return;
// Defense-in-depth: subscription already filters to kind:9 only.
if (event.kind !== 9) return;
// Dedup by event ID (covers reconnect replay).
// Dedup by event ID if a relay repeats live fan-out.
if (seenEventIds.has(event.id)) return;
seenEventIds.add(event.id);
seenOrder.push(event.id);
@@ -78,20 +204,15 @@ export function useTtsSubscription(
if (oldest !== undefined) seenEventIds.delete(oldest);
}
// Fail-closed: don't speak until agent list is loaded.
if (!agentsLoaded) return;
// Only speak agent messages — skip human STT transcripts.
if (!agentPubkeys.has(event.pubkey)) return;
if (event.pubkey === selfPubkeyRef.current) return;
if (event.content.trim().length <= 1) return;
// Legacy: skip [System]-prefixed messages from before kind:48106.
if (event.content.startsWith("[System]")) return;
invoke("speak_agent_message", { text: event.content }).catch((err) => {
console.warn(
"[huddle] TTS speak failed (backpressure or pipeline unavailable):",
err,
// Preserve arrival order while the initial authoritative membership
// lookup is pending. A failed lookup clears this buffer fail-closed.
const routeId = allocateTtsRouteId();
if (!agentsLoaded) {
console.debug(
`[huddle] tts stage=eligibility status=deferred reason=membership_unavailable route_id=${routeId}`,
);
});
}
initialMembershipGate.push({ event, routeId });
})
.then((dispose) => {
if (disposed) {
@@ -106,7 +227,9 @@ export function useTtsSubscription(
return () => {
disposed = true;
speakInOrder.setEnabled(false);
cleanup?.();
unlistenHuddleState?.();
window.clearInterval(agentRefreshId);
};
}, [ephemeralChannelId, selfPubkeyRef]);
@@ -21,6 +21,7 @@ import {
SunMoon,
Ticket,
UserRound,
Volume2,
type LucideIcon,
} from "lucide-react";
import type {
@@ -83,10 +84,12 @@ import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup";
import { ProfileSettingsCard } from "./ProfileSettingsCard";
import { UpdateChecker } from "../UpdateChecker";
import { SettingsSectionHeader } from "./SettingsSectionHeader";
import { VoiceSettingsCard } from "./VoiceSettingsCard";
export type SettingsSection =
| "profile"
| "notifications"
| "voice"
| "experimental"
| "agents"
| "channel-templates"
@@ -106,6 +109,7 @@ export const DEFAULT_SETTINGS_SECTION: SettingsSection = "profile";
const SETTINGS_SECTION_VALUES: readonly SettingsSection[] = [
"profile",
"notifications",
"voice",
"experimental",
"agents",
"channel-templates",
@@ -167,6 +171,11 @@ export const settingsSections: SettingsSectionDescriptor[] = [
label: "Notifications",
icon: BellRing,
},
{
value: "voice",
label: "Voice",
icon: Volume2,
},
{
value: "experimental",
label: "Experiments",
@@ -807,6 +816,8 @@ export function renderSettingsSection(
onSetSoundForSlot={props.onSetSoundForSlot}
/>
);
case "voice":
return <VoiceSettingsCard />;
case "experimental":
return <ExperimentalFeaturesCard />;
case "agents":
@@ -58,6 +58,7 @@ const settingsNavGroups: Array<{
"profile",
"appearance",
"notifications",
"voice",
"shortcuts",
"custom-emoji",
"local-archive",
@@ -0,0 +1,256 @@
import * as React from "react";
import { ChevronDown, Play, Volume2 } from "lucide-react";
import { invokeTauri } from "@/shared/api/tauri";
import { cn } from "@/shared/lib/cn";
import { Button } from "@/shared/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuRadioGroup,
DropdownMenuRadioItem,
DropdownMenuTrigger,
} from "@/shared/ui/dropdown-menu";
import { Switch } from "@/shared/ui/switch";
import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup";
import { SettingsSectionHeader } from "./SettingsSectionHeader";
import {
selectedVoiceForBackend,
type VoiceRegistryEntry,
voiceOptionLabel,
voicesForBackend,
} from "./voiceSettingsLogic";
export type TtsSettings = {
version: number;
agentTextToSpeech: boolean;
voicePreferences: string[];
};
export function VoiceSettingsCard() {
const [settings, setSettings] = React.useState<TtsSettings | null>(null);
const [registry, setRegistry] = React.useState<VoiceRegistryEntry[]>([]);
const [busy, setBusy] = React.useState(false);
const [previewing, setPreviewing] = React.useState(false);
const [error, setError] = React.useState<string | null>(null);
React.useEffect(() => {
let disposed = false;
Promise.all([
invokeTauri<TtsSettings>("get_tts_settings"),
invokeTauri<VoiceRegistryEntry[]>("list_voice_registry"),
])
.then(([nextSettings, nextRegistry]) => {
if (!disposed) {
setSettings(nextSettings);
setRegistry(nextRegistry);
}
})
.catch((loadError) => {
if (!disposed) {
setError(
loadError instanceof Error
? loadError.message
: "Voice settings could not be loaded.",
);
}
});
return () => {
disposed = true;
};
}, []);
const saveEnabled = React.useCallback(async (enabled: boolean) => {
setBusy(true);
setError(null);
try {
const saved = await invokeTauri<TtsSettings>("set_tts_enabled", {
enabled,
});
setSettings(saved);
} catch (saveError) {
try {
const state = await invokeTauri<{ tts_enabled: boolean }>(
"get_huddle_state",
);
setSettings((current) =>
current
? { ...current, agentTextToSpeech: state.tts_enabled }
: current,
);
} catch {
// Keep the last confirmed state when native reconciliation is
// unavailable; the visible save error makes the failure explicit.
}
setError(
saveError instanceof Error
? saveError.message
: "Voice settings could not be saved.",
);
} finally {
setBusy(false);
}
}, []);
const savePocketVoice = React.useCallback(async (voiceKey: string) => {
setBusy(true);
setError(null);
try {
const saved = await invokeTauri<TtsSettings>("set_pocket_voice", {
voiceKey,
});
setSettings(saved);
} catch (saveError) {
setError(
saveError instanceof Error
? saveError.message
: "Voice settings could not be saved.",
);
} finally {
setBusy(false);
}
}, []);
const voices = voicesForBackend(registry, "pocket");
const selectedVoice = selectedVoiceForBackend(
settings?.voicePreferences ?? [],
voices,
);
const enabled = settings?.agentTextToSpeech ?? true;
const controlsDisabled = !settings || busy || !enabled;
return (
<section className="min-w-0" data-testid="settings-voice">
<SettingsSectionHeader
title="Voice"
description="Choose whether Buzz reads new agent responses aloud during an active huddle."
/>
<div className="flex flex-col gap-4">
<SettingsOptionGroup>
<SettingsOptionRow>
<div className="min-w-0">
<label
className="text-sm font-medium"
htmlFor="agent-text-to-speech-switch"
>
Agent text to speech
</label>
<p className="text-sm text-muted-foreground">
Read new agent messages aloud in the order they arrive.
</p>
</div>
<Switch
checked={enabled}
data-testid="agent-text-to-speech-toggle"
disabled={!settings || busy}
id="agent-text-to-speech-switch"
onCheckedChange={(checked) => {
if (settings) void saveEnabled(checked);
}}
/>
</SettingsOptionRow>
</SettingsOptionGroup>
<div
aria-disabled={!enabled}
className={cn(
"transition-opacity",
!enabled && "pointer-events-none opacity-45",
)}
data-testid="pocket-voice-controls"
>
<SettingsOptionGroup>
<SettingsOptionRow>
<div className="min-w-0">
<p className="text-sm font-medium">Pocket TTS voice</p>
<p className="text-sm text-muted-foreground">
Voice files stay private on this device.
</p>
</div>
<div className="flex shrink-0 items-center gap-2">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
aria-label={`Pocket TTS voice: ${selectedVoice?.displayName ?? "Mary"}`}
className="min-w-32 justify-between"
data-testid="pocket-voice-selector"
disabled={controlsDisabled}
variant="outline"
>
{selectedVoice
? voiceOptionLabel(selectedVoice, voices)
: "Mary"}
<ChevronDown className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent
align="end"
className="max-h-80 overflow-y-auto"
>
<DropdownMenuRadioGroup
onValueChange={(voiceKey) => {
if (settings) void savePocketVoice(voiceKey);
}}
value={selectedVoice?.key}
>
{voices.map((voice) => (
<DropdownMenuRadioItem
key={voice.key}
value={voice.key}
>
{voiceOptionLabel(voice, voices)}
</DropdownMenuRadioItem>
))}
</DropdownMenuRadioGroup>
</DropdownMenuContent>
</DropdownMenu>
<Button
aria-label={`Preview ${selectedVoice?.displayName ?? "Mary"}`}
data-testid="pocket-voice-preview"
disabled={controlsDisabled || previewing || !selectedVoice}
onClick={() => {
if (!selectedVoice) return;
setPreviewing(true);
setError(null);
void invokeTauri<void>("preview_pocket_voice", {
voiceKey: selectedVoice.key,
})
.catch((previewError) => {
setError(
previewError instanceof Error
? previewError.message
: "Voice preview could not be played.",
);
})
.finally(() => setPreviewing(false));
}}
size="sm"
variant="outline"
>
{previewing ? (
<Volume2 className="h-4 w-4 animate-pulse" />
) : (
<Play className="h-4 w-4" />
)}
Preview
</Button>
</div>
</SettingsOptionRow>
</SettingsOptionGroup>
</div>
{error && (
<p
className="text-sm text-destructive"
data-testid="voice-settings-error"
role="alert"
>
{error}
</p>
)}
</div>
</section>
);
}
@@ -0,0 +1,60 @@
import assert from "node:assert/strict";
import test from "node:test";
import {
selectedVoiceForBackend,
voiceOptionLabel,
voicesForBackend,
} from "./voiceSettingsLogic.ts";
const voice = (key, displayName, fallbackKey = "pocket:mary") => ({
key,
displayName,
backend: "pocket",
backendName: "Pocket TTS",
availability: "bundled",
fallbackKey,
referenceFile: `${key}.wav`,
provenance: {
source: "bundled",
contentHash: null,
license: null,
sourceUrl: null,
},
});
test("Pocket-only V1 filters the shared registry by backend", () => {
const registry = [
voice("pocket:mary", "Mary", null),
{ ...voice("siri:aaron", "Aaron"), backend: "siri" },
];
assert.deepEqual(
voicesForBackend(registry, "pocket").map((entry) => entry.key),
["pocket:mary"],
);
});
test("local selection uses the first compatible qualified preference", () => {
const voices = [
voice("pocket:mary", "Mary", null),
voice("pocket:eve", "Eve"),
];
assert.equal(
selectedVoiceForBackend(["siri:aaron", "pocket:eve", "pocket:mary"], voices)
?.key,
"pocket:eve",
);
});
test("duplicate display labels remain distinct by content-derived key", () => {
const voices = [
voice("pocket:imported:aaa", "Jim"),
voice("pocket:imported:bbb", "Jim"),
];
assert.equal(
selectedVoiceForBackend(["pocket:imported:bbb"], voices)?.key,
"pocket:imported:bbb",
);
assert.equal(voiceOptionLabel(voices[0], voices), "Jim · aaa");
assert.equal(voiceOptionLabel(voices[1], voices), "Jim · bbb");
});
@@ -0,0 +1,57 @@
export type VoiceAvailability =
| "bundled"
| "installed"
| "downloadable"
| "unavailable";
export type VoiceRegistryEntry = {
key: string;
displayName: string;
backend: string;
backendName: string;
availability: VoiceAvailability;
fallbackKey: string | null;
referenceFile: string | null;
provenance: {
source: string;
contentHash: string | null;
license: string | null;
sourceUrl: string | null;
};
};
export function voicesForBackend(
registry: readonly VoiceRegistryEntry[],
backend: string,
): VoiceRegistryEntry[] {
return registry.filter(
(voice) =>
voice.backend === backend &&
(voice.availability === "bundled" || voice.availability === "installed"),
);
}
export function selectedVoiceForBackend(
preferences: readonly string[],
voices: readonly VoiceRegistryEntry[],
): VoiceRegistryEntry | undefined {
for (const key of preferences) {
const voice = voices.find((candidate) => candidate.key === key);
if (voice) return voice;
}
return voices.find((voice) => voice.fallbackKey === null) ?? voices[0];
}
export function voiceOptionLabel(
voice: VoiceRegistryEntry,
voices: readonly VoiceRegistryEntry[],
): string {
const duplicateLabel = voices.some(
(candidate) =>
candidate.key !== voice.key &&
candidate.displayName === voice.displayName,
);
if (!duplicateLabel) return voice.displayName;
const identitySuffix = voice.key.split(":").at(-1)?.slice(-8) ?? voice.key;
return `${voice.displayName} · ${identitySuffix}`;
}
@@ -12,7 +12,7 @@ import {
AUTH_TIMEOUT_MS,
HISTORY_TIMEOUT_MS,
PUBLISH_TIMEOUT_MS,
} from "@/shared/api/relayClientSession";
} from "@/shared/api/relayClientTimings";
type PendingHistory = {
events: RelayEvent[];
@@ -6,6 +6,7 @@ import {
buildChannelAuxFilter,
buildChannelReactionAuxFilter,
buildChannelStructuralAuxFilter,
buildHuddleTtsLiveFilter,
} from "./relayChannelFilters.ts";
const CHANNEL = "36411e44-0e2d-4cfe-bd6e-567eb169db9f";
@@ -14,6 +15,14 @@ const IDS = [
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
];
test("huddle TTS filter is future-only for both message kinds", () => {
assert.deepEqual(buildHuddleTtsLiveFilter(CHANNEL), {
kinds: [9, 40002],
"#h": [CHANNEL],
limit: 0,
});
});
// Regression: reaction (kind:7) and reaction-removal (kind:5) events carry only
// an `e` tag, no channel `h` tag. An `#h`-scoped aux query never matches them,
// so removed historical reactions reappear. The aux filters must key on `#e`
@@ -6,6 +6,8 @@ import {
KIND_DELETION,
KIND_NIP29_DELETE_EVENT,
KIND_REACTION,
KIND_STREAM_MESSAGE,
KIND_STREAM_MESSAGE_V2,
KIND_STREAM_MESSAGE_EDIT,
} from "@/shared/constants/kinds";
import type { RelaySubscriptionFilter } from "@/shared/api/relayClientShared";
@@ -40,6 +42,17 @@ export function buildChannelFilter(
return filter;
}
/** Strictly live huddle message filter: zero stored rows, future messages only. */
export function buildHuddleTtsLiveFilter(
channelId: string,
): RelaySubscriptionFilter {
return {
kinds: [KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_V2],
"#h": [channelId],
limit: 0,
};
}
/**
* History filter for cold-load and scrollback: message kinds *only*, so the
* `limit` budget buys visible message depth. Auxiliary events (reactions,
+11 -12
View File
@@ -53,20 +53,19 @@ import {
} from "@/shared/api/relayReconnectPolicy";
import { RelayReconnectWaiters } from "@/shared/api/relayReconnectWaiters";
import { RelayStallWatchdog } from "@/shared/api/relayStallWatchdog";
import {
AUTH_TIMEOUT_MS,
BACKOFF_RESET_STABLE_MS,
EVENT_BATCH_MS,
HISTORY_TIMEOUT_MS,
PUBLISH_TIMEOUT_MS,
RECONNECT_BASE_DELAY_MS,
RECONNECT_MAX_DELAY_MS,
STALL_CHECK_INTERVAL_MS,
STALL_IDLE_TIMEOUT_MS,
} from "@/shared/api/relayClientTimings";
import { closeWebSocket } from "@/shared/api/relayWebSocketClose";
import { buildThreadReferenceTags } from "@/features/messages/lib/threading";
const RECONNECT_BASE_DELAY_MS = 1_000,
RECONNECT_MAX_DELAY_MS = 30_000,
EVENT_BATCH_MS = 16;
export const AUTH_TIMEOUT_MS = 25_000;
export const HISTORY_TIMEOUT_MS = 25_000;
export const PUBLISH_TIMEOUT_MS = 25_000;
export const BACKOFF_RESET_STABLE_MS = 60_000;
const STALL_CHECK_INTERVAL_MS = 10_000;
const STALL_IDLE_TIMEOUT_MS = 60_000;
export class RelayClient {
private wsId: number | null = null;
@@ -0,0 +1,20 @@
export const RECONNECT_BASE_DELAY_MS = 1_000;
export const RECONNECT_MAX_DELAY_MS = 30_000;
export const EVENT_BATCH_MS = 16;
/**
* Op-level timeouts tolerate degraded networks where TLS handshakes and DNS
* resolution can take several seconds.
*/
export const AUTH_TIMEOUT_MS = 25_000;
export const HISTORY_TIMEOUT_MS = 25_000;
export const PUBLISH_TIMEOUT_MS = 25_000;
/**
* A stability-gated reset prevents reconnect flapping from erasing backoff.
*/
export const BACKOFF_RESET_STABLE_MS = 60_000;
/** Passive liveness thresholds for the relay heartbeat stream. */
export const STALL_CHECK_INTERVAL_MS = 10_000;
export const STALL_IDLE_TIMEOUT_MS = 60_000;
@@ -5,6 +5,7 @@ import {
buildReconnectReplayFilter,
replayLiveSubscriptions,
REPLAY_BATCH_SIZE,
shouldPageReconnectReplay,
} from "./relayReconnectReplay.ts";
import { buildChannelFilter } from "./relayChannelFilters.ts";
@@ -113,6 +114,31 @@ test("reconnect replay caps large steady-state limits", () => {
});
});
test("reconnect replay preserves the live-only zero-history contract", () => {
const filter = {
kinds: [9],
"#h": ["channel-1"],
limit: 0,
};
assert.deepEqual(replayFilter(filter, 123), {
kinds: [9],
"#h": ["channel-1"],
limit: 0,
since: 123,
});
});
test("live-only subscriptions do not page reconnect history", () => {
const filter = {
kinds: [9],
"#h": ["channel-1"],
limit: 0,
};
assert.equal(shouldPageReconnectReplay(filter), false);
});
test("reconnect replay keeps the stricter existing since window", () => {
const filter = {
kinds: [9],
+160
View File
@@ -161,6 +161,11 @@ type MockHuddleSeed = {
type E2eConfig = {
mode?: "mock" | "relay";
mock?: {
ttsSettings?: {
version: number;
agentTextToSpeech: boolean;
voicePreferences: string[];
};
/** Advertised HEAD for the first mock project without adding that branch. */
projectHeadBranch?: string;
/** Builderlab account returned by hosted-community onboarding. Null/omitted = signed out. */
@@ -9954,6 +9959,161 @@ export function maybeInstallE2eTauriMocks() {
}
case "get_model_status":
return { stt: "ready", tts: "ready" };
case "get_tts_settings":
return (
activeConfig?.mock?.ttsSettings ?? {
version: 1,
agentTextToSpeech: true,
voicePreferences: ["pocket:mary"],
}
);
case "list_voice_registry":
return [
[
"anna",
"Anna",
"anna.wav",
"p228_023_enhanced.wav",
"0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856",
],
[
"vera",
"Vera",
"vera.wav",
"p229_023_enhanced.wav",
"309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b",
],
[
"fantine",
"Fantine",
"fantine.wav",
"p244_023_enhanced.wav",
"5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b",
],
[
"charles",
"Charles",
"charles.wav",
"p254_023_enhanced.wav",
"6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756",
],
[
"paul",
"Paul",
"paul.wav",
"p259_023_enhanced.wav",
"7aba504fe0b3b16478b69ed27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b",
],
[
"eponine",
"Eponine",
"eponine.wav",
"p262_023_enhanced.wav",
"a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b",
],
[
"azelma",
"Azelma",
"azelma.wav",
"p303_023_enhanced.wav",
"60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026",
],
[
"george",
"George",
"george.wav",
"p315_023_enhanced.wav",
"29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae",
],
[
"mary",
"Mary",
"reference_sample.wav",
"p333_023_enhanced.wav",
"a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f",
],
[
"jane",
"Jane",
"jane.wav",
"p339_023_enhanced.wav",
"2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a",
],
[
"michael",
"Michael",
"michael.wav",
"p360_023_enhanced.wav",
"b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad",
],
[
"eve",
"Eve",
"eve.wav",
"p361_023_enhanced.wav",
"396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd",
],
].map(
([id, displayName, referenceFile, upstreamFile, contentHash]) => ({
key: `pocket:${id}`,
displayName,
backend: "pocket",
backendName: "Pocket TTS",
availability: "bundled",
fallbackKey: id === "mary" ? null : "pocket:mary",
referenceFile,
provenance: {
source: "bundled",
contentHash,
license: "CC-BY-4.0",
sourceUrl: `https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/vctk/${upstreamFile}`,
},
}),
);
case "set_tts_enabled": {
const enabled = (payload as { enabled?: boolean })?.enabled;
if (typeof enabled !== "boolean")
throw new Error("Missing text-to-speech enabled state");
const settings = {
version: 1,
agentTextToSpeech: enabled,
voicePreferences: activeConfig?.mock?.ttsSettings
?.voicePreferences ?? ["pocket:mary"],
};
if (activeConfig) {
activeConfig.mock ??= {};
activeConfig.mock.ttsSettings = settings;
}
return settings;
}
case "set_pocket_voice": {
const voiceKey = (payload as { voiceKey?: string })?.voiceKey;
if (!voiceKey) throw new Error("Missing Pocket voice key");
const current = activeConfig?.mock?.ttsSettings ?? {
version: 1,
agentTextToSpeech: true,
voicePreferences: ["pocket:mary"],
};
const firstPocketIndex = current.voicePreferences.findIndex((key) =>
key.startsWith("pocket:"),
);
const preferences = current.voicePreferences.filter(
(key) => !key.startsWith("pocket:"),
);
preferences.splice(
firstPocketIndex < 0 ? preferences.length : firstPocketIndex,
0,
voiceKey,
);
const settings = { ...current, voicePreferences: preferences };
if (activeConfig) {
activeConfig.mock ??= {};
activeConfig.mock.ttsSettings = settings;
}
return settings;
}
case "preview_pocket_voice":
return null;
case "get_builderlab_auth":
return activeConfig?.mock?.builderlabAuth ?? null;
case "start_builderlab_login": {
+119
View File
@@ -0,0 +1,119 @@
import { expect, test } from "@playwright/test";
import { waitForAnimations } from "../helpers/animations";
import { installMockBridge } from "../helpers/bridge";
import { openSettings } from "../helpers/settings";
const SCREENSHOT_PATH = "test-results/voice-settings/pocket-voices.png";
test.describe("Pocket voice settings", () => {
test.use({ viewport: { width: 1100, height: 760 } });
test("selects and retains a bundled voice while text to speech is off", async ({
page,
}) => {
await installMockBridge(page);
await page.goto("/", { waitUntil: "domcontentloaded" });
await openSettings(page, "voice");
const card = page.getByTestId("settings-voice");
await expect(card).toBeVisible();
await expect(
page.getByText("Agent text to speech", { exact: true }),
).toBeVisible();
await expect(
page.getByText("Pocket TTS voice", { exact: true }),
).toBeVisible();
await expect(card).not.toContainText("April INT8");
await page.getByTestId("pocket-voice-selector").click();
await expect(page.getByRole("menuitemradio")).toHaveCount(12);
await page.getByRole("menuitemradio", { name: "Eve" }).click();
await expect(page.getByTestId("pocket-voice-selector")).toContainText(
"Eve",
);
await expect(
page.getByRole("button", { name: "Pocket TTS voice: Eve" }),
).toBeVisible();
await page.getByTestId("agent-text-to-speech-toggle").click();
await expect(
page.getByTestId("agent-text-to-speech-toggle"),
).toHaveAttribute("aria-checked", "false");
await expect(page.getByTestId("pocket-voice-controls")).toHaveAttribute(
"aria-disabled",
"true",
);
await expect(page.getByTestId("pocket-voice-selector")).toContainText(
"Eve",
);
const savedCommands = await page.evaluate(() =>
(window.__BUZZ_E2E_COMMAND_LOG__ ?? [])
.filter((entry) =>
["set_pocket_voice", "set_tts_enabled"].includes(entry.command),
)
.map((entry) => ({ command: entry.command, payload: entry.payload })),
);
expect(savedCommands).toEqual([
{
command: "set_pocket_voice",
payload: { voiceKey: "pocket:eve" },
},
{
command: "set_tts_enabled",
payload: { enabled: false },
},
]);
});
test("captures the complete VCTK preset settings surface", async ({
page,
}) => {
await installMockBridge(page, {
ttsSettings: {
version: 1,
agentTextToSpeech: true,
voicePreferences: ["pocket:eve"],
},
});
await page.goto("/", { waitUntil: "domcontentloaded" });
await openSettings(page, "voice");
const card = page.getByTestId("settings-voice");
await expect(card).toBeVisible();
await expect(page.getByTestId("pocket-voice-selector")).toContainText(
"Eve",
);
await page.getByTestId("pocket-voice-selector").click();
await expect(page.getByRole("menuitemradio")).toHaveCount(12);
const menu = page.getByRole("menu");
await expect(menu).toBeVisible();
await waitForAnimations(page);
const cardBox = await card.boundingBox();
const menuBox = await menu.boundingBox();
const viewport = page.viewportSize();
if (!cardBox || !menuBox || !viewport) {
throw new Error("Voice settings screenshot bounds are unavailable");
}
const x = Math.max(0, Math.min(cardBox.x, menuBox.x) - 16);
const y = Math.max(0, Math.min(cardBox.y, menuBox.y) - 16);
const right = Math.min(
viewport.width,
Math.max(cardBox.x + cardBox.width, menuBox.x + menuBox.width) + 16,
);
const bottom = Math.min(
viewport.height,
Math.max(cardBox.y + cardBox.height, menuBox.y + menuBox.height) + 16,
);
await page.screenshot({
path: SCREENSHOT_PATH,
clip: {
x: Math.floor(x),
y: Math.floor(y),
width: Math.ceil(right - x),
height: Math.ceil(bottom - y),
},
});
});
});
+5
View File
@@ -159,6 +159,11 @@ type MockInstallRuntimeResult = {
};
type MockBridgeOptions = {
ttsSettings?: {
version: number;
agentTextToSpeech: boolean;
voicePreferences: string[];
};
/** Advertised HEAD for the first mock project without adding that branch. */
projectHeadBranch?: string;
/** Relay NIP-11 identity used to sign authoritative repository state. */
+1
View File
@@ -3,6 +3,7 @@ import { expect, type Page } from "@playwright/test";
type SettingsSection =
| "profile"
| "notifications"
| "voice"
| "agents"
| "channel-templates"
| "compute"
+3
View File
@@ -84,6 +84,9 @@ run_unit_tests() {
run_test_step "buzz-auth unit tests" \
cargo test -p buzz-auth --lib -- --nocapture
run_test_step "buzz-voice tests" \
cargo test -p buzz-voice --lib -- --nocapture
run_test_step "buzz-cli tests" \
cargo test -p buzz-cli -- --nocapture