mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
Add pitch-preserving TTS playback speed
Signed-off-by: John Tennant <jtennant@squareup.com>
This commit is contained in:
John Tennant
committed by
John Tennant
John Tennant
parent
6186fc2313
commit
4e6570ac7d
Generated
+133
-14
@@ -1090,6 +1090,7 @@ dependencies = [
|
||||
"serde_yaml",
|
||||
"sha2 0.11.0",
|
||||
"sherpa-onnx",
|
||||
"ssstretch",
|
||||
"strip-ansi-escapes",
|
||||
"tar",
|
||||
"tauri",
|
||||
@@ -1531,6 +1532,17 @@ dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"termcolor",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
@@ -1549,7 +1561,7 @@ version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1787,7 +1799,7 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"windows 0.61.3",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2103,6 +2115,68 @@ dependencies = [
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.198"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fe442a792c7c736eea18b32a7f8a3b63cf8aafabda6760042dc2fdeda456291"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxx-build",
|
||||
"cxxbridge-cmd",
|
||||
"cxxbridge-flags",
|
||||
"cxxbridge-macro",
|
||||
"foldhash 0.2.0",
|
||||
"link-cplusplus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.198"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3184a94384c663718698311a78a51ac00c484c10b4eeac06fb0a068c5f64fa2"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
"indexmap 2.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"scratch",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-cmd"
|
||||
version = "1.0.198"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0148d8fd1199329ddf1d157a5e134e51ceff37c6a7ddd38615c399d81cb05d8d"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"codespan-reporting",
|
||||
"indexmap 2.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.198"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52850339faed2eaadd24e286dc1d8268cc6f8a7bd9524d713adc9099566b4c89"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.198"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c77c856545d886c9bd5215409ebb63b925e262135248b50c79e5a5f194ee47c"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.11"
|
||||
@@ -2210,7 +2284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090"
|
||||
dependencies = [
|
||||
"data-encoding",
|
||||
"syn 1.0.109",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3177,8 +3251,8 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"rustversion",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result 0.3.4",
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3923,7 +3997,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"windows-registry 0.5.3",
|
||||
"windows-registry 0.6.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3938,7 +4012,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core 0.61.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4773,6 +4847,15 @@ dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "link-cplusplus"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "link-section"
|
||||
version = "0.19.0"
|
||||
@@ -6327,7 +6410,7 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
dependencies = [
|
||||
"proc-macro-crate 1.3.1",
|
||||
"proc-macro-crate 3.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.118",
|
||||
@@ -6913,7 +6996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.45.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7672,7 +7755,7 @@ version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"heck 0.5.0",
|
||||
"itertools",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8870,6 +8953,12 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scratch"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2"
|
||||
|
||||
[[package]]
|
||||
name = "scrypt"
|
||||
version = "0.11.0"
|
||||
@@ -9651,6 +9740,16 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ssstretch"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ee31a0a494b76c8d3047aac804c5f4eb4b6e96c75414e3043b2212301bb8c6"
|
||||
dependencies = [
|
||||
"cxx",
|
||||
"cxx-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
@@ -9931,6 +10030,17 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sync_wrapper"
|
||||
version = "1.0.2"
|
||||
@@ -10536,7 +10646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -10551,6 +10661,15 @@ dependencies = [
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termina"
|
||||
version = "0.3.3"
|
||||
@@ -12226,7 +12345,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12855,8 +12974,8 @@ dependencies = [
|
||||
"log",
|
||||
"serde",
|
||||
"thiserror 2.0.18",
|
||||
"windows 0.61.3",
|
||||
"windows-core 0.61.2",
|
||||
"windows 0.62.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -121,6 +121,7 @@ regex = "1"
|
||||
rusqlite = { version = "0.37", features = ["bundled"] }
|
||||
axum = "0.8"
|
||||
rodio = "0.22"
|
||||
ssstretch = "0.1.0"
|
||||
earshot = "1.0"
|
||||
rubato = "3.0"
|
||||
audioadapter-buffers = "3.0"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
use std::time::Instant;
|
||||
|
||||
const SAMPLE_RATE: u32 = 24_000;
|
||||
|
||||
fn main() {
|
||||
let input: Vec<f32> = (0..SAMPLE_RATE * 10)
|
||||
.map(|index| {
|
||||
(2.0 * std::f32::consts::PI * 220.0 * index as f32 / SAMPLE_RATE as f32).sin() * 0.25
|
||||
})
|
||||
.collect();
|
||||
|
||||
for speed in [0.75_f32, 1.25, 1.5] {
|
||||
let mut processor = ssstretch::Stretch::new();
|
||||
processor.preset_default(1, SAMPLE_RATE as f32);
|
||||
let output_len = (input.len() as f32 / speed).round() as usize;
|
||||
let inputs = [input.clone()];
|
||||
let mut outputs = [Vec::with_capacity(output_len)];
|
||||
let latency_ms = processor.output_latency() as f64 * 1_000.0 / SAMPLE_RATE as f64;
|
||||
let started = Instant::now();
|
||||
processor.process_vec(&inputs, input.len() as i32, &mut outputs, output_len as i32);
|
||||
let elapsed = started.elapsed();
|
||||
println!(
|
||||
"{speed:.2}x: latency={latency_ms:.1}ms, CPU={:.2}ms for 10s ({:.3}% realtime), output={}",
|
||||
elapsed.as_secs_f64() * 1_000.0,
|
||||
elapsed.as_secs_f64() / 10.0 * 100.0,
|
||||
outputs[0].len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,7 @@ pub struct AppState {
|
||||
///
|
||||
/// Set once during `setup()` in `lib.rs`; never cleared.
|
||||
pub app_handle: Mutex<Option<AppHandle>>,
|
||||
pub tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl,
|
||||
/// Port of the localhost media streaming proxy (set during setup).
|
||||
pub media_proxy_port: AtomicU16,
|
||||
/// Set when identity resolution detected a "keyring-locked" state: the
|
||||
@@ -213,6 +214,7 @@ pub fn build_app_state() -> AppState {
|
||||
huddle_state: Mutex::new(HuddleState::default()),
|
||||
huddle_audio: Default::default(),
|
||||
app_handle: Mutex::new(None),
|
||||
tts_playback_speed: crate::huddle::playback_speed::PlaybackSpeedControl::default(),
|
||||
media_proxy_port: AtomicU16::new(0),
|
||||
prevent_sleep: Arc::new(Mutex::new(
|
||||
crate::prevent_sleep::PreventSleepState::default(),
|
||||
|
||||
@@ -29,6 +29,7 @@ pub mod audio_output;
|
||||
pub mod jitter;
|
||||
pub mod models;
|
||||
pub mod pipeline;
|
||||
pub mod playback_speed;
|
||||
pub mod playout;
|
||||
pub mod pocket;
|
||||
pub mod preprocessing;
|
||||
|
||||
@@ -234,6 +234,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
|
||||
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
|
||||
})?;
|
||||
let constructed_voice = initial_voice.clone();
|
||||
let playback_speed = state.tts_playback_speed.clone();
|
||||
let constructed = tokio::task::spawn_blocking(move || {
|
||||
tts::TtsPipeline::new_with_voice(
|
||||
model_dir,
|
||||
@@ -241,6 +242,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
|
||||
tts_cancel,
|
||||
&initial_voice,
|
||||
output_device,
|
||||
playback_speed,
|
||||
)
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
//! Pitch-preserving playback speed for generated speech.
|
||||
//!
|
||||
//! This stage sits between Pocket synthesis and rodio playback. It is
|
||||
//! deliberately independent of Pocket's model parameters, and it does not use
|
||||
//! rodio's speed control because rodio changes pitch and speed together.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicU32, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::{AppHandle, Manager, State};
|
||||
|
||||
use crate::app_state::AppState;
|
||||
|
||||
/// Slowest supported generated-speech playback speed.
|
||||
pub const MIN_PLAYBACK_SPEED: f32 = 0.75;
|
||||
/// Fastest supported generated-speech playback speed.
|
||||
pub const MAX_PLAYBACK_SPEED: f32 = 1.5;
|
||||
/// Default generated-speech playback speed.
|
||||
pub const DEFAULT_PLAYBACK_SPEED: f32 = 1.0;
|
||||
|
||||
const SETTINGS_FILE: &str = "tts-playback-settings.json";
|
||||
const UNITY_EPSILON: f32 = 0.000_1;
|
||||
|
||||
/// Lock-free shared control read by the TTS worker before each synthesis chunk.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PlaybackSpeedControl(Arc<AtomicU32>);
|
||||
|
||||
impl Default for PlaybackSpeedControl {
|
||||
fn default() -> Self {
|
||||
Self(Arc::new(AtomicU32::new(DEFAULT_PLAYBACK_SPEED.to_bits())))
|
||||
}
|
||||
}
|
||||
|
||||
impl PlaybackSpeedControl {
|
||||
/// Return the current generated-speech playback speed.
|
||||
pub fn get(&self) -> f32 {
|
||||
f32::from_bits(self.0.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
/// Update the in-memory speed after validation.
|
||||
pub fn set(&self, speed: f32) -> Result<(), String> {
|
||||
validate_speed(speed)?;
|
||||
self.0.store(speed.to_bits(), Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
struct PersistedPlaybackSettings {
|
||||
speed: f32,
|
||||
}
|
||||
|
||||
/// Load the global playback speed during app setup.
|
||||
pub fn load_playback_speed(app: &AppHandle, control: &PlaybackSpeedControl) -> Result<(), String> {
|
||||
let path = settings_path(app)?;
|
||||
let speed = load_from_path(&path)?;
|
||||
control.set(speed)
|
||||
}
|
||||
|
||||
/// Return the globally configured generated-speech playback speed.
|
||||
#[tauri::command]
|
||||
pub fn get_tts_playback_speed(state: State<'_, AppState>) -> f32 {
|
||||
state.tts_playback_speed.get()
|
||||
}
|
||||
|
||||
/// Persist and apply the global generated-speech playback speed.
|
||||
#[tauri::command]
|
||||
pub fn set_tts_playback_speed(
|
||||
speed: f32,
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
) -> Result<(), String> {
|
||||
validate_speed(speed)?;
|
||||
save_to_path(&settings_path(&app)?, speed)?;
|
||||
state.tts_playback_speed.set(speed)
|
||||
}
|
||||
|
||||
fn settings_path(app: &AppHandle) -> Result<PathBuf, String> {
|
||||
app.path()
|
||||
.app_data_dir()
|
||||
.map(|directory| directory.join(SETTINGS_FILE))
|
||||
.map_err(|error| format!("resolve TTS playback settings directory: {error}"))
|
||||
}
|
||||
|
||||
fn load_from_path(path: &Path) -> Result<f32, String> {
|
||||
let bytes = match std::fs::read(path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(DEFAULT_PLAYBACK_SPEED);
|
||||
}
|
||||
Err(error) => return Err(format!("read {}: {error}", path.display())),
|
||||
};
|
||||
let settings: PersistedPlaybackSettings = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("parse {}: {error}", path.display()))?;
|
||||
validate_speed(settings.speed)?;
|
||||
Ok(settings.speed)
|
||||
}
|
||||
|
||||
fn save_to_path(path: &Path, speed: f32) -> Result<(), String> {
|
||||
validate_speed(speed)?;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("create {}: {error}", parent.display()))?;
|
||||
}
|
||||
let payload = serde_json::to_vec_pretty(&PersistedPlaybackSettings { speed })
|
||||
.map_err(|error| format!("serialize TTS playback settings: {error}"))?;
|
||||
crate::managed_agents::storage::atomic_write_json(path, &payload)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ProcessorKind {
|
||||
Bypass,
|
||||
Signalsmith,
|
||||
}
|
||||
|
||||
/// Stateful streaming pitch-preserving processor.
|
||||
///
|
||||
/// Calls to [`Self::process`] preserve input order. The current Pocket path
|
||||
/// uses [`Self::process_complete_chunk`] because Pocket already hands the
|
||||
/// player one complete synthesis chunk; a future progressive synthesizer can
|
||||
/// feed deltas through `process` and pay only the one-time reported latency.
|
||||
pub struct PlaybackSpeedProcessor {
|
||||
inner: Processor,
|
||||
speed: f32,
|
||||
input_samples: usize,
|
||||
output_samples: usize,
|
||||
cancelled: bool,
|
||||
}
|
||||
|
||||
enum Processor {
|
||||
Bypass,
|
||||
Signalsmith(ssstretch::Stretch),
|
||||
}
|
||||
|
||||
impl PlaybackSpeedProcessor {
|
||||
/// Select bypass at 1x or Signalsmith Stretch at a non-unity speed.
|
||||
pub fn new(speed: f32, sample_rate: u32) -> Result<Self, String> {
|
||||
validate_speed(speed)?;
|
||||
let inner = if (speed - DEFAULT_PLAYBACK_SPEED).abs() <= UNITY_EPSILON {
|
||||
Processor::Bypass
|
||||
} else {
|
||||
let mut stretch = ssstretch::Stretch::new();
|
||||
stretch.preset_default(1, sample_rate as f32);
|
||||
Processor::Signalsmith(stretch)
|
||||
};
|
||||
Ok(Self {
|
||||
inner,
|
||||
speed,
|
||||
input_samples: 0,
|
||||
output_samples: 0,
|
||||
cancelled: false,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn kind(&self) -> ProcessorKind {
|
||||
match self.inner {
|
||||
Processor::Bypass => ProcessorKind::Bypass,
|
||||
Processor::Signalsmith(_) => ProcessorKind::Signalsmith,
|
||||
}
|
||||
}
|
||||
|
||||
/// One-time output-side algorithmic latency in samples.
|
||||
pub fn output_latency(&self) -> usize {
|
||||
match &self.inner {
|
||||
Processor::Bypass => 0,
|
||||
Processor::Signalsmith(stretch) => stretch.output_latency().max(0) as usize,
|
||||
}
|
||||
}
|
||||
|
||||
/// Process a progressive input delta without reordering prior deltas.
|
||||
pub fn process(&mut self, input: &[f32]) -> Result<Vec<f32>, String> {
|
||||
if input.is_empty() || self.cancelled {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
match &mut self.inner {
|
||||
Processor::Bypass => Ok(input.to_vec()),
|
||||
Processor::Signalsmith(stretch) => {
|
||||
self.input_samples = self.input_samples.saturating_add(input.len());
|
||||
let target_output =
|
||||
(self.input_samples as f64 / self.speed as f64).round() as usize;
|
||||
let output_len = target_output.saturating_sub(self.output_samples);
|
||||
let input_len = i32_len(input.len())?;
|
||||
let output_len_i32 = i32_len(output_len)?;
|
||||
let inputs = [input.to_vec()];
|
||||
let mut outputs = [Vec::with_capacity(output_len)];
|
||||
stretch.process_vec(&inputs, input_len, &mut outputs, output_len_i32);
|
||||
self.output_samples = target_output;
|
||||
Ok(std::mem::take(&mut outputs[0]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process one already-buffered Pocket chunk and compensate DSP pre-roll.
|
||||
///
|
||||
/// This returns exactly `input.len() / speed` samples and does not buffer
|
||||
/// any later Pocket chunk or the remainder of the response.
|
||||
pub fn process_complete_chunk(&mut self, input: &[f32]) -> Result<Vec<f32>, String> {
|
||||
if matches!(self.inner, Processor::Bypass) {
|
||||
return self.process(input);
|
||||
}
|
||||
|
||||
let expected = (input.len() as f64 / self.speed as f64).round() as usize;
|
||||
let latency = self.output_latency();
|
||||
let mut output = self.process(input)?;
|
||||
output.extend(self.drain()?);
|
||||
|
||||
let end = latency.saturating_add(expected);
|
||||
if output.len() < end {
|
||||
return Err(format!(
|
||||
"time stretcher produced {} samples, need {end}",
|
||||
output.len()
|
||||
));
|
||||
}
|
||||
Ok(output[latency..end].to_vec())
|
||||
}
|
||||
|
||||
/// Discard all pending processor output after barge-in.
|
||||
#[allow(dead_code)] // Used by progressive synthesis integrations; current Pocket chunks are atomic.
|
||||
pub fn cancel(&mut self) {
|
||||
self.cancelled = true;
|
||||
if let Processor::Signalsmith(stretch) = &mut self.inner {
|
||||
stretch.reset();
|
||||
}
|
||||
}
|
||||
|
||||
fn drain(&mut self) -> Result<Vec<f32>, String> {
|
||||
if self.cancelled {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let Processor::Signalsmith(stretch) = &mut self.inner else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let input_latency = stretch.input_latency().max(0) as usize;
|
||||
let output_latency = stretch.output_latency().max(0) as usize;
|
||||
let drain_output = (input_latency as f64 / self.speed as f64).ceil() as usize;
|
||||
|
||||
let inputs = [vec![0.0; input_latency]];
|
||||
let mut processed = [Vec::with_capacity(drain_output)];
|
||||
stretch.process_vec(
|
||||
&inputs,
|
||||
i32_len(input_latency)?,
|
||||
&mut processed,
|
||||
i32_len(drain_output)?,
|
||||
);
|
||||
let mut flushed = [Vec::with_capacity(output_latency)];
|
||||
stretch.flush_vec(&mut flushed, i32_len(output_latency)?);
|
||||
processed[0].extend_from_slice(&flushed[0]);
|
||||
Ok(std::mem::take(&mut processed[0]))
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate a generated-speech playback speed.
|
||||
pub fn validate_speed(speed: f32) -> Result<(), String> {
|
||||
if speed.is_finite() && (MIN_PLAYBACK_SPEED..=MAX_PLAYBACK_SPEED).contains(&speed) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"Speech playback speed must be between {MIN_PLAYBACK_SPEED} and {MAX_PLAYBACK_SPEED}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn i32_len(length: usize) -> Result<i32, String> {
|
||||
i32::try_from(length).map_err(|_| "audio chunk is too large to process".to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_RATE: u32 = 24_000;
|
||||
|
||||
#[test]
|
||||
fn selects_bypass_only_at_unity() {
|
||||
assert_eq!(
|
||||
PlaybackSpeedProcessor::new(1.0, SAMPLE_RATE)
|
||||
.expect("bypass")
|
||||
.kind(),
|
||||
ProcessorKind::Bypass
|
||||
);
|
||||
assert_eq!(
|
||||
PlaybackSpeedProcessor::new(1.25, SAMPLE_RATE)
|
||||
.expect("stretcher")
|
||||
.kind(),
|
||||
ProcessorKind::Signalsmith
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunked_processing_preserves_length_pitch_and_order() {
|
||||
let input: Vec<f32> = (0..24_000)
|
||||
.map(|sample| {
|
||||
let frequency = if sample < 12_000 { 220.0 } else { 440.0 };
|
||||
(2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin()
|
||||
})
|
||||
.collect();
|
||||
let mut processor = PlaybackSpeedProcessor::new(1.25, SAMPLE_RATE).expect("processor");
|
||||
let output = processor
|
||||
.process_complete_chunk(&input)
|
||||
.expect("complete output");
|
||||
|
||||
assert_eq!(output.len(), 19_200);
|
||||
let first_frequency = zero_crossing_frequency(&output[2_000..7_000], SAMPLE_RATE);
|
||||
let second_frequency = zero_crossing_frequency(&output[12_000..17_000], SAMPLE_RATE);
|
||||
assert!(
|
||||
(first_frequency - 220.0).abs() < 8.0,
|
||||
"first segment measured {first_frequency} Hz"
|
||||
);
|
||||
assert!(
|
||||
(second_frequency - 440.0).abs() <= 10.0,
|
||||
"second segment measured {second_frequency} Hz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_discards_subsequent_chunks_and_tail() {
|
||||
let mut processor = PlaybackSpeedProcessor::new(1.25, SAMPLE_RATE).expect("processor");
|
||||
assert!(!processor
|
||||
.process(&vec![0.25; 4_800])
|
||||
.expect("first chunk")
|
||||
.is_empty());
|
||||
processor.cancel();
|
||||
assert!(processor
|
||||
.process(&vec![0.5; 4_800])
|
||||
.expect("cancelled chunk")
|
||||
.is_empty());
|
||||
assert!(processor.drain().expect("cancelled tail").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn processor_preserves_sine_pitch() {
|
||||
let frequency = 220.0_f32;
|
||||
let input: Vec<f32> = (0..SAMPLE_RATE * 2)
|
||||
.map(|sample| {
|
||||
(2.0 * std::f32::consts::PI * frequency * sample as f32 / SAMPLE_RATE as f32).sin()
|
||||
})
|
||||
.collect();
|
||||
let mut processor = PlaybackSpeedProcessor::new(1.5, SAMPLE_RATE).expect("processor");
|
||||
let output = processor
|
||||
.process_complete_chunk(&input)
|
||||
.expect("complete output");
|
||||
let measured = zero_crossing_frequency(&output[2_000..], SAMPLE_RATE);
|
||||
assert!(
|
||||
(measured - frequency).abs() < 3.0,
|
||||
"expected {frequency} Hz, measured {measured} Hz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_unity_latency_stays_below_75_ms() {
|
||||
let processor = PlaybackSpeedProcessor::new(1.25, SAMPLE_RATE).expect("processor");
|
||||
let latency_ms = processor.output_latency() as f64 * 1_000.0 / SAMPLE_RATE as f64;
|
||||
assert!(latency_ms <= 75.0, "algorithmic latency was {latency_ms}ms");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persisted_speed_round_trips_and_rejects_invalid_values() {
|
||||
let directory = tempfile::tempdir().expect("tempdir");
|
||||
let path = directory.path().join(SETTINGS_FILE);
|
||||
save_to_path(&path, 1.25).expect("save");
|
||||
assert_eq!(load_from_path(&path).expect("load"), 1.25);
|
||||
|
||||
std::fs::write(&path, br#"{"speed":2.0}"#).expect("invalid fixture");
|
||||
assert!(load_from_path(&path).is_err());
|
||||
}
|
||||
|
||||
fn zero_crossing_frequency(samples: &[f32], sample_rate: u32) -> f32 {
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| pair[0] <= 0.0 && pair[1] > 0.0)
|
||||
.count();
|
||||
crossings as f32 * sample_rate as f32 / samples.len() as f32
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,7 @@ use std::{
|
||||
use super::pocket::{
|
||||
load_text_to_speech, load_voice_style, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT,
|
||||
};
|
||||
use super::playback_speed::{PlaybackSpeedControl, PlaybackSpeedProcessor};
|
||||
use super::preprocessing::{preprocess_for_tts, split_sentences};
|
||||
|
||||
#[path = "tts_voice_transition.rs"]
|
||||
@@ -167,6 +168,7 @@ impl TtsPipeline {
|
||||
cancel: Arc<AtomicBool>,
|
||||
voice: &str,
|
||||
output_device: Option<String>,
|
||||
playback_speed: PlaybackSpeedControl,
|
||||
) -> Result<Self, String> {
|
||||
let (text_tx, text_rx) = mpsc::sync_channel::<QueuedText>(TEXT_QUEUE_DEPTH);
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
@@ -204,6 +206,7 @@ impl TtsPipeline {
|
||||
),
|
||||
output_device,
|
||||
startup_tx,
|
||||
playback_speed,
|
||||
)
|
||||
})
|
||||
.map_err(|e| format!("failed to spawn tts-worker thread: {e}"))?;
|
||||
@@ -310,6 +313,7 @@ fn tts_worker(
|
||||
control_state: WorkerControlState,
|
||||
output_device: Option<String>,
|
||||
startup_tx: mpsc::SyncSender<Result<(), String>>,
|
||||
playback_speed: PlaybackSpeedControl,
|
||||
) {
|
||||
let (selected_voice, voice_generation, voice_change_ack) = voice_state;
|
||||
let (tts_active, shutdown, cancel_signals) = control_state;
|
||||
@@ -754,6 +758,20 @@ fn tts_worker(
|
||||
}
|
||||
match synthesis {
|
||||
Ok(samples) if !samples.is_empty() => {
|
||||
let speed = playback_speed.get();
|
||||
let samples = match PlaybackSpeedProcessor::new(speed, SAMPLE_RATE)
|
||||
.and_then(|mut processor| {
|
||||
processor.process_complete_chunk(&samples)
|
||||
}) {
|
||||
Ok(processed) => processed,
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"buzz-desktop: TTS playback-speed processing failed at \
|
||||
{speed:.2}x: {error}; using 1x playback"
|
||||
);
|
||||
samples
|
||||
}
|
||||
};
|
||||
if let Some(prepared) = playback_audio.push(
|
||||
samples,
|
||||
chunk_index,
|
||||
|
||||
@@ -548,6 +548,7 @@ pub async fn preview_pocket_voice(
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.clone();
|
||||
let playback_speed = state.tts_playback_speed.clone();
|
||||
let voice_name = voice
|
||||
.reference_file
|
||||
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
|
||||
@@ -561,6 +562,7 @@ pub async fn preview_pocket_voice(
|
||||
cancel,
|
||||
&voice_name,
|
||||
output_device,
|
||||
playback_speed,
|
||||
)?;
|
||||
pipeline.speak("Hello! This is how I’ll read agent responses.".to_string())?;
|
||||
let started = std::time::Instant::now();
|
||||
|
||||
@@ -41,6 +41,7 @@ use deep_link::{
|
||||
use huddle::audio_output::{
|
||||
get_audio_output_device, list_audio_output_devices, set_audio_output_device,
|
||||
};
|
||||
use huddle::playback_speed::{get_tts_playback_speed, load_playback_speed, set_tts_playback_speed};
|
||||
use huddle::reconnect::reconnect_huddle_audio;
|
||||
use huddle::{
|
||||
add_agent_to_huddle, check_pipeline_hotstart, confirm_huddle_active, download_voice_models,
|
||||
@@ -402,6 +403,9 @@ pub fn run() {
|
||||
// that will be lost on restart, as that silently breaks channel
|
||||
// memberships, DMs, and relay identity.
|
||||
let state = app_handle.state::<AppState>();
|
||||
if let Err(error) = load_playback_speed(&app_handle, &state.tts_playback_speed) {
|
||||
eprintln!("buzz-desktop: failed to load TTS playback speed; using 1x: {error}");
|
||||
}
|
||||
if let Err(e) = resolve_persisted_identity(&app_handle, &state) {
|
||||
eprintln!("buzz-desktop: fatal: identity resolution failed: {e}");
|
||||
std::process::exit(1);
|
||||
@@ -885,6 +889,8 @@ pub fn run() {
|
||||
list_audio_output_devices,
|
||||
set_audio_output_device,
|
||||
get_audio_output_device,
|
||||
get_tts_playback_speed,
|
||||
set_tts_playback_speed,
|
||||
start_pairing,
|
||||
confirm_pairing_sas,
|
||||
cancel_pairing,
|
||||
|
||||
@@ -81,6 +81,7 @@ import { PreventSleepSettingsCard } from "./PreventSleepSettingsCard";
|
||||
import { AgentDefaultsSettingsCard } from "./AgentDefaultsSettingsCard";
|
||||
import { HostedCommunitiesSettingsCard } from "./HostedCommunitiesSettingsCard";
|
||||
import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup";
|
||||
import { SpeechPlaybackSettings } from "./SpeechPlaybackSettings";
|
||||
import { ProfileSettingsCard } from "./ProfileSettingsCard";
|
||||
import { UpdateChecker } from "../UpdateChecker";
|
||||
import { SettingsSectionHeader } from "./SettingsSectionHeader";
|
||||
@@ -655,6 +656,7 @@ function ThemeSettingsCard() {
|
||||
</AnimatePresence>
|
||||
)}
|
||||
|
||||
<SpeechPlaybackSettings />
|
||||
<ThreadLayoutSetting />
|
||||
</section>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { getTtsPlaybackSpeed, setTtsPlaybackSpeed } from "@/shared/api/tauri";
|
||||
import { Button } from "@/shared/ui/button";
|
||||
import { SettingsOptionGroup, SettingsOptionRow } from "./SettingsOptionGroup";
|
||||
|
||||
const DEFAULT_SPEED = 1;
|
||||
const MIN_SPEED = 0.75;
|
||||
const MAX_SPEED = 1.5;
|
||||
const SPEED_STEP = 0.05;
|
||||
|
||||
export function SpeechPlaybackSettings() {
|
||||
const [speed, setSpeed] = useState(DEFAULT_SPEED);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const committedSpeed = useRef(DEFAULT_SPEED);
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
getTtsPlaybackSpeed()
|
||||
.then((savedSpeed) => {
|
||||
if (active) {
|
||||
committedSpeed.current = savedSpeed;
|
||||
setSpeed(savedSpeed);
|
||||
}
|
||||
})
|
||||
.catch((cause) => {
|
||||
if (active) setError(String(cause));
|
||||
});
|
||||
return () => {
|
||||
active = false;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const commitSpeed = async (nextSpeed: number) => {
|
||||
const previous = committedSpeed.current;
|
||||
if (nextSpeed === previous) return;
|
||||
committedSpeed.current = nextSpeed;
|
||||
setSpeed(nextSpeed);
|
||||
setError(null);
|
||||
try {
|
||||
await setTtsPlaybackSpeed(nextSpeed);
|
||||
} catch (cause) {
|
||||
if (committedSpeed.current === nextSpeed) {
|
||||
committedSpeed.current = previous;
|
||||
setSpeed(previous);
|
||||
setError(String(cause));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<SettingsOptionGroup className="mt-8">
|
||||
<SettingsOptionRow className="items-start">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div>
|
||||
<label
|
||||
className="text-sm font-medium"
|
||||
htmlFor="speech-playback-speed"
|
||||
>
|
||||
Speech playback speed
|
||||
</label>
|
||||
<p className="text-sm font-normal text-muted-foreground">
|
||||
Changes generated speech playback without changing voice pitch.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<span
|
||||
aria-live="polite"
|
||||
className="min-w-10 text-right text-sm tabular-nums"
|
||||
>
|
||||
{speed.toFixed(2)}x
|
||||
</span>
|
||||
<Button
|
||||
disabled={speed === DEFAULT_SPEED}
|
||||
onClick={() => commitSpeed(DEFAULT_SPEED)}
|
||||
size="sm"
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
Reset
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<input
|
||||
aria-valuetext={`${speed.toFixed(2)} times`}
|
||||
className="mt-3 w-full accent-primary"
|
||||
id="speech-playback-speed"
|
||||
max={MAX_SPEED}
|
||||
min={MIN_SPEED}
|
||||
onBlur={(event) => commitSpeed(Number(event.currentTarget.value))}
|
||||
onChange={(event) => setSpeed(Number(event.currentTarget.value))}
|
||||
onKeyUp={(event) => commitSpeed(Number(event.currentTarget.value))}
|
||||
onPointerUp={(event) =>
|
||||
commitSpeed(Number(event.currentTarget.value))
|
||||
}
|
||||
step={SPEED_STEP}
|
||||
type="range"
|
||||
value={speed}
|
||||
/>
|
||||
{error ? (
|
||||
<p className="mt-1 text-xs text-destructive" role="alert">
|
||||
Could not save playback speed: {error}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
</SettingsOptionRow>
|
||||
</SettingsOptionGroup>
|
||||
);
|
||||
}
|
||||
@@ -1197,6 +1197,14 @@ export const setPreventSleepActive = (active: boolean) =>
|
||||
export const setAgentManagedProfiles = (enabled: boolean) =>
|
||||
invokeTauri("set_agent_managed_profiles", { enabled });
|
||||
|
||||
export function getTtsPlaybackSpeed(): Promise<number> {
|
||||
return invokeTauri<number>("get_tts_playback_speed");
|
||||
}
|
||||
|
||||
export function setTtsPlaybackSpeed(speed: number): Promise<void> {
|
||||
return invokeTauri<void>("set_tts_playback_speed", { speed });
|
||||
}
|
||||
|
||||
/** Returns true on macOS, Windows, and Linux AppImage installs.
|
||||
* Returns false on Linux non-AppImage packages (e.g. .deb) where
|
||||
* Tauri's updater cannot swap the binary. */
|
||||
|
||||
Reference in New Issue
Block a user