feat(desktop): import local Pocket voices

Signed-off-by: John Tennant <jtennant@squareup.com>
This commit is contained in:
John Tennant
2026-07-29 15:23:52 -04:00
parent 6186fc2313
commit 49c03afd7f
9 changed files with 1018 additions and 145 deletions
+1
View File
@@ -39,6 +39,7 @@ pub mod stt;
pub mod transcription;
pub mod tts;
pub mod tts_settings;
mod tts_voice_import;
mod tts_voice_registry;
pub mod wire;
+45 -18
View File
@@ -195,6 +195,40 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
None => return Ok(false),
};
// Avoid resolving and hashing imported voice files on every hot-start poll
// when TTS is already disabled or running. The guarded claim below repeats
// these checks after the fallible work to close the race.
{
let huddle = state.huddle()?;
if huddle.tts_pipeline.is_some() || !huddle.tts_enabled {
return Ok(false);
}
}
// Resolve all fallible construction inputs before claiming the sentinel so
// an unreadable optional voice registry cannot wedge future start attempts.
let output_device = state
.huddle_audio
.output_device
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
let app = state
.app_handle
.lock()
.map_err(|error| format!("app handle lock poisoned: {error}"))?
.clone();
let voice_preferences = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| settings.voice_preferences.clone())?;
let initial_voice = match app {
Some(app) => super::tts_settings::pocket_voice_reference(&app, &voice_preferences)?,
None => super::tts_settings::bundled_pocket_voice_reference(&voice_preferences),
};
// Atomically check preconditions and claim the construction slot.
// The sentinel prevents a second caller from starting construction
// while we're building outside the lock.
@@ -219,20 +253,6 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
// Construct outside the lock — this spawns the TTS worker thread and
// loads ONNX sessions (~200ms). If this fails, clear the sentinel.
let output_device = state
.huddle_audio
.output_device
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
let initial_voice = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| {
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
})?;
let constructed_voice = initial_voice.clone();
let constructed = tokio::task::spawn_blocking(move || {
tts::TtsPipeline::new_with_voice(
@@ -316,14 +336,21 @@ fn finalize_tts_pipeline_start(
{
return Ok(false);
}
let voice = state
let app = state
.app_handle
.lock()
.map_err(|error| format!("app handle lock poisoned: {error}"))?
.clone();
let preferences = state
.huddle_audio
.tts
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))
.map(|settings| {
super::tts_settings::pocket_voice_name(&settings.voice_preferences).to_string()
})?;
.map(|settings| settings.voice_preferences.clone())?;
let voice = match app {
Some(app) => super::tts_settings::pocket_voice_reference(&app, &preferences)?,
None => super::tts_settings::bundled_pocket_voice_reference(&preferences),
};
publish(&voice, &mut huddle);
Ok(true)
}
+165 -23
View File
@@ -86,7 +86,7 @@ pub type VoicePreferences = Vec<String>;
/// V1 contains Pocket entries only. Siri, Kokoro, imported voices, and
/// per-agent assignment can add entries or reuse the preference type without
/// changing the registry/settings boundary.
pub fn voice_registry() -> Vec<VoiceRegistryEntry> {
pub fn bundled_voice_registry() -> Vec<VoiceRegistryEntry> {
POCKET_VOICES
.iter()
.map(|voice| VoiceRegistryEntry {
@@ -107,6 +107,33 @@ pub fn voice_registry() -> Vec<VoiceRegistryEntry> {
.collect()
}
pub fn voice_registry(app: &AppHandle) -> Vec<VoiceRegistryEntry> {
let mut registry = bundled_voice_registry();
match super::tts_voice_import::load_registry(app) {
Ok(imported) => registry.extend(imported.into_iter().map(|voice| VoiceRegistryEntry {
key: voice.key,
display_name: voice.display_name,
backend: POCKET_BACKEND_ID.to_string(),
backend_name: "Pocket TTS".to_string(),
availability: VOICE_AVAILABILITY_INSTALLED.to_string(),
fallback_key: Some(MARY_VOICE_KEY.to_string()),
reference_file: Some(voice.file_name),
provenance: VoiceProvenance {
source: "local import".to_string(),
content_hash: Some(voice.content_hash),
license: None,
source_url: None,
},
})),
Err(error) => {
eprintln!(
"buzz-desktop: {error}; imported Pocket voices are unavailable for this session"
);
}
}
registry
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct TtsSettings {
@@ -125,8 +152,10 @@ impl Default for TtsSettings {
}
}
pub fn voice_by_key(key: &str) -> Option<VoiceRegistryEntry> {
voice_registry().into_iter().find(|voice| voice.key == key)
pub fn voice_by_key(app: &AppHandle, key: &str) -> Option<VoiceRegistryEntry> {
voice_registry(app)
.into_iter()
.find(|voice| voice.key == key)
}
fn is_qualified_voice_key(key: &str) -> bool {
@@ -141,11 +170,19 @@ fn is_locally_available(availability: &str) -> bool {
)
}
#[cfg(test)]
pub fn resolve_voice_for_backend(
preferences: &[String],
backend: &str,
) -> Result<VoiceRegistryEntry, String> {
let registry = voice_registry();
resolve_voice_for_backend_in_registry(preferences, backend, &bundled_voice_registry())
}
fn resolve_voice_for_backend_in_registry(
preferences: &[String],
backend: &str,
registry: &[VoiceRegistryEntry],
) -> Result<VoiceRegistryEntry, String> {
preferences
.iter()
.filter_map(|key| registry.iter().find(|voice| voice.key == *key))
@@ -161,14 +198,31 @@ pub fn resolve_voice_for_backend(
.ok_or_else(|| format!("No locally available fallback voice for backend {backend}"))
}
pub fn pocket_voice_name(preferences: &[String]) -> String {
resolve_voice_for_backend(preferences, POCKET_BACKEND_ID)
pub fn bundled_pocket_voice_reference(preferences: &[String]) -> String {
resolve_voice_for_backend_in_registry(preferences, POCKET_BACKEND_ID, &bundled_voice_registry())
.ok()
.and_then(|voice| voice.reference_file)
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
.unwrap_or_else(|| DEFAULT_VOICE.to_string())
}
pub fn pocket_voice_reference(app: &AppHandle, preferences: &[String]) -> Result<String, String> {
let registry = voice_registry(app);
let voice = resolve_voice_for_backend_in_registry(preferences, POCKET_BACKEND_ID, &registry)?;
if voice.key.starts_with("pocket:imported:") {
let imported = super::tts_voice_import::load_registry(app)?
.into_iter()
.find(|candidate| candidate.key == voice.key)
.ok_or_else(|| format!("Imported voice {} is unavailable", voice.display_name))?;
return super::tts_voice_import::resolve_file(app, &imported)
.map(|path| path.to_string_lossy().into_owned());
}
Ok(voice
.reference_file
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
.unwrap_or_else(|| DEFAULT_VOICE.to_string()))
}
pub(crate) fn settings_path(app: &AppHandle) -> Result<PathBuf, String> {
app.path()
.app_data_dir()
@@ -290,8 +344,8 @@ pub fn get_tts_settings(state: State<'_, AppState>) -> Result<TtsSettings, Strin
}
#[tauri::command]
pub fn list_voice_registry() -> Vec<VoiceRegistryEntry> {
voice_registry()
pub fn list_voice_registry(app: AppHandle) -> Vec<VoiceRegistryEntry> {
voice_registry(&app)
}
fn ensure_settings_writable(state: &AppState) -> Result<(), String> {
@@ -394,8 +448,8 @@ async fn apply_tts_settings(
if settings.agent_text_to_speech {
let (active, voice_change_ack) = {
let mut huddle = state.huddle()?;
let voice_change_ack =
enable_tts_runtime(&mut huddle, &pocket_voice_name(&settings.voice_preferences));
let voice_reference = pocket_voice_reference(app, &settings.voice_preferences)?;
let voice_change_ack = enable_tts_runtime(&mut huddle, &voice_reference);
(
matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active),
voice_change_ack,
@@ -479,10 +533,22 @@ pub async fn set_tts_enabled(
}
fn settings_with_pocket_voice(
settings: TtsSettings,
voice_key: &str,
app: &AppHandle,
) -> Result<TtsSettings, String> {
settings_with_pocket_voice_from_registry(settings, voice_key, &voice_registry(app))
}
fn settings_with_pocket_voice_from_registry(
mut settings: TtsSettings,
voice_key: &str,
registry: &[VoiceRegistryEntry],
) -> Result<TtsSettings, String> {
let voice = voice_by_key(voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
let voice = registry
.iter()
.find(|voice| voice.key == voice_key)
.ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
if voice.backend != POCKET_BACKEND_ID || !is_locally_available(&voice.availability) {
return Err("The selected Pocket voice is not available on this device".to_string());
}
@@ -515,7 +581,7 @@ pub async fn set_pocket_voice(
.lock()
.map_err(|error| format!("text-to-speech settings lock poisoned: {error}"))?
.clone();
let settings = settings_with_pocket_voice(settings, &voice_key)?;
let settings = settings_with_pocket_voice(settings, &voice_key, &app)?;
let voice_change = apply_tts_settings(settings, &app, &state).await?;
drop(transition);
if let Err(error) = finish_voice_change(voice_change).await {
@@ -532,9 +598,11 @@ pub async fn set_pocket_voice(
#[tauri::command]
pub async fn preview_pocket_voice(
voice_key: String,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<(), String> {
let voice = voice_by_key(&voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
let voice =
voice_by_key(&app, &voice_key).ok_or_else(|| format!("Unknown voice: {voice_key}"))?;
if voice.backend != POCKET_BACKEND_ID {
return Err("Only Pocket voices can be previewed in this build".to_string());
}
@@ -548,10 +616,7 @@ pub async fn preview_pocket_voice(
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
let voice_name = voice
.reference_file
.and_then(|file| file.strip_suffix(".wav").map(str::to_string))
.ok_or_else(|| format!("Voice {voice_key} has no local Pocket reference file"))?;
let voice_name = pocket_voice_reference(&app, std::slice::from_ref(&voice_key))?;
tokio::task::spawn_blocking(move || {
let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
@@ -579,6 +644,69 @@ pub async fn preview_pocket_voice(
.map_err(|error| format!("Voice preview task failed: {error}"))?
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct TtsVoiceMutation {
pub settings: TtsSettings,
pub registry: Vec<VoiceRegistryEntry>,
}
#[tauri::command]
pub async fn import_pocket_voice(
app: AppHandle,
state: State<'_, AppState>,
) -> Result<Option<TtsVoiceMutation>, String> {
let Some(imported) = super::tts_voice_import::pick_and_import(&app).await? else {
return Ok(None);
};
let transition = state.tts_settings_transition.lock().await;
let settings = current_settings(&state)?;
let settings = settings_with_pocket_voice(settings, &imported.key, &app)?;
let voice_change = apply_tts_settings(settings, &app, &state).await?;
drop(transition);
finish_voice_change(voice_change).await?;
Ok(Some(TtsVoiceMutation {
settings: current_settings(&state)?,
registry: voice_registry(&app),
}))
}
#[tauri::command]
pub async fn delete_pocket_voice(
voice_key: String,
app: AppHandle,
state: State<'_, AppState>,
) -> Result<TtsVoiceMutation, String> {
if !voice_key.starts_with("pocket:imported:") {
return Err("Bundled voices cannot be deleted".to_string());
}
if voice_by_key(&app, &voice_key).is_none() {
return Err(format!("Unknown imported voice: {voice_key}"));
}
let transition = state.tts_settings_transition.lock().await;
let current = current_settings(&state)?;
let selected = resolve_voice_for_backend_in_registry(
&current.voice_preferences,
POCKET_BACKEND_ID,
&voice_registry(&app),
)
.is_ok_and(|voice| voice.key == voice_key);
let voice_change = if selected {
let fallback = settings_with_pocket_voice(current, MARY_VOICE_KEY, &app)?;
apply_tts_settings(fallback, &app, &state).await?
} else {
None
};
drop(transition);
finish_voice_change(voice_change).await?;
super::tts_voice_import::delete(&app, &voice_key)?;
Ok(TtsVoiceMutation {
settings: current_settings(&state)?,
registry: voice_registry(&app),
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -619,7 +747,7 @@ mod tests {
#[test]
fn registry_has_all_official_english_vctk_presets() {
assert_eq!(
voice_registry()
bundled_voice_registry()
.iter()
.map(|voice| {
(
@@ -680,7 +808,7 @@ mod tests {
fn identity_is_qualified_key_not_display_label() {
assert!(is_qualified_voice_key("pocket:imported:audio-content-hash"));
assert_ne!(MARY_VOICE_KEY, EVE_VOICE_KEY);
let mut registry = voice_registry();
let mut registry = bundled_voice_registry();
registry[0].display_name = "Jim".to_string();
registry[1].display_name = "Jim".to_string();
assert_eq!(registry[0].display_name, registry[1].display_name);
@@ -792,7 +920,12 @@ mod tests {
voice_preferences: vec!["siri:aaron".to_string(), MARY_VOICE_KEY.to_string()],
..TtsSettings::default()
};
let updated = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
let updated = settings_with_pocket_voice_from_registry(
current,
EVE_VOICE_KEY,
&bundled_voice_registry(),
)
.expect("available voice");
assert!(!updated.agent_text_to_speech);
assert_eq!(updated.voice_preferences, vec!["siri:aaron", EVE_VOICE_KEY]);
}
@@ -805,8 +938,12 @@ mod tests {
// This models the next command after the OFF save fails: it must merge
// from effective memory state, not the stale last-persisted ON value.
let current = state.huddle_audio.tts.lock().expect("settings").clone();
let voice_update =
settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
let voice_update = settings_with_pocket_voice_from_registry(
current,
EVE_VOICE_KEY,
&bundled_voice_registry(),
)
.expect("available voice");
assert!(!voice_update.agent_text_to_speech);
}
@@ -820,7 +957,12 @@ mod tests {
.expect("settings")
.agent_text_to_speech = false;
let current = state.huddle_audio.tts.lock().expect("settings").clone();
let unsaved = settings_with_pocket_voice(current, EVE_VOICE_KEY).expect("available voice");
let unsaved = settings_with_pocket_voice_from_registry(
current,
EVE_VOICE_KEY,
&bundled_voice_registry(),
)
.expect("available voice");
// This is the only pre-persistence mutation for an OFF candidate.
commit_effective_off(&state).expect("commit effective OFF state");
@@ -0,0 +1,451 @@
//! Local Pocket reference-voice import storage and WAV canonicalization.
use std::{
fs,
path::{Path, PathBuf},
};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tauri::{AppHandle, Manager};
use crate::managed_agents::storage::atomic_write_json_restricted;
const MAX_SOURCE_BYTES: u64 = 25 * 1024 * 1024;
const MIN_SAMPLE_RATE: u32 = 8_000;
const MAX_SAMPLE_RATE: u32 = 96_000;
const MIN_DURATION_SECONDS: f64 = 2.0;
const MAX_DURATION_SECONDS: f64 = 30.0;
const CANONICAL_SAMPLE_RATE: u32 = 32_000;
const REGISTRY_VERSION: u32 = 1;
const REGISTRY_FILE: &str = "registry.json";
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct ImportedVoice {
pub key: String,
pub display_name: String,
pub content_hash: String,
pub file_name: String,
}
#[derive(Default, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct ImportedVoiceRegistry {
version: u32,
voices: Vec<ImportedVoice>,
}
pub fn voices_dir(app: &AppHandle) -> Result<PathBuf, String> {
app.path()
.app_data_dir()
.map(|path| path.join("tts").join("pocket-voices"))
.map_err(|error| format!("could not locate local voice storage: {error}"))
}
fn registry_path(app: &AppHandle) -> Result<PathBuf, String> {
Ok(voices_dir(app)?.join(REGISTRY_FILE))
}
fn ensure_storage_dir(path: &Path) -> Result<(), String> {
fs::create_dir_all(path)
.map_err(|error| format!("could not create local voice storage: {error}"))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(path, fs::Permissions::from_mode(0o700))
.map_err(|error| format!("could not restrict local voice storage: {error}"))?;
}
Ok(())
}
pub fn load_registry(app: &AppHandle) -> Result<Vec<ImportedVoice>, String> {
let path = registry_path(app)?;
if !path.exists() {
return Ok(Vec::new());
}
let bytes =
fs::read(&path).map_err(|error| format!("could not read imported voices: {error}"))?;
let registry: ImportedVoiceRegistry = serde_json::from_slice(&bytes)
.map_err(|error| format!("imported voice registry is invalid: {error}"))?;
if registry.version > REGISTRY_VERSION {
return Err(format!(
"imported voice registry version {} is newer than this Buzz build supports",
registry.version
));
}
Ok(registry
.voices
.into_iter()
.filter(|voice| {
valid_hash(&voice.content_hash)
&& voice.key == format!("pocket:imported:{}", voice.content_hash)
&& voice.file_name == format!("{}.wav", voice.content_hash)
})
.filter(|voice| resolve_file(app, voice).is_ok())
.collect())
}
fn save_registry(app: &AppHandle, voices: &[ImportedVoice]) -> Result<(), String> {
let dir = voices_dir(app)?;
ensure_storage_dir(&dir)?;
let payload = serde_json::to_vec_pretty(&ImportedVoiceRegistry {
version: REGISTRY_VERSION,
voices: voices.to_vec(),
})
.map_err(|error| format!("could not encode imported voice registry: {error}"))?;
atomic_write_json_restricted(&dir.join(REGISTRY_FILE), &payload)
.map_err(|error| format!("could not save imported voice registry: {error}"))
}
pub fn resolve_file(app: &AppHandle, voice: &ImportedVoice) -> Result<PathBuf, String> {
if !valid_hash(&voice.content_hash) || voice.file_name != format!("{}.wav", voice.content_hash)
{
return Err("Imported voice registry contains an invalid file identity".to_string());
}
let path = voices_dir(app)?.join(&voice.file_name);
if !is_regular_file_without_symlink(&path) {
return Err(format!("Imported voice {} is missing", voice.display_name));
}
let bytes =
fs::read(&path).map_err(|error| format!("could not verify imported voice: {error}"))?;
if hex::encode(Sha256::digest(bytes)) != voice.content_hash {
return Err(format!(
"Imported voice {} does not match its content identity",
voice.display_name
));
}
Ok(path)
}
pub async fn pick_and_import(app: &AppHandle) -> Result<Option<ImportedVoice>, String> {
use tauri_plugin_dialog::DialogExt;
let (sender, receiver) = tokio::sync::oneshot::channel();
app.dialog()
.file()
.add_filter("WAV audio", &["wav"])
.pick_file(move |path| {
let _ = sender.send(path);
});
let Some(file_path) = receiver
.await
.map_err(|_| "voice picker closed unexpectedly".to_string())?
else {
return Ok(None);
};
let path = file_path
.as_path()
.ok_or("the selected voice path is invalid")?
.to_path_buf();
let app = app.clone();
tokio::task::spawn_blocking(move || import_path(&app, &path))
.await
.map_err(|error| format!("voice import task failed: {error}"))?
.map(Some)
}
pub fn import_path(app: &AppHandle, source: &Path) -> Result<ImportedVoice, String> {
let metadata =
fs::metadata(source).map_err(|error| format!("could not inspect selected WAV: {error}"))?;
if metadata.len() > MAX_SOURCE_BYTES {
return Err("Voice WAV must be 25 MB or smaller".to_string());
}
let source_bytes =
fs::read(source).map_err(|error| format!("could not read selected WAV: {error}"))?;
let samples = decode_wav(&source_bytes)?;
let canonical_samples = resample_linear(&samples.samples, samples.sample_rate);
let canonical = encode_pcm16_wav(&canonical_samples, CANONICAL_SAMPLE_RATE);
let hash = hex::encode(Sha256::digest(&canonical));
let key = format!("pocket:imported:{hash}");
let file_name = format!("{hash}.wav");
let display_name = source
.file_stem()
.and_then(|name| name.to_str())
.map(str::trim)
.filter(|name| !name.is_empty())
.unwrap_or("Imported voice")
.chars()
.take(80)
.collect::<String>();
let dir = voices_dir(app)?;
ensure_storage_dir(&dir)?;
let file_path = dir.join(&file_name);
let file_created = !file_path.exists();
if file_created {
atomic_write_json_restricted(&file_path, &canonical)
.map_err(|error| format!("could not save imported voice audio: {error}"))?;
} else {
if !is_regular_file_without_symlink(&file_path) {
return Err("Imported voice storage contains an unsafe file entry".to_string());
}
let existing = fs::read(&file_path)
.map_err(|error| format!("could not verify imported voice audio: {error}"))?;
if hex::encode(Sha256::digest(&existing)) != hash {
return Err("Imported voice storage contains mismatched audio data".to_string());
}
}
let mut imported = ImportedVoice {
key,
display_name,
content_hash: hash,
file_name,
};
let mut voices = load_registry(app)?;
if let Some(existing) = voices
.iter()
.find(|voice| voice.content_hash == imported.content_hash)
{
imported = existing.clone();
} else {
voices.push(imported.clone());
}
if let Err(error) = save_registry(app, &voices) {
if file_created {
let _ = fs::remove_file(&file_path);
}
return Err(error);
}
Ok(imported)
}
pub fn delete(app: &AppHandle, key: &str) -> Result<(), String> {
let mut voices = load_registry(app)?;
let index = voices
.iter()
.position(|voice| voice.key == key)
.ok_or_else(|| format!("Unknown imported voice: {key}"))?;
let previous_voices = voices.clone();
let removed = voices.remove(index);
save_registry(app, &voices)?;
let path = voices_dir(app)?.join(removed.file_name);
match fs::remove_file(path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => {
save_registry(app, &previous_voices).map_err(|rollback_error| {
format!(
"Imported voice audio could not be deleted ({error}), and its registry entry \
could not be restored ({rollback_error})"
)
})?;
Err(format!(
"Imported voice audio could not be deleted: {error}"
))
}
}
}
fn valid_hash(hash: &str) -> bool {
hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn is_regular_file_without_symlink(path: &Path) -> bool {
fs::symlink_metadata(path)
.is_ok_and(|metadata| metadata.file_type().is_file() && !metadata.file_type().is_symlink())
}
#[derive(Debug)]
struct DecodedWav {
sample_rate: u32,
samples: Vec<f32>,
}
fn decode_wav(bytes: &[u8]) -> Result<DecodedWav, String> {
if bytes.len() < 12 || &bytes[..4] != b"RIFF" || &bytes[8..12] != b"WAVE" {
return Err("Selected file is not a valid RIFF/WAVE file".to_string());
}
let mut offset = 12usize;
let mut format = None;
let mut data = None;
while offset.checked_add(8).is_some_and(|end| end <= bytes.len()) {
let id = &bytes[offset..offset + 4];
let size =
u32::from_le_bytes(bytes[offset + 4..offset + 8].try_into().unwrap_or([0; 4])) as usize;
let start = offset + 8;
let end = start.checked_add(size).ok_or("WAV chunk size overflow")?;
if end > bytes.len() {
return Err("Selected WAV contains a truncated chunk".to_string());
}
if id == b"fmt " {
format = Some(&bytes[start..end]);
} else if id == b"data" {
data = Some(&bytes[start..end]);
}
offset = end + (size & 1);
}
let format = format.ok_or("Selected WAV has no format chunk")?;
let data = data.ok_or("Selected WAV has no audio data")?;
if format.len() < 16 {
return Err("Selected WAV has an invalid format chunk".to_string());
}
let encoding = u16::from_le_bytes(format[0..2].try_into().unwrap_or([0; 2]));
let encoding = if encoding == 0xfffe && format.len() >= 40 {
u16::from_le_bytes(format[24..26].try_into().unwrap_or([0; 2]))
} else {
encoding
};
let channels = u16::from_le_bytes(format[2..4].try_into().unwrap_or([0; 2]));
let sample_rate = u32::from_le_bytes(format[4..8].try_into().unwrap_or([0; 4]));
let block_align = u16::from_le_bytes(format[12..14].try_into().unwrap_or([0; 2])) as usize;
let bits = u16::from_le_bytes(format[14..16].try_into().unwrap_or([0; 2]));
if channels != 1 {
return Err("Voice WAV must be mono".to_string());
}
if !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&sample_rate) {
return Err("Voice WAV sample rate must be between 8 and 96 kHz".to_string());
}
let bytes_per_sample = usize::from(bits.div_ceil(8));
if block_align != bytes_per_sample || block_align == 0 || data.len() % block_align != 0 {
return Err("Voice WAV has invalid sample alignment".to_string());
}
if !matches!((encoding, bits), (1, 8 | 16 | 24 | 32) | (3, 32)) {
return Err("Voice WAV must contain PCM or 32-bit float audio".to_string());
}
let frames = data.len() / block_align;
let duration = frames as f64 / f64::from(sample_rate);
if !(MIN_DURATION_SECONDS..=MAX_DURATION_SECONDS).contains(&duration) {
return Err("Voice WAV must be between 2 and 30 seconds long".to_string());
}
let mut samples = Vec::with_capacity(frames);
for chunk in data.chunks_exact(block_align) {
let sample = match (encoding, bits) {
(1, 8) => (f32::from(chunk[0]) - 128.0) / 128.0,
(1, 16) => f32::from(i16::from_le_bytes([chunk[0], chunk[1]])) / 32768.0,
(1, 24) => {
let raw = i32::from_le_bytes([
chunk[0],
chunk[1],
chunk[2],
if chunk[2] & 0x80 == 0 { 0 } else { 0xff },
]);
raw as f32 / 8_388_608.0
}
(1, 32) => {
i32::from_le_bytes(chunk.try_into().map_err(|_| "invalid PCM sample")?) as f32
/ 2_147_483_648.0
}
(3, 32) => f32::from_le_bytes(
chunk
.try_into()
.map_err(|_| "invalid floating-point sample")?,
),
_ => unreachable!(),
};
if !sample.is_finite() {
return Err("Voice WAV contains non-finite samples".to_string());
}
samples.push(sample.clamp(-1.0, 1.0));
}
let peak = samples
.iter()
.fold(0.0_f32, |peak, sample| peak.max(sample.abs()));
let rms =
(samples.iter().map(|sample| sample * sample).sum::<f32>() / samples.len() as f32).sqrt();
if peak < 0.001 || rms < 0.0001 {
return Err("Voice WAV is silent or too quiet to clone".to_string());
}
Ok(DecodedWav {
sample_rate,
samples,
})
}
fn resample_linear(samples: &[f32], source_rate: u32) -> Vec<f32> {
if source_rate == CANONICAL_SAMPLE_RATE {
return samples.to_vec();
}
let output_len = ((samples.len() as u64 * u64::from(CANONICAL_SAMPLE_RATE)
+ u64::from(source_rate) / 2)
/ u64::from(source_rate)) as usize;
(0..output_len)
.map(|index| {
let source = index as f64 * f64::from(source_rate) / f64::from(CANONICAL_SAMPLE_RATE);
let left = source.floor() as usize;
let fraction = (source - left as f64) as f32;
let a = samples[left.min(samples.len() - 1)];
let b = samples[(left + 1).min(samples.len() - 1)];
a + (b - a) * fraction
})
.collect()
}
fn encode_pcm16_wav(samples: &[f32], sample_rate: u32) -> Vec<u8> {
let data_len = (samples.len() * 2) as u32;
let mut bytes = Vec::with_capacity(44 + data_len as usize);
bytes.extend_from_slice(b"RIFF");
bytes.extend_from_slice(&(36 + data_len).to_le_bytes());
bytes.extend_from_slice(b"WAVEfmt ");
bytes.extend_from_slice(&16_u32.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&sample_rate.to_le_bytes());
bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes());
bytes.extend_from_slice(&2_u16.to_le_bytes());
bytes.extend_from_slice(&16_u16.to_le_bytes());
bytes.extend_from_slice(b"data");
bytes.extend_from_slice(&data_len.to_le_bytes());
for sample in samples {
let value = (sample.clamp(-1.0, 1.0) * f32::from(i16::MAX)).round() as i16;
bytes.extend_from_slice(&value.to_le_bytes());
}
bytes
}
#[cfg(test)]
mod tests {
use super::*;
fn fixture(sample_rate: u32, seconds: usize, amplitude: f32) -> Vec<u8> {
let samples = (0..sample_rate as usize * seconds)
.map(|index| {
amplitude
* (std::f32::consts::TAU * 220.0 * index as f32 / sample_rate as f32).sin()
})
.collect::<Vec<_>>();
encode_pcm16_wav(&samples, sample_rate)
}
#[test]
fn validates_and_canonicalizes_supported_wav() {
let decoded = decode_wav(&fixture(44_100, 2, 0.5)).expect("valid WAV");
let canonical = resample_linear(&decoded.samples, decoded.sample_rate);
assert_eq!(canonical.len(), 64_000);
let encoded = encode_pcm16_wav(&canonical, CANONICAL_SAMPLE_RATE);
let reparsed = decode_wav(&encoded).expect("canonical WAV");
assert_eq!(reparsed.sample_rate, 32_000);
assert_eq!(reparsed.samples.len(), 64_000);
}
#[test]
fn rejects_silent_and_short_wav() {
assert!(decode_wav(&fixture(32_000, 2, 0.0))
.expect_err("silence rejected")
.contains("silent"));
assert!(decode_wav(&fixture(32_000, 1, 0.5))
.expect_err("short rejected")
.contains("between 2 and 30"));
}
#[test]
fn rejects_stereo_out_of_range_and_overlong_wav() {
let mut stereo_header = fixture(32_000, 2, 0.5);
stereo_header[22..24].copy_from_slice(&2_u16.to_le_bytes());
assert!(decode_wav(&stereo_header)
.expect_err("stereo rejected")
.contains("mono"));
let mut low_rate_header = fixture(32_000, 2, 0.5);
low_rate_header[24..28].copy_from_slice(&4_000_u32.to_le_bytes());
assert!(decode_wav(&low_rate_header)
.expect_err("low sample rate rejected")
.contains("between 8 and 96"));
assert!(decode_wav(&fixture(8_000, 31, 0.5))
.expect_err("overlong rejected")
.contains("between 2 and 30"));
}
}
@@ -120,7 +120,7 @@ pub(super) fn reconcile_selected_voice(
return true;
}
let requested_path = model_dir.join(format!("{requested_voice}.{VOICE_FILE_EXT}"));
let requested_path = voice_path(model_dir, &requested_voice);
match load_voice_style(&requested_path) {
Ok(requested_style) => {
*style = requested_style;
@@ -151,6 +151,15 @@ pub(super) fn reconcile_selected_voice(
}
}
pub(super) fn voice_path(model_dir: &Path, voice: &str) -> std::path::PathBuf {
let path = Path::new(voice);
if path.is_absolute() {
path.to_path_buf()
} else {
model_dir.join(format!("{voice}.{VOICE_FILE_EXT}"))
}
}
pub(super) fn retain_cancelled_text(
deferred_text: &mut VecDeque<QueuedText>,
current_text: &mut Option<QueuedText>,
+2
View File
@@ -874,6 +874,8 @@ pub fn run() {
huddle::tts_settings::list_voice_registry,
huddle::tts_settings::set_pocket_voice,
huddle::tts_settings::preview_pocket_voice,
huddle::tts_settings::import_pocket_voice,
huddle::tts_settings::delete_pocket_voice,
speak_agent_message,
add_agent_to_huddle,
check_pipeline_hotstart,
@@ -1,9 +1,19 @@
import * as React from "react";
import { ChevronDown, Play, Volume2 } from "lucide-react";
import { ChevronDown, Play, Trash2, Upload, Volume2 } from "lucide-react";
import { invokeTauri } from "@/shared/api/tauri";
import { cn } from "@/shared/lib/cn";
import { Button } from "@/shared/ui/button";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/shared/ui/alert-dialog";
import {
DropdownMenu,
DropdownMenuContent,
@@ -27,11 +37,18 @@ export type TtsSettings = {
voicePreferences: string[];
};
type TtsVoiceMutation = {
settings: TtsSettings;
registry: VoiceRegistryEntry[];
};
export function VoiceSettingsCard() {
const [settings, setSettings] = React.useState<TtsSettings | null>(null);
const [registry, setRegistry] = React.useState<VoiceRegistryEntry[]>([]);
const [busy, setBusy] = React.useState(false);
const [previewing, setPreviewing] = React.useState(false);
const [deleteCandidate, setDeleteCandidate] =
React.useState<VoiceRegistryEntry | null>(null);
const [error, setError] = React.useState<string | null>(null);
React.useEffect(() => {
@@ -111,6 +128,50 @@ export function VoiceSettingsCard() {
}
}, []);
const importPocketVoice = React.useCallback(async () => {
setBusy(true);
setError(null);
try {
const result = await invokeTauri<TtsVoiceMutation | null>(
"import_pocket_voice",
);
if (result) {
setSettings(result.settings);
setRegistry(result.registry);
}
} catch (importError) {
setError(
importError instanceof Error
? importError.message
: "Voice could not be imported.",
);
} finally {
setBusy(false);
}
}, []);
const deletePocketVoice = React.useCallback(async (voiceKey: string) => {
setBusy(true);
setError(null);
try {
const result = await invokeTauri<TtsVoiceMutation>(
"delete_pocket_voice",
{ voiceKey },
);
setSettings(result.settings);
setRegistry(result.registry);
setDeleteCandidate(null);
} catch (deleteError) {
setError(
deleteError instanceof Error
? deleteError.message
: "Voice could not be deleted.",
);
} finally {
setBusy(false);
}
}, []);
const voices = voicesForBackend(registry, "pocket");
const selectedVoice = selectedVoiceForBackend(
settings?.voicePreferences ?? [],
@@ -236,6 +297,28 @@ export function VoiceSettingsCard() {
)}
Preview
</Button>
<Button
data-testid="pocket-voice-import"
disabled={controlsDisabled}
onClick={() => void importPocketVoice()}
size="sm"
variant="outline"
>
<Upload className="h-4 w-4" />
Add voice
</Button>
{selectedVoice?.key.startsWith("pocket:imported:") && (
<Button
aria-label={`Delete ${selectedVoice.displayName}`}
data-testid="pocket-voice-delete"
disabled={controlsDisabled}
onClick={() => setDeleteCandidate(selectedVoice)}
size="icon"
variant="ghost"
>
<Trash2 className="h-4 w-4" />
</Button>
)}
</div>
</SettingsOptionRow>
</SettingsOptionGroup>
@@ -251,6 +334,41 @@ export function VoiceSettingsCard() {
</p>
)}
</div>
<AlertDialog
onOpenChange={(open) => {
if (!open) setDeleteCandidate(null);
}}
open={deleteCandidate !== null}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Delete imported voice?</AlertDialogTitle>
<AlertDialogDescription>
{deleteCandidate
? `${deleteCandidate.displayName} and its local audio file will be removed.`
: "This imported voice and its local audio file will be removed."}
{selectedVoice?.key === deleteCandidate?.key &&
" Mary will be selected instead."}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel disabled={busy}>Cancel</AlertDialogCancel>
<AlertDialogAction
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
data-testid="confirm-pocket-voice-delete"
disabled={busy || !deleteCandidate}
onClick={(event) => {
event.preventDefault();
if (deleteCandidate) {
void deletePocketVoice(deleteCandidate.key);
}
}}
>
Delete voice
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</section>
);
}
+186 -102
View File
@@ -9665,7 +9665,25 @@ export function maybeInstallE2eTauriMocks() {
deviceId: state === "running" ? "mock-endpoint-id" : null,
deviceName: state === "running" ? "Mock desktop" : null,
});
const handleMockCommand = async (command: string, payload: unknown) => {
let mockImportedVoices: Array<{
key: string;
displayName: string;
backend: string;
backendName: string;
availability: "installed";
fallbackKey: string;
referenceFile: string;
provenance: {
source: string;
contentHash: string;
license: null;
sourceUrl: null;
};
}> = [];
const handleMockCommand = async (
command: string,
payload: unknown,
): Promise<unknown> => {
const activeConfig = getConfig();
const identity = getActiveIdentity(activeConfig);
window.__BUZZ_E2E_COMMANDS__?.push(command);
@@ -9693,107 +9711,110 @@ export function maybeInstallE2eTauriMocks() {
);
case "list_voice_registry":
return [
[
"anna",
"Anna",
"anna.wav",
"p228_023_enhanced.wav",
"0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856",
],
[
"vera",
"Vera",
"vera.wav",
"p229_023_enhanced.wav",
"309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b",
],
[
"fantine",
"Fantine",
"fantine.wav",
"p244_023_enhanced.wav",
"5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b",
],
[
"charles",
"Charles",
"charles.wav",
"p254_023_enhanced.wav",
"6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756",
],
[
"paul",
"Paul",
"paul.wav",
"p259_023_enhanced.wav",
"7aba504fe0b3b16478b69ed27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b",
],
[
"eponine",
"Eponine",
"eponine.wav",
"p262_023_enhanced.wav",
"a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b",
],
[
"azelma",
"Azelma",
"azelma.wav",
"p303_023_enhanced.wav",
"60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026",
],
[
"george",
"George",
"george.wav",
"p315_023_enhanced.wav",
"29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae",
],
[
"mary",
"Mary",
"reference_sample.wav",
"p333_023_enhanced.wav",
"a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f",
],
[
"jane",
"Jane",
"jane.wav",
"p339_023_enhanced.wav",
"2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a",
],
[
"michael",
"Michael",
"michael.wav",
"p360_023_enhanced.wav",
"b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad",
],
[
"eve",
"Eve",
"eve.wav",
"p361_023_enhanced.wav",
"396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd",
],
].map(
([id, displayName, referenceFile, upstreamFile, contentHash]) => ({
key: `pocket:${id}`,
displayName,
backend: "pocket",
backendName: "Pocket TTS",
availability: "bundled",
fallbackKey: id === "mary" ? null : "pocket:mary",
referenceFile,
provenance: {
source: "bundled",
contentHash,
license: "CC-BY-4.0",
sourceUrl: `https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/vctk/${upstreamFile}`,
},
}),
);
...[
[
"anna",
"Anna",
"anna.wav",
"p228_023_enhanced.wav",
"0a6de25cf12bf1540beb85979f306a92be81fecc051c547c5395e7e5237a3856",
],
[
"vera",
"Vera",
"vera.wav",
"p229_023_enhanced.wav",
"309cf91a895830f15842b398f69a4962cb1f7e0bfab10e25dd27838e826c204b",
],
[
"fantine",
"Fantine",
"fantine.wav",
"p244_023_enhanced.wav",
"5f07d4e2a3f20a15572aae885156b43ef3fc12ef3812996fd135680d9956448b",
],
[
"charles",
"Charles",
"charles.wav",
"p254_023_enhanced.wav",
"6b681a429198f16e378d53bccb08d06939da7b00144a7696111d4f8f76be7756",
],
[
"paul",
"Paul",
"paul.wav",
"p259_023_enhanced.wav",
"7aba504fe0b3b16478b69ed27ce6007e3cb42b0c1915b5f1c6a6024ae37d679b",
],
[
"eponine",
"Eponine",
"eponine.wav",
"p262_023_enhanced.wav",
"a13c27fb47627b05223691a0ef2974358a18c886e6c2f9d2762ff1d02c20926b",
],
[
"azelma",
"Azelma",
"azelma.wav",
"p303_023_enhanced.wav",
"60e3d26cdf2efdec5df712152c839928f4d5522821e6554ae11fd96c57ab1026",
],
[
"george",
"George",
"george.wav",
"p315_023_enhanced.wav",
"29a41f93bf5236e5b21501091d7774c255d5f3d4e62fa4f9fdf0a92a793c84ae",
],
[
"mary",
"Mary",
"reference_sample.wav",
"p333_023_enhanced.wav",
"a35b0468382218e9f37a9a7494d1e4b74deaf18d7ced22265b4e325bb55c183f",
],
[
"jane",
"Jane",
"jane.wav",
"p339_023_enhanced.wav",
"2f12e7f155eb3118f55425394f1b049e5b1b67bdc9b3932c8ba4521420aeb84a",
],
[
"michael",
"Michael",
"michael.wav",
"p360_023_enhanced.wav",
"b6743e9195e5e3fd34fe9d1633ae93f7ffab787b249e45f6467d7d6f7a6ee6ad",
],
[
"eve",
"Eve",
"eve.wav",
"p361_023_enhanced.wav",
"396e7cbd066b0f3fb6d67fa26e7904076958239d736d4390f15b5fe88feb14cd",
],
].map(
([id, displayName, referenceFile, upstreamFile, contentHash]) => ({
key: `pocket:${id}`,
displayName,
backend: "pocket",
backendName: "Pocket TTS",
availability: "bundled",
fallbackKey: id === "mary" ? null : "pocket:mary",
referenceFile,
provenance: {
source: "bundled",
contentHash,
license: "CC-BY-4.0",
sourceUrl: `https://huggingface.co/kyutai/tts-voices/blob/323332d33f997de8394f24a193e1a76df720e01a/vctk/${upstreamFile}`,
},
}),
),
...mockImportedVoices,
];
case "set_tts_enabled": {
const enabled = (payload as { enabled?: boolean })?.enabled;
if (typeof enabled !== "boolean")
@@ -9838,6 +9859,69 @@ export function maybeInstallE2eTauriMocks() {
}
case "preview_pocket_voice":
return null;
case "import_pocket_voice": {
const contentHash = "1".repeat(64);
const imported = {
key: `pocket:imported:${contentHash}`,
displayName: "My voice",
backend: "pocket",
backendName: "Pocket TTS",
availability: "installed" as const,
fallbackKey: "pocket:mary",
referenceFile: `${contentHash}.wav`,
provenance: {
source: "local import",
contentHash,
license: null,
sourceUrl: null,
},
};
mockImportedVoices = [imported];
const current = activeConfig?.mock?.ttsSettings ?? {
version: 1,
agentTextToSpeech: true,
voicePreferences: ["pocket:mary"],
};
const settings = {
...current,
voicePreferences: [imported.key],
};
if (activeConfig) {
activeConfig.mock ??= {};
activeConfig.mock.ttsSettings = settings;
}
return {
settings,
registry: await handleMockCommand("list_voice_registry", null),
};
}
case "delete_pocket_voice": {
const voiceKey = (payload as { voiceKey?: string })?.voiceKey;
if (!voiceKey?.startsWith("pocket:imported:"))
throw new Error("Missing imported Pocket voice key");
mockImportedVoices = mockImportedVoices.filter(
(voice) => voice.key !== voiceKey,
);
const current = activeConfig?.mock?.ttsSettings ?? {
version: 1,
agentTextToSpeech: true,
voicePreferences: ["pocket:mary"],
};
const settings = {
...current,
voicePreferences: current.voicePreferences.includes(voiceKey)
? ["pocket:mary"]
: current.voicePreferences,
};
if (activeConfig) {
activeConfig.mock ??= {};
activeConfig.mock.ttsSettings = settings;
}
return {
settings,
registry: await handleMockCommand("list_voice_registry", null),
};
}
case "get_builderlab_auth":
return activeConfig?.mock?.builderlabAuth ?? null;
case "start_builderlab_login": {
+39
View File
@@ -116,4 +116,43 @@ test.describe("Pocket voice settings", () => {
},
});
});
test("imports, selects, and safely deletes a local voice", async ({
page,
}) => {
await installMockBridge(page);
await page.goto("/", { waitUntil: "domcontentloaded" });
await openSettings(page, "voice");
await page.getByTestId("pocket-voice-import").click();
await expect(page.getByTestId("pocket-voice-selector")).toContainText(
"My voice",
);
await expect(page.getByTestId("pocket-voice-delete")).toBeVisible();
await page.getByTestId("pocket-voice-delete").click();
await expect(page.getByText("Delete imported voice?")).toBeVisible();
await page.getByTestId("confirm-pocket-voice-delete").click();
await expect(page.getByTestId("pocket-voice-selector")).toContainText(
"Mary",
);
await expect(page.getByTestId("pocket-voice-delete")).toBeHidden();
const mutations = await page.evaluate(() =>
(window.__BUZZ_E2E_COMMAND_LOG__ ?? [])
.filter((entry) =>
["import_pocket_voice", "delete_pocket_voice"].includes(
entry.command,
),
)
.map((entry) => ({ command: entry.command, payload: entry.payload })),
);
expect(mutations).toEqual([
{ command: "import_pocket_voice", payload: {} },
{
command: "delete_pocket_voice",
payload: { voiceKey: `pocket:imported:${"1".repeat(64)}` },
},
]);
});
});