From a612c1962c49dc652643912d5399bf654b17794d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 29 Jul 2026 20:28:10 -0400 Subject: [PATCH] Accept common audio formats for Pocket voices Signed-off-by: John Tennant --- Cargo.lock | 165 +++++++++++ crates/buzz-voice/Cargo.toml | 1 + crates/buzz-voice/src/imported.rs | 269 +++++++++++++++--- desktop/src-tauri/Cargo.lock | 12 + .../src-tauri/src/huddle/tts_voice_import.rs | 5 +- 5 files changed, 407 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e134ec30..045e6057d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,6 +1312,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "sherpa-onnx", + "symphonia", "tempfile", "tokenizers", ] @@ -2773,6 +2774,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fancy-regex" version = "0.11.0" @@ -9037,6 +9044,164 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8413fa754942ac16a73634c9dfd1500ed5c61430956b33728567f667fdd393ab" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +dependencies = [ + "log", + "symphonia-core", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +dependencies = [ + "encoding_rs", + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", + "symphonia-utils-xiph", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-utils-xiph" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" +dependencies = [ + "symphonia-core", + "symphonia-metadata", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/crates/buzz-voice/Cargo.toml b/crates/buzz-voice/Cargo.toml index f45fdff8a..beff5b4a5 100644 --- a/crates/buzz-voice/Cargo.toml +++ b/crates/buzz-voice/Cargo.toml @@ -18,6 +18,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = { workspace = true } sherpa-onnx = "1.12" +symphonia = { version = "0.5", default-features = false, features = ["aac", "aiff", "alac", "flac", "isomp4", "mp3", "ogg", "pcm", "vorbis", "wav"] } tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] } [dev-dependencies] diff --git a/crates/buzz-voice/src/imported.rs b/crates/buzz-voice/src/imported.rs index 4c494e226..6f0ea71ca 100644 --- a/crates/buzz-voice/src/imported.rs +++ b/crates/buzz-voice/src/imported.rs @@ -9,6 +9,10 @@ use std::{ use atomic_write_file::AtomicWriteFile; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use symphonia::core::{ + audio::SampleBuffer, codecs::DecoderOptions, errors::Error as SymphoniaError, + formats::FormatOptions, io::MediaSourceStream, meta::MetadataOptions, probe::Hint, +}; const MAX_SOURCE_BYTES: u64 = 25 * 1024 * 1024; const MIN_SAMPLE_RATE: u32 = 8_000; @@ -112,13 +116,22 @@ impl PocketVoiceLibrary { pub fn import_path(&self, source: &Path) -> Result { let metadata = fs::metadata(source) - .map_err(|error| format!("could not inspect selected WAV: {error}"))?; + .map_err(|error| format!("could not inspect selected audio: {error}"))?; if metadata.len() > MAX_SOURCE_BYTES { - return Err("Voice WAV must be 25 MB or smaller".to_string()); + return Err("Voice audio must be 25 MB or smaller".to_string()); } - let source_bytes = - fs::read(source).map_err(|error| format!("could not read selected WAV: {error}"))?; - let samples = decode_wav(&source_bytes)?; + let extension = source + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_ascii_lowercase) + .ok_or_else(|| "Voice audio must have a supported file extension".to_string())?; + let samples = if extension == "wav" { + let source_bytes = fs::read(source) + .map_err(|error| format!("could not read selected audio: {error}"))?; + decode_wav(&source_bytes)? + } else { + decode_media(source, &extension)? + }; let canonical_samples = resample_linear(&samples.samples, samples.sample_rate); let canonical = encode_pcm16_wav(&canonical_samples, CANONICAL_SAMPLE_RATE); let hash = hex::encode(Sha256::digest(&canonical)); @@ -300,12 +313,12 @@ fn is_regular_file_without_symlink(path: &Path) -> bool { } #[derive(Debug)] -struct DecodedWav { +struct DecodedAudio { sample_rate: u32, samples: Vec, } -fn decode_wav(bytes: &[u8]) -> Result { +fn decode_wav(bytes: &[u8]) -> Result { if bytes.len() < 12 || &bytes[..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { return Err("Selected file is not a valid RIFF/WAVE file".to_string()); } @@ -343,14 +356,17 @@ fn decode_wav(bytes: &[u8]) -> Result { let sample_rate = u32::from_le_bytes(format[4..8].try_into().unwrap_or([0; 4])); let block_align = u16::from_le_bytes(format[12..14].try_into().unwrap_or([0; 2])) as usize; let bits = u16::from_le_bytes(format[14..16].try_into().unwrap_or([0; 2])); - if channels != 1 { - return Err("Voice WAV must be mono".to_string()); + if channels == 0 || channels > 8 { + return Err("Voice WAV must contain between 1 and 8 channels".to_string()); } if !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&sample_rate) { return Err("Voice WAV sample rate must be between 8 and 96 kHz".to_string()); } let bytes_per_sample = usize::from(bits.div_ceil(8)); - if block_align != bytes_per_sample || block_align == 0 || data.len() % block_align != 0 { + if block_align != bytes_per_sample * usize::from(channels) + || block_align == 0 + || data.len() % block_align != 0 + { return Err("Voice WAV has invalid sample alignment".to_string()); } if !matches!((encoding, bits), (1, 8 | 16 | 24 | 32) | (3, 32)) { @@ -363,45 +379,148 @@ fn decode_wav(bytes: &[u8]) -> Result { } let mut samples = Vec::with_capacity(frames); - for chunk in data.chunks_exact(block_align) { - let sample = match (encoding, bits) { - (1, 8) => (f32::from(chunk[0]) - 128.0) / 128.0, - (1, 16) => f32::from(i16::from_le_bytes([chunk[0], chunk[1]])) / 32768.0, - (1, 24) => { - let raw = i32::from_le_bytes([ - chunk[0], - chunk[1], - chunk[2], - if chunk[2] & 0x80 == 0 { 0 } else { 0xff }, - ]); - raw as f32 / 8_388_608.0 + for frame in data.chunks_exact(block_align) { + let mut mono = 0.0_f32; + for chunk in frame.chunks_exact(bytes_per_sample) { + let sample = match (encoding, bits) { + (1, 8) => (f32::from(chunk[0]) - 128.0) / 128.0, + (1, 16) => f32::from(i16::from_le_bytes([chunk[0], chunk[1]])) / 32768.0, + (1, 24) => { + let raw = i32::from_le_bytes([ + chunk[0], + chunk[1], + chunk[2], + if chunk[2] & 0x80 == 0 { 0 } else { 0xff }, + ]); + raw as f32 / 8_388_608.0 + } + (1, 32) => { + i32::from_le_bytes(chunk.try_into().map_err(|_| "invalid PCM sample")?) as f32 + / 2_147_483_648.0 + } + (3, 32) => f32::from_le_bytes( + chunk + .try_into() + .map_err(|_| "invalid floating-point sample")?, + ), + _ => unreachable!(), + }; + if !sample.is_finite() { + return Err("Voice WAV contains non-finite samples".to_string()); } - (1, 32) => { - i32::from_le_bytes(chunk.try_into().map_err(|_| "invalid PCM sample")?) as f32 - / 2_147_483_648.0 - } - (3, 32) => f32::from_le_bytes( - chunk - .try_into() - .map_err(|_| "invalid floating-point sample")?, - ), - _ => unreachable!(), - }; - if !sample.is_finite() { - return Err("Voice WAV contains non-finite samples".to_string()); + mono += sample; } - samples.push(sample.clamp(-1.0, 1.0)); + samples.push((mono / f32::from(channels)).clamp(-1.0, 1.0)); } let stats = PcmStats::analyze(&samples, sample_rate); if !stats.is_non_silent() { return Err("Voice WAV is silent or too quiet to clone".to_string()); } - Ok(DecodedWav { + Ok(DecodedAudio { sample_rate, samples, }) } +fn decode_media(source: &Path, extension: &str) -> Result { + let supported = ["m4a", "mp3", "flac", "ogg", "oga", "aif", "aiff"]; + if !supported.contains(&extension) { + return Err(format!( + "Unsupported voice audio format .{extension}. Choose WAV, M4A, MP3, FLAC, OGG, or AIFF" + )); + } + + let file = fs::File::open(source) + .map_err(|error| format!("could not read selected audio: {error}"))?; + let media = MediaSourceStream::new(Box::new(file), Default::default()); + let mut hint = Hint::new(); + hint.with_extension(extension); + let probed = symphonia::default::get_probe() + .format( + &hint, + media, + &FormatOptions::default(), + &MetadataOptions::default(), + ) + .map_err(|error| format!("could not recognize selected audio: {error}"))?; + let mut format = probed.format; + let track = format + .default_track() + .ok_or_else(|| "Selected audio has no decodable track".to_string())?; + let track_id = track.id; + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &DecoderOptions::default()) + .map_err(|error| format!("could not initialize audio decoder: {error}"))?; + let mut sample_rate = None; + let mut samples = Vec::new(); + + loop { + let packet = match format.next_packet() { + Ok(packet) => packet, + Err(SymphoniaError::ResetRequired) => { + return Err("Selected audio changes format mid-stream".to_string()); + } + Err(SymphoniaError::IoError(error)) + if error.kind() == std::io::ErrorKind::UnexpectedEof => + { + break; + } + Err(error) => return Err(format!("could not read selected audio: {error}")), + }; + if packet.track_id() != track_id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(decoded) => decoded, + Err(SymphoniaError::DecodeError(_)) => continue, + Err(error) => return Err(format!("could not decode selected audio: {error}")), + }; + let spec = *decoded.spec(); + if !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&spec.rate) { + return Err("Voice audio sample rate must be between 8 and 96 kHz".to_string()); + } + if sample_rate.is_some_and(|rate| rate != spec.rate) { + return Err("Selected audio changes sample rate mid-stream".to_string()); + } + sample_rate = Some(spec.rate); + let channels = spec.channels.count(); + if channels == 0 || channels > 8 { + return Err("Voice audio must contain between 1 and 8 channels".to_string()); + } + let mut buffer = SampleBuffer::::new(decoded.capacity() as u64, spec); + buffer.copy_interleaved_ref(decoded); + for frame in buffer.samples().chunks_exact(channels) { + let mono = frame.iter().copied().sum::() / channels as f32; + if !mono.is_finite() { + return Err("Voice audio contains non-finite samples".to_string()); + } + samples.push(mono.clamp(-1.0, 1.0)); + } + if samples.len() as f64 > MAX_DURATION_SECONDS * f64::from(spec.rate) { + return Err("Voice audio must be between 2 and 30 seconds long".to_string()); + } + } + + let sample_rate = + sample_rate.ok_or_else(|| "Selected audio contains no samples".to_string())?; + validate_decoded_audio(&samples, sample_rate)?; + Ok(DecodedAudio { + sample_rate, + samples, + }) +} + +fn validate_decoded_audio(samples: &[f32], sample_rate: u32) -> Result<(), String> { + let stats = PcmStats::analyze(samples, sample_rate); + if !(MIN_DURATION_SECONDS..=MAX_DURATION_SECONDS).contains(&stats.duration_seconds) { + return Err("Voice audio must be between 2 and 30 seconds long".to_string()); + } + if !stats.is_non_silent() { + return Err("Voice audio is silent or too quiet to clone".to_string()); + } + Ok(()) +} + fn resample_linear(samples: &[f32], source_rate: u32) -> Vec { if source_rate == CANONICAL_SAMPLE_RATE { return samples.to_vec(); @@ -457,6 +576,24 @@ mod tests { encode_pcm16_wav(&samples, sample_rate) } + fn stereo_fixture(sample_rate: u32, seconds: usize, amplitude: f32) -> Vec { + let mono = fixture(sample_rate, seconds, amplitude); + let mono_data = &mono[44..]; + let mut stereo_data = Vec::with_capacity(mono_data.len() * 2); + for sample in mono_data.chunks_exact(2) { + stereo_data.extend_from_slice(sample); + stereo_data.extend_from_slice(sample); + } + let mut stereo = mono[..44].to_vec(); + stereo[4..8].copy_from_slice(&(36 + stereo_data.len() as u32).to_le_bytes()); + stereo[22..24].copy_from_slice(&2_u16.to_le_bytes()); + stereo[28..32].copy_from_slice(&(sample_rate * 4).to_le_bytes()); + stereo[32..34].copy_from_slice(&4_u16.to_le_bytes()); + stereo[40..44].copy_from_slice(&(stereo_data.len() as u32).to_le_bytes()); + stereo.extend_from_slice(&stereo_data); + stereo + } + #[test] fn imports_persists_reloads_and_deletes_canonical_voice() { let temp = tempfile::tempdir().expect("temp voice workspace"); @@ -492,6 +629,52 @@ mod tests { assert!(!stored.exists()); } + #[test] + fn common_stereo_audio_is_downmixed_to_canonical_mono() { + let temp = tempfile::tempdir().expect("temp voice workspace"); + let source = temp.path().join("stereo.wav"); + fs::write(&source, stereo_fixture(44_100, 2, 0.5)).expect("write stereo"); + let library = PocketVoiceLibrary::new(temp.path().join("library")); + + let imported = library.import_path(&source).expect("import stereo"); + let stored = library + .resolve_file(&imported) + .expect("resolve stored voice"); + let decoded = decode_wav(&fs::read(stored).expect("read stored voice")) + .expect("decode canonical voice"); + assert_eq!(decoded.sample_rate, CANONICAL_SAMPLE_RATE); + assert_eq!(decoded.samples.len(), CANONICAL_SAMPLE_RATE as usize * 2); + } + + #[test] + #[ignore = "requires BUZZ_VOICE_IMPORT_TEST_DIR with common-format fixtures"] + fn imports_common_audio_format_fixtures() { + let fixtures = + PathBuf::from(std::env::var("BUZZ_VOICE_IMPORT_TEST_DIR").expect("fixture directory")); + let temp = tempfile::tempdir().expect("temp voice workspace"); + let library = PocketVoiceLibrary::new(temp.path().join("library")); + + for file_name in [ + "voice.wav", + "voice.m4a", + "voice.mp3", + "voice.flac", + "voice.ogg", + "voice.aiff", + ] { + let imported = library + .import_path(&fixtures.join(file_name)) + .unwrap_or_else(|error| panic!("import {file_name}: {error}")); + let stored = library + .resolve_file(&imported) + .unwrap_or_else(|error| panic!("resolve {file_name}: {error}")); + let decoded = decode_wav(&fs::read(stored).expect("read canonical voice")) + .expect("decode canonical voice"); + assert_eq!(decoded.sample_rate, CANONICAL_SAMPLE_RATE); + assert!(decoded.samples.len() >= CANONICAL_SAMPLE_RATE as usize * 2); + } + } + #[test] fn invalid_unsupported_and_silent_files_do_not_mutate_registry() { let temp = tempfile::tempdir().expect("temp voice workspace"); @@ -511,14 +694,12 @@ mod tests { .expect_err("silence rejected") .contains("silent")); - let mut stereo = fixture(32_000, 2, 0.5); - stereo[22..24].copy_from_slice(&2_u16.to_le_bytes()); - let stereo_path = temp.path().join("stereo.wav"); - fs::write(&stereo_path, stereo).expect("write stereo"); + let unsupported_container = temp.path().join("voice.txt"); + fs::write(&unsupported_container, b"not audio").expect("write unsupported container"); assert!(library - .import_path(&stereo_path) - .expect_err("stereo rejected") - .contains("mono")); + .import_path(&unsupported_container) + .expect_err("container rejected") + .contains("Unsupported voice audio format")); let mut unsupported = fixture(32_000, 2, 0.5); unsupported[20..22].copy_from_slice(&6_u16.to_le_bytes()); diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 43afe26c3..351517d02 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1184,6 +1184,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "sherpa-onnx", + "symphonia", "tokenizers", ] @@ -9775,6 +9776,7 @@ dependencies = [ "symphonia-bundle-flac", "symphonia-bundle-mp3", "symphonia-codec-aac", + "symphonia-codec-alac", "symphonia-codec-pcm", "symphonia-codec-vorbis", "symphonia-core", @@ -9819,6 +9821,16 @@ dependencies = [ "symphonia-core", ] +[[package]] +name = "symphonia-codec-alac" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8413fa754942ac16a73634c9dfd1500ed5c61430956b33728567f667fdd393ab" +dependencies = [ + "log", + "symphonia-core", +] + [[package]] name = "symphonia-codec-pcm" version = "0.5.5" diff --git a/desktop/src-tauri/src/huddle/tts_voice_import.rs b/desktop/src-tauri/src/huddle/tts_voice_import.rs index 1813b19ab..cdcc7761e 100644 --- a/desktop/src-tauri/src/huddle/tts_voice_import.rs +++ b/desktop/src-tauri/src/huddle/tts_voice_import.rs @@ -30,7 +30,10 @@ pub async fn pick_and_import(app: &AppHandle) -> Result, S let (sender, receiver) = tokio::sync::oneshot::channel(); app.dialog() .file() - .add_filter("WAV audio", &["wav"]) + .add_filter( + "Audio", + &["wav", "m4a", "mp3", "flac", "ogg", "oga", "aif", "aiff"], + ) .pick_file(move |path| { let _ = sender.send(path); });