From f01c3d28323ddf9586e5b205b600f225fe20bb58 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 14 Aug 2026 12:04:58 -0400 Subject: [PATCH] Add inline audio uploads and playback Co-authored-by: Max Signed-off-by: Max --- desktop/playwright.config.ts | 1 + desktop/src-tauri/src/commands/media.rs | 13 +- desktop/src-tauri/src/commands/media_audio.rs | 592 ++++++++++++++++++ desktop/src-tauri/src/commands/mod.rs | 1 + .../messages/lib/imetaMediaMarkdown.test.mjs | 27 + .../messages/lib/imetaMediaMarkdown.ts | 16 +- desktop/src/shared/lib/rehypeImageGallery.ts | 13 +- .../shared/styles/globals/media-controls.css | 53 ++ desktop/src/shared/ui/markdown.test.mjs | 32 +- desktop/src/shared/ui/markdown.tsx | 20 +- .../ui/markdown/MarkdownAudioPlayer.tsx | 123 ++++ .../shared/ui/markdown/mediaEntry.test.mjs | 23 +- desktop/src/shared/ui/markdown/mediaEntry.ts | 24 +- .../e2e/audio-player-screenshots.spec.ts | 123 ++++ 14 files changed, 1029 insertions(+), 32 deletions(-) create mode 100644 desktop/src-tauri/src/commands/media_audio.rs create mode 100644 desktop/src/shared/ui/markdown/MarkdownAudioPlayer.tsx create mode 100644 desktop/tests/e2e/audio-player-screenshots.spec.ts diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index a9ec17a24..6bd982eb9 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -59,6 +59,7 @@ export default defineConfig({ "**/pubkey-display-screenshots.spec.ts", "**/file-attachment.spec.ts", "**/image-attachment-gallery.spec.ts", + "**/audio-player-screenshots.spec.ts", "**/composer-image-draw.spec.ts", "**/video-attachment.spec.ts", "**/spoiler.spec.ts", diff --git a/desktop/src-tauri/src/commands/media.rs b/desktop/src-tauri/src/commands/media.rs index 8da845c07..63999f55d 100644 --- a/desktop/src-tauri/src/commands/media.rs +++ b/desktop/src-tauri/src/commands/media.rs @@ -299,8 +299,9 @@ pub(crate) fn sanitize_image_for_upload(body: Vec, mime: &str) -> Result Result { - let mime = infer::get(body) - .map(|t| t.mime_type().to_string()) + let mime = super::media_audio::sniff_audio_mime(body) + .map(str::to_string) + .or_else(|| infer::get(body).map(|t| t.mime_type().to_string())) .unwrap_or_else(|| "application/octet-stream".to_string()); if BLOCKED_MIME.contains(&mime.as_str()) { return Err(format!("unsupported file type: {mime}")); @@ -415,7 +416,7 @@ pub(crate) async fn upload_image_bytes( if !mime.starts_with("image/") { return Err("profile avatar must be an image".to_string()); } - let body = sanitize_image_for_upload(body, &mime)?; + let body = super::media_audio::sanitize_for_upload(body, &mime)?; do_upload(body, &mime, state, None, None).await } @@ -520,7 +521,7 @@ pub async fn upload_media( } let mime = detect_and_validate_mime(&body)?; - let body = sanitize_image_for_upload(body, &mime)?; + let body = super::media_audio::sanitize_for_upload(body, &mime)?; do_upload(body, &mime, &state, None, None).await } @@ -592,7 +593,7 @@ async fn process_picked_path( .map_err(|e| format!("transcode task failed: {e}"))??; let mime = detect_and_validate_mime(&body)?; - let body = sanitize_image_for_upload(body, &mime)?; + let body = super::media_audio::sanitize_for_upload(body, &mime)?; // Image-only surfaces (e.g. "Send feedback"): reject anything that didn't // sniff as an image, BEFORE the upload leaves the client. @@ -773,7 +774,7 @@ pub(super) async fn upload_media_bytes_inner( }; let mime = detect_and_validate_mime(&body)?; - let body = sanitize_image_for_upload(body, &mime)?; + let body = super::media_audio::sanitize_for_upload(body, &mime)?; // Upload video first, then poster (best-effort). let progress = progress_id.as_ref().map(|id| (app.clone(), id.clone())); diff --git a/desktop/src-tauri/src/commands/media_audio.rs b/desktop/src-tauri/src/commands/media_audio.rs new file mode 100644 index 000000000..8e70621a6 --- /dev/null +++ b/desktop/src-tauri/src/commands/media_audio.rs @@ -0,0 +1,592 @@ +//! Canonical metadata-free audio containers for relay upload admission. + +const EMPTY_VORBIS_COMMENT: &[u8] = b"\x03vorbis\0\0\0\0\0\0\0\0\x01"; + +pub(super) fn sniff_audio_mime(body: &[u8]) -> Option<&'static str> { + if body.len() >= 12 && body.starts_with(b"RIFF") && &body[8..12] == b"WAVE" { + Some("audio/wav") + } else if body.starts_with(b"OggS") { + Some("audio/ogg") + } else if mp3_audio_start(body).is_some() { + Some("audio/mpeg") + } else { + None + } +} + +pub(super) fn sanitize_for_upload(body: Vec, mime: &str) -> Result, String> { + let body = sanitize_audio(body, mime)?; + super::media::sanitize_image_for_upload(body, mime) +} + +pub(super) fn sanitize_audio(body: Vec, mime: &str) -> Result, String> { + match mime { + "audio/mpeg" | "audio/mp3" => sanitize_mp3(&body), + "audio/wav" | "audio/x-wav" | "audio/wave" => sanitize_wav(&body), + "audio/ogg" | "application/ogg" => sanitize_ogg_vorbis(&body), + _ => Ok(body), + } +} + +fn synchsafe(value: &[u8]) -> Option { + (value.len() == 4 && value.iter().all(|byte| byte & 0x80 == 0)).then(|| { + value + .iter() + .fold(0usize, |out, byte| (out << 7) | usize::from(*byte)) + }) +} + +fn mp3_audio_start(body: &[u8]) -> Option { + let start = if body.starts_with(b"ID3") { + if body.len() < 10 { + return None; + } + let size = synchsafe(&body[6..10])?; + let footer = usize::from(body[5] & 0x10 != 0) * 10; + 10usize.checked_add(size)?.checked_add(footer)? + } else { + 0 + }; + mp3_frame_len(body.get(start..start + 4)?).map(|_| start) +} + +fn mp3_frame_len(header: &[u8]) -> Option { + let bits = u32::from_be_bytes(header.try_into().ok()?); + if bits >> 21 != 0x7ff { + return None; + } + let version = (bits >> 19) & 3; + let layer = (bits >> 17) & 3; + let bitrate_index = ((bits >> 12) & 0xf) as usize; + let sample_index = ((bits >> 10) & 3) as usize; + if version == 1 || layer != 1 || bitrate_index == 0 || bitrate_index == 15 || sample_index == 3 + { + return None; + } + const MPEG1_BITRATES: [usize; 16] = [ + 0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0, + ]; + const MPEG2_BITRATES: [usize; 16] = [ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0, + ]; + let bitrate = if version == 3 { + MPEG1_BITRATES[bitrate_index] + } else { + MPEG2_BITRATES[bitrate_index] + } * 1000; + let base_rate = [44_100usize, 48_000, 32_000][sample_index]; + let sample_rate = match version { + 3 => base_rate, + 2 => base_rate / 2, + 0 => base_rate / 4, + _ => return None, + }; + let coefficient = if version == 3 { 144 } else { 72 }; + Some(coefficient * bitrate / sample_rate + ((bits >> 9) & 1) as usize) +} + +fn sanitize_mp3(body: &[u8]) -> Result, String> { + let start = mp3_audio_start(body).ok_or("invalid MP3 header")?; + let mut offset = start; + let mut end = start; + while let Some(header) = body.get(offset..offset + 4) { + let Some(length) = mp3_frame_len(header) else { + break; + }; + let next = offset + .checked_add(length) + .ok_or("MP3 frame length overflow")?; + if next > body.len() { + return Err("truncated MP3 frame".into()); + } + end = next; + offset = next; + } + if end == start { + return Err("MP3 contains no complete frames".into()); + } + // Known and arbitrary trailers are all removed by returning only the walked frame run. + Ok(body[start..end].to_vec()) +} + +fn sanitize_wav(body: &[u8]) -> Result, String> { + if body.len() < 12 || !body.starts_with(b"RIFF") || &body[8..12] != b"WAVE" { + return Err("invalid RIFF/WAVE header".into()); + } + let declared = u32::from_le_bytes(body[4..8].try_into().unwrap()) as usize; + let riff_end = declared.checked_add(8).ok_or("WAV RIFF length overflow")?; + if riff_end > body.len() { + return Err("invalid WAV RIFF length".into()); + } + let mut offset = 12usize; + let mut fmt = None; + let mut data = None; + while offset < riff_end { + if offset + 8 > riff_end { + return Err("truncated WAV chunk".into()); + } + let id = &body[offset..offset + 4]; + let length = u32::from_le_bytes(body[offset + 4..offset + 8].try_into().unwrap()) as usize; + let payload_start = offset + 8; + let payload_end = payload_start + .checked_add(length) + .ok_or("WAV chunk length overflow")?; + let end = payload_end + .checked_add(length & 1) + .ok_or("WAV chunk length overflow")?; + if end > riff_end { + return Err("truncated WAV chunk".into()); + } + match id { + b"fmt " if fmt.is_none() => fmt = Some(body[payload_start..payload_end].to_vec()), + b"fmt " => return Err("WAV contains duplicate fmt chunk".into()), + b"data" if data.is_none() => data = Some(body[payload_start..payload_end].to_vec()), + b"data" => return Err("WAV contains duplicate data chunk".into()), + _ => {} + } + offset = end; + } + let fmt = fmt.ok_or("WAV is missing fmt chunk")?; + let data = data.ok_or("WAV is missing data chunk")?; + let mut out = b"RIFF\0\0\0\0WAVE".to_vec(); + append_riff_chunk(&mut out, b"fmt ", &fmt)?; + append_riff_chunk(&mut out, b"data", &data)?; + let size = u32::try_from(out.len() - 8).map_err(|_| "WAV is too large")?; + out[4..8].copy_from_slice(&size.to_le_bytes()); + Ok(out) +} + +fn append_riff_chunk(out: &mut Vec, id: &[u8; 4], payload: &[u8]) -> Result<(), String> { + out.extend_from_slice(id); + out.extend_from_slice( + &u32::try_from(payload.len()) + .map_err(|_| "WAV chunk is too large")? + .to_le_bytes(), + ); + out.extend_from_slice(payload); + if payload.len() & 1 != 0 { + out.push(0); + } + Ok(()) +} + +#[derive(Clone, Debug)] +struct OggPacket { + bytes: Vec, + granule: u64, +} + +fn sanitize_ogg_vorbis(body: &[u8]) -> Result, String> { + let (serial, mut packets) = parse_ogg(body)?; + if packets + .first() + .is_some_and(|p| p.bytes.starts_with(b"OpusHead")) + { + return Err("Opus audio is not supported".into()); + } + if packets.len() < 3 + || !packets[0].bytes.starts_with(b"\x01vorbis") + || !packets[1].bytes.starts_with(b"\x03vorbis") + || !packets[2].bytes.starts_with(b"\x05vorbis") + { + return Err("invalid Vorbis headers".into()); + } + packets[1].bytes = EMPTY_VORBIS_COMMENT.to_vec(); + let mut out = Vec::new(); + let mut sequence = 0u32; + for (index, packet) in packets[..3].iter().enumerate() { + emit_packet_pages( + &mut out, + serial, + &mut sequence, + packet, + index == 0, + packets.len() == 3 && index == 2, + )?; + } + if packets.len() > 3 { + emit_packet_stream(&mut out, serial, &mut sequence, &packets[3..])?; + } + Ok(out) +} + +fn parse_ogg(body: &[u8]) -> Result<(u32, Vec), String> { + let mut offset = 0usize; + let mut serial = None; + let mut expected_sequence = 0u32; + let mut partial = Vec::new(); + let mut packets = Vec::new(); + let mut saw_eos = false; + while offset < body.len() { + if saw_eos + || offset + 27 > body.len() + || &body[offset..offset + 4] != b"OggS" + || body[offset + 4] != 0 + { + return Err("malformed Ogg page or trailing bytes".into()); + } + let flags = body[offset + 5]; + let granule = u64::from_le_bytes(body[offset + 6..offset + 14].try_into().unwrap()); + let page_serial = u32::from_le_bytes(body[offset + 14..offset + 18].try_into().unwrap()); + let sequence = u32::from_le_bytes(body[offset + 18..offset + 22].try_into().unwrap()); + if serial.get_or_insert(page_serial) != &page_serial { + return Err("multiplexed Ogg streams are not supported".into()); + } + if sequence != expected_sequence { + return Err("invalid Ogg page sequence".into()); + } + expected_sequence = expected_sequence + .checked_add(1) + .ok_or("Ogg sequence overflow")?; + if (flags & 1 != 0) != !partial.is_empty() { + return Err("invalid Ogg packet continuation".into()); + } + if sequence == 0 && flags & 2 == 0 { + return Err("Ogg stream is missing BOS".into()); + } + let segments = body[offset + 26] as usize; + let header_end = offset + 27 + segments; + if header_end > body.len() { + return Err("truncated Ogg lacing table".into()); + } + let payload_len: usize = body[offset + 27..header_end] + .iter() + .map(|v| *v as usize) + .sum(); + let page_end = header_end + .checked_add(payload_len) + .ok_or("Ogg page length overflow")?; + if page_end > body.len() { + return Err("truncated Ogg page".into()); + } + let mut page = body[offset..page_end].to_vec(); + let stored_crc = u32::from_le_bytes(page[22..26].try_into().unwrap()); + page[22..26].fill(0); + if ogg_crc(&page) != stored_crc { + return Err("invalid Ogg page CRC".into()); + } + let mut cursor = header_end; + let first_packet_on_page = packets.len(); + for lace in &body[offset + 27..header_end] { + let end = cursor + *lace as usize; + partial.extend_from_slice(&body[cursor..end]); + cursor = end; + if *lace < 255 { + packets.push(OggPacket { + bytes: std::mem::take(&mut partial), + // An Ogg page's granule applies only to its final completed + // packet. Earlier completions have an unknown position. + granule: u64::MAX, + }); + } + } + if packets.len() > first_packet_on_page { + packets.last_mut().unwrap().granule = granule; + } + saw_eos = flags & 4 != 0; + offset = page_end; + } + if !saw_eos || !partial.is_empty() || packets.is_empty() { + return Err("incomplete Ogg stream".into()); + } + Ok((serial.unwrap(), packets)) +} + +fn emit_packet_stream( + out: &mut Vec, + serial: u32, + sequence: &mut u32, + packets: &[OggPacket], +) -> Result<(), String> { + let mut index = 0usize; + while index < packets.len() { + let mut page_packets = 0usize; + let mut segment_count = 0usize; + let mut payload_len = 0usize; + while index + page_packets < packets.len() { + let packet = &packets[index + page_packets]; + let segments = packet.bytes.len() / 255 + 1; + if page_packets > 0 + && (segments > 255 - segment_count || payload_len + packet.bytes.len() > 65_025) + { + break; + } + if segments > 255 { + break; + } + page_packets += 1; + segment_count += segments; + payload_len += packet.bytes.len(); + // Preserve each source page's completed-packet grouping so its + // granule position remains attached to the same terminal packet. + if packet.granule != u64::MAX { + break; + } + } + if page_packets == 0 { + emit_packet_pages( + out, + serial, + sequence, + &packets[index], + false, + index + 1 == packets.len(), + )?; + index += 1; + continue; + } + let selected = &packets[index..index + page_packets]; + let segment_count: usize = selected + .iter() + .map(|packet| packet.bytes.len() / 255 + 1) + .sum(); + let mut page = Vec::with_capacity(27 + segment_count + payload_len); + page.extend_from_slice(b"OggS\0"); + page.push(if index + page_packets == packets.len() { + 4 + } else { + 0 + }); + page.extend_from_slice(&selected.last().unwrap().granule.to_le_bytes()); + page.extend_from_slice(&serial.to_le_bytes()); + page.extend_from_slice(&sequence.to_le_bytes()); + page.extend_from_slice(&[0; 4]); + page.push(segment_count as u8); + for packet in selected { + page.extend(std::iter::repeat_n(255, packet.bytes.len() / 255)); + page.push((packet.bytes.len() % 255) as u8); + } + for packet in selected { + page.extend_from_slice(&packet.bytes); + } + let crc = ogg_crc(&page); + page[22..26].copy_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&page); + *sequence = sequence.checked_add(1).ok_or("Ogg sequence overflow")?; + index += page_packets; + } + Ok(()) +} + +fn emit_packet_pages( + out: &mut Vec, + serial: u32, + sequence: &mut u32, + packet: &OggPacket, + bos: bool, + eos: bool, +) -> Result<(), String> { + let mut consumed = 0usize; + let mut first = true; + loop { + let remaining = packet.bytes.len() - consumed; + let full = remaining / 255; + let terminates = full < 255; + let segment_count = if terminates { full + 1 } else { 255 }; + let payload_len = if terminates { remaining } else { 255 * 255 }; + let mut page = Vec::with_capacity(27 + segment_count + payload_len); + page.extend_from_slice(b"OggS\0"); + let mut flags = if first && bos { + 2 + } else if !first { + 1 + } else { + 0 + }; + if terminates && eos { + flags |= 4; + } + page.push(flags); + page.extend_from_slice(&(if terminates { packet.granule } else { u64::MAX }).to_le_bytes()); + page.extend_from_slice(&serial.to_le_bytes()); + page.extend_from_slice(&sequence.to_le_bytes()); + page.extend_from_slice(&[0; 4]); + page.push(segment_count as u8); + page.extend(std::iter::repeat_n( + 255, + if terminates { full } else { 255 }, + )); + if terminates { + page.push((remaining % 255) as u8); + } + page.extend_from_slice(&packet.bytes[consumed..consumed + payload_len]); + let crc = ogg_crc(&page); + page[22..26].copy_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&page); + *sequence = sequence.checked_add(1).ok_or("Ogg sequence overflow")?; + consumed += payload_len; + if terminates { + return Ok(()); + } + first = false; + } +} + +fn ogg_crc(bytes: &[u8]) -> u32 { + let mut crc = 0u32; + for byte in bytes { + crc ^= u32::from(*byte) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { + (crc << 1) ^ 0x04c1_1db7 + } else { + crc << 1 + }; + } + } + crc +} + +#[cfg(test)] +mod tests { + use super::*; + + fn mp3_frame(version: u8, bitrate_index: u8, sample_index: u8) -> Vec { + let bits = 0xffe0_0000u32 + | ((version as u32) << 19) + | (1 << 17) + | (1 << 16) + | ((bitrate_index as u32) << 12) + | ((sample_index as u32) << 10); + let header = bits.to_be_bytes(); + let mut frame = vec![0; mp3_frame_len(&header).unwrap()]; + frame[..4].copy_from_slice(&header); + frame + } + + #[test] + fn mp3_strips_id3_and_all_trailing_bytes_for_every_mpeg_generation() { + for version in [3, 2, 0] { + let frame = mp3_frame(version, 9, 0); + let mut tagged = b"ID3\x04\0\0\0\0\0\x03abc".to_vec(); + tagged.extend_from_slice(&frame); + tagged.extend_from_slice(b"TAGarbitrary APE"); + assert_eq!(sanitize_mp3(&tagged).unwrap(), frame); + } + } + + #[test] + fn mp3_rejects_truncated_and_invalid_frames() { + let mut frame = mp3_frame(3, 9, 0); + frame.pop(); + assert!(sanitize_mp3(&frame).is_err()); + assert!(sanitize_mp3(&[0xff, 0xfb, 0xf0, 0]).is_err()); + } + + fn riff(chunks: &[(&[u8; 4], &[u8])]) -> Vec { + let mut out = b"RIFF\0\0\0\0WAVE".to_vec(); + for (id, bytes) in chunks { + append_riff_chunk(&mut out, id, bytes).unwrap(); + } + let len = (out.len() - 8) as u32; + out[4..8].copy_from_slice(&len.to_le_bytes()); + out + } + + #[test] + fn wav_keeps_only_fmt_then_data_with_canonical_padding() { + let input = riff(&[ + (b"LIST", b"metadata"), + (b"data", b"abc"), + (b"fmt ", b"format"), + ]); + let expected = riff(&[(b"fmt ", b"format"), (b"data", b"abc")]); + assert_eq!(sanitize_wav(&input).unwrap(), expected); + } + + #[test] + fn wav_rejects_bad_lengths_and_duplicate_required_chunks() { + let mut truncated = riff(&[(b"fmt ", b"f"), (b"data", b"d")]); + truncated.pop(); + assert!(sanitize_wav(&truncated).is_err()); + assert!(sanitize_wav(&riff(&[(b"fmt ", b"a"), (b"fmt ", b"b"), (b"data", b"d")])).is_err()); + } + + fn packet_page( + serial: u32, + sequence: &mut u32, + packet: &[u8], + bos: bool, + eos: bool, + ) -> Vec { + let packet = OggPacket { + bytes: packet.to_vec(), + granule: 0, + }; + let mut out = Vec::new(); + emit_packet_pages(&mut out, serial, sequence, &packet, bos, eos).unwrap(); + out + } + + #[test] + fn ogg_replaces_comment_and_emits_canonical_crc_and_sequences() { + let mut sequence = 0; + let mut input = packet_page(7, &mut sequence, b"\x01vorbis-ident", true, false); + input.extend(packet_page( + 7, + &mut sequence, + b"\x03vorbis-secret metadata", + false, + false, + )); + input.extend(packet_page( + 7, + &mut sequence, + b"\x05vorbis-setup", + false, + false, + )); + input.extend(packet_page(7, &mut sequence, b"audio", false, true)); + let output = sanitize_ogg_vorbis(&input).unwrap(); + let (_, packets) = parse_ogg(&output).unwrap(); + assert_eq!(packets[1].bytes, EMPTY_VORBIS_COMMENT); + assert!(!output.windows(6).any(|w| w == b"secret")); + assert_eq!(output.windows(4).filter(|w| *w == b"OggS").count(), 4); + } + + #[test] + #[ignore = "requires ffmpeg-generated fixtures; run with BUZZ_AUDIO_FIXTURES"] + fn real_fixtures_emit_sanitized_outputs_for_external_oracles() { + let input_dir = std::path::PathBuf::from( + std::env::var("BUZZ_AUDIO_FIXTURES").expect("BUZZ_AUDIO_FIXTURES input directory"), + ); + let output_dir = input_dir.join("sanitized"); + std::fs::create_dir_all(&output_dir).unwrap(); + for name in [ + "tagged.mp3", + "mpeg2_22k.mp3", + "mpeg25_11k.mp3", + "tagged.wav", + "tagged.ogg", + "bigart.ogg", + "long.ogg", + ] { + let mime = if name.ends_with(".mp3") { + "audio/mpeg" + } else if name.ends_with(".wav") { + "audio/wav" + } else { + "audio/ogg" + }; + let body = std::fs::read(input_dir.join(name)).unwrap(); + let sanitized = sanitize_audio(body, mime).unwrap(); + std::fs::write(output_dir.join(name), sanitized).unwrap(); + } + } + + #[test] + fn ogg_rejects_opus_crc_and_sequence_corruption() { + let mut sequence = 0; + let opus = packet_page(1, &mut sequence, b"OpusHead", true, true); + assert!(sanitize_ogg_vorbis(&opus).unwrap_err().contains("Opus")); + let mut broken = opus.clone(); + broken[22] ^= 1; + assert!(parse_ogg(&broken).unwrap_err().contains("CRC")); + let mut broken = opus; + broken[18] = 2; + broken[22..26].fill(0); + let crc = ogg_crc(&broken); + broken[22..26].copy_from_slice(&crc.to_le_bytes()); + assert!(parse_ogg(&broken).unwrap_err().contains("sequence")); + } +} diff --git a/desktop/src-tauri/src/commands/mod.rs b/desktop/src-tauri/src/commands/mod.rs index 1ab3bb70d..5e0cb53f4 100644 --- a/desktop/src-tauri/src/commands/mod.rs +++ b/desktop/src-tauri/src/commands/mod.rs @@ -27,6 +27,7 @@ mod legacy_storage; mod link_preview; pub(crate) mod media; mod media_animated; +mod media_audio; mod media_download; mod media_gif; mod media_raw; diff --git a/desktop/src/features/messages/lib/imetaMediaMarkdown.test.mjs b/desktop/src/features/messages/lib/imetaMediaMarkdown.test.mjs index 79c193fb1..bf5dfd646 100644 --- a/desktop/src/features/messages/lib/imetaMediaMarkdown.test.mjs +++ b/desktop/src/features/messages/lib/imetaMediaMarkdown.test.mjs @@ -181,6 +181,33 @@ test("buildImetaTags keeps media filenames in imeta", () => { ); }); +test("formatImetaMediaLine: audio mime → ![audio] line (regardless of URL suffix)", () => { + assert.equal( + formatImetaMediaLine({ url: "https://cdn/blob/xyz", type: "audio/mpeg" }), + "\n![audio](https://cdn/blob/xyz)", + ); +}); + +test("formatImetaMediaLine: spoilered audio → wrapped ![audio] line", () => { + assert.equal( + formatImetaMediaLine( + { url: "https://cdn/blob/xyz", type: "audio/ogg" }, + { spoiler: true }, + ), + "\n||![audio](https://cdn/blob/xyz)||", + ); +}); + +test("audio round-trips through edit stripping and spoiler recovery", () => { + const media = [{ url: "https://b/audio", type: "audio/wav" }]; + const body = "note\n||![audio](https://b/audio)||"; + assert.deepEqual( + [...findSpoileredImetaMediaUrls(body, media)], + ["https://b/audio"], + ); + assert.equal(stripImetaMediaLines(body, media), "note"); +}); + test("formatImetaMediaLine: video mime → ![video] line (regardless of URL suffix)", () => { assert.equal( formatImetaMediaLine({ url: "https://cdn/blob/xyz", type: "video/mp4" }), diff --git a/desktop/src/features/messages/lib/imetaMediaMarkdown.ts b/desktop/src/features/messages/lib/imetaMediaMarkdown.ts index fde849228..ae5d2b0fa 100644 --- a/desktop/src/features/messages/lib/imetaMediaMarkdown.ts +++ b/desktop/src/features/messages/lib/imetaMediaMarkdown.ts @@ -13,7 +13,7 @@ * - on edit-load, seed the composer's `pendingImeta` with the original * event's imeta entries (full BlobDescriptor shape, so the send-path * mediaTags builder works unchanged); strip any matching trailing - * `![image|video](url)` lines from the body so the user only sees text; + * `![image|video|audio](url)` lines from the body so the user only sees text; * - on submit, pass `mediaTags` (built from the current `pendingImeta`) * alongside the edited content so the edit event carries a full new * imeta tag set; @@ -105,9 +105,9 @@ export function buildImetaTags( } const MEDIA_LINE_RE = - /^(?:\|\|)?!\[(?:image|video)\]\(([^)\s]+)\)(?:\|\|)?\s*$/; + /^(?:\|\|)?!\[(?:audio|image|video)\]\(([^)\s]+)\)(?:\|\|)?\s*$/; const SPOILERED_MEDIA_LINE_RE = - /^\|\|!\[(?:image|video)\]\(([^)\s]+)\)\|\|\s*$/; + /^\|\|!\[(?:audio|image|video)\]\(([^)\s]+)\)\|\|\s*$/; const BLOCK_SPOILER_DELIMITER_RE = /^\s*\|\|\s*$/; /** * Matches a generic file-attachment line `[label](url)` (no leading `!`, so it's @@ -183,7 +183,7 @@ function findTrailingBlockSpoilerMediaStart( } /** - * Remove trailing `![image|video](url)` lines whose URL matches an entry in + * Remove trailing `![image|video|audio](url)` lines whose URL matches an entry in * `imetaMedia`. Stops at the first non-matching/non-blank line so attachments * that have been moved or interleaved with text are left alone (the composer * only ever produces trailing lines, but defending against shape drift is @@ -271,7 +271,7 @@ export function findSpoileredImetaMediaUrls( /** * Format a single imeta entry as a leading-newline markdown line. * - * Images and video use `![image|video](url)` so the renderer draws them inline. + * Images, video, and audio use `![image|video|audio](url)` so the renderer draws them inline. * Generic files use a plain `[filename](url)` link — the renderer recognises the * href as a local media blob with a non-media MIME and upgrades it to a file * card. Agent snapshot PNGs deliberately use the file-link form despite their @@ -286,6 +286,10 @@ export function formatImetaMediaLine( const lower = filename?.toLowerCase(); const isSnapshotPng = lower?.endsWith(".agent.png") || lower?.endsWith(".team.png"); + if (type.startsWith("audio/")) { + const line = `![audio](${url})`; + return options.spoiler ? `\n||${line}||` : `\n${line}`; + } if (type.startsWith("video/")) { const line = `![video](${url})`; return options.spoiler ? `\n||${line}||` : `\n${line}`; @@ -308,7 +312,7 @@ export function formatImetaMediaLine( /** * Build the body + tags pair for an outgoing message (initial send or - * edit). Appends `![image|video](url)` markdown lines for each attachment + * edit). Appends `![image|video|audio](url)` markdown lines for each attachment * to the body so the renderer (which keys on URLs literally present in * the content) draws them, and returns the matching imeta tag set. * diff --git a/desktop/src/shared/lib/rehypeImageGallery.ts b/desktop/src/shared/lib/rehypeImageGallery.ts index 08d74945f..dbf55a1a7 100644 --- a/desktop/src/shared/lib/rehypeImageGallery.ts +++ b/desktop/src/shared/lib/rehypeImageGallery.ts @@ -39,6 +39,12 @@ function isText(node: HastNode): node is HastText { return node.type === "text"; } +function isGalleryImage(node: HastNode): node is HastElement { + return ( + isElement(node) && node.tagName === "img" && node.properties.alt !== "audio" + ); +} + function isIgnorableImageSeparator(node: HastNode): boolean { return ( (isText(node) && node.value.trim() === "") || @@ -56,8 +62,7 @@ function isImageOnlyParagraph(node: HastNode): node is HastElement { ); return ( - meaningful.length >= 1 && - meaningful.every((child) => isElement(child) && child.tagName === "img") + meaningful.length >= 1 && meaningful.every((child) => isGalleryImage(child)) ); } @@ -78,7 +83,7 @@ function splitTrailingImageRun(node: HastNode): HastNode[] { while (cursor >= 0) { const child = node.children[cursor]; - if (isElement(child) && child.tagName === "img") { + if (isGalleryImage(child)) { trailingImages.unshift(child); cursor -= 1; continue; @@ -119,7 +124,7 @@ export default function rehypeImageGallery() { const allImages: HastNode[] = []; for (const p of imageRun) { for (const child of p.children) { - if (isElement(child) && child.tagName === "img") { + if (isGalleryImage(child)) { allImages.push(child); } } diff --git a/desktop/src/shared/styles/globals/media-controls.css b/desktop/src/shared/styles/globals/media-controls.css index 58a82ab4f..63fa34451 100644 --- a/desktop/src/shared/styles/globals/media-controls.css +++ b/desktop/src/shared/styles/globals/media-controls.css @@ -52,3 +52,56 @@ background: white; box-shadow: 0 1px 3px rgb(0 0 0 / 0.3); } + +/* ── Audio player scrubber (shared/ui/markdown/MarkdownAudioPlayer) ───── */ +.audio-progress-slider { + appearance: none; + height: 16px; + background: transparent; +} + +.audio-progress-slider:focus-visible { + outline: 2px solid var(--ring); + outline-offset: 2px; +} + +.audio-progress-slider::-webkit-slider-runnable-track { + height: 4px; + border-radius: 9999px; + background: linear-gradient( + to right, + currentColor var(--audio-progress-fill, 0%), + color-mix(in srgb, currentColor 20%, transparent) + var(--audio-progress-fill, 0%) + ); +} + +.audio-progress-slider::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + margin-top: -4px; + height: 12px; + width: 12px; + border-radius: 9999px; + background: currentColor; +} + +.audio-progress-slider::-moz-range-track { + height: 4px; + border-radius: 9999px; + background: color-mix(in srgb, currentColor 20%, transparent); +} + +.audio-progress-slider::-moz-range-progress { + height: 4px; + border-radius: 9999px; + background: currentColor; +} + +.audio-progress-slider::-moz-range-thumb { + border: none; + height: 12px; + width: 12px; + border-radius: 9999px; + background: currentColor; +} diff --git a/desktop/src/shared/ui/markdown.test.mjs b/desktop/src/shared/ui/markdown.test.mjs index 08168c105..3e63d17b0 100644 --- a/desktop/src/shared/ui/markdown.test.mjs +++ b/desktop/src/shared/ui/markdown.test.mjs @@ -73,8 +73,15 @@ function isHastImageOnlyParagraph(node) { (child) => !isIgnorableImageSeparator(child), ); return ( - meaningful.length >= 1 && - meaningful.every((child) => isHastElement(child) && child.tagName === "img") + meaningful.length >= 1 && meaningful.every((child) => isGalleryImage(child)) + ); +} + +function isGalleryImage(node) { + return ( + isHastElement(node) && + node.tagName === "img" && + node.properties.alt !== "audio" ); } @@ -92,7 +99,7 @@ function splitTrailingImageRun(node) { const trailingImages = []; while (cursor >= 0) { const child = node.children[cursor]; - if (isHastElement(child) && child.tagName === "img") { + if (isGalleryImage(child)) { trailingImages.unshift(child); cursor -= 1; continue; @@ -129,7 +136,7 @@ function rehypeImageGallery() { const allImages = []; for (const p of imageRun) { for (const child of p.children) { - if (isHastElement(child) && child.tagName === "img") { + if (isGalleryImage(child)) { allImages.push(child); } } @@ -425,6 +432,23 @@ test("rehypeImageGallery: three consecutive images merge into one paragraph", () assert.equal(tree.children[0].children.length, 3); }); +test("rehypeImageGallery: audio attachments are not grouped as images", () => { + const audio = (src) => ({ + ...hastImg(src), + properties: { src, alt: "audio" }, + }); + const tree = { + type: "root", + children: [hastP(audio("a.mp3")), hastP(audio("b.ogg"))], + }; + + rehypeImageGallery()(tree); + + assert.equal(tree.children.length, 2); + assert.equal(tree.children[0].children[0].properties.src, "a.mp3"); + assert.equal(tree.children[1].children[0].properties.src, "b.ogg"); +}); + test("rehypeImageGallery: single image paragraph is not grouped", () => { const tree = { type: "root", diff --git a/desktop/src/shared/ui/markdown.tsx b/desktop/src/shared/ui/markdown.tsx index 433a8ce6e..7a5bf4a8b 100644 --- a/desktop/src/shared/ui/markdown.tsx +++ b/desktop/src/shared/ui/markdown.tsx @@ -69,7 +69,7 @@ import { type MediaContextMenuPosition, useDismissMediaContextMenu, } from "./markdown/MediaContextMenu"; -import { isVideoMedia } from "./markdown/mediaEntry"; +import { isAudioMedia, isVideoMedia } from "./markdown/mediaEntry"; import { clampImageLightboxZoom, type ImageGalleryDirection, @@ -109,6 +109,7 @@ import { normalizedWheelDeltaY, visibleImageGalleryForTrigger, } from "./markdown/imageLightbox"; +import { MarkdownAudioAttachment } from "./markdown/MarkdownAudioPlayer"; import { MarkdownTable } from "./markdown/MarkdownTable"; import { ProgressiveImage } from "./markdown/ProgressiveImage"; import { MessageLinkPill } from "./markdown/MessageLinkPill"; @@ -1505,10 +1506,17 @@ function createMarkdownComponents( const { imetaByUrl } = useMarkdownRuntime(); const entry = src ? imetaByUrl?.get(src) : undefined; const isVideo = src ? isVideoMedia(src, entry?.m) : false; + const isAudio = src ? isAudioMedia(src, entry?.m) : false; if (!interactive) { - const fallbackLabel = isVideo ? "Video attachment" : "Image attachment"; + const fallbackLabel = isAudio + ? "Audio attachment" + : isVideo + ? "Video attachment" + : "Image attachment"; return {alt?.trim() || fallbackLabel}; } + if (isAudio) + return ; const resolvedSrc = src ? rewriteRelayUrl(src) : src; if (isVideo && src && resolvedSrc) { @@ -1547,15 +1555,8 @@ function createMarkdownComponents(
    {children}
), p: ({ children }) => { - // Detect media-only paragraphs (images +
from remarkBreaks). - // Multi-image: render as a compact, count-aware mosaic. Two images split - // a row, three form a hero-and-stack triptych, and larger odd counts let - // the final image span both columns. - // Single media: render as a plain
to avoid invalid

nesting - // (the img component returns block-level wrappers for lightbox/video). const childArray = React.Children.toArray(children); const { imageChildren } = classifyChildren(childArray); - if (isImageOnlyParagraph(childArray)) { return {imageChildren}; } @@ -1563,7 +1564,6 @@ function createMarkdownComponents( if (hasBlockMedia(childArray)) { return
{children}
; } - return

{children}

; }, pre: ({ children }) => { diff --git a/desktop/src/shared/ui/markdown/MarkdownAudioPlayer.tsx b/desktop/src/shared/ui/markdown/MarkdownAudioPlayer.tsx new file mode 100644 index 000000000..83cfce45e --- /dev/null +++ b/desktop/src/shared/ui/markdown/MarkdownAudioPlayer.tsx @@ -0,0 +1,123 @@ +import * as React from "react"; +import { Pause, Play } from "lucide-react"; + +import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; + +import { isAudioMedia } from "./mediaEntry"; +import type { ImetaEntry } from "./types"; + +function formatTime(seconds: number): string { + if (!Number.isFinite(seconds) || seconds < 0) return "0:00"; + const minutes = Math.floor(seconds / 60); + const remainder = Math.floor(seconds % 60); + return `${minutes}:${remainder.toString().padStart(2, "0")}`; +} + +export function MarkdownAudioPlayer({ + alt, + resolvedSrc, +}: { + alt?: string; + resolvedSrc: string; +}) { + const audioRef = React.useRef(null); + const sliderRef = React.useRef(null); + const timeRef = React.useRef(null); + const [playing, setPlaying] = React.useState(false); + + const syncProgress = React.useCallback((audio: HTMLAudioElement) => { + const duration = Number.isFinite(audio.duration) ? audio.duration : 0; + const currentTime = Math.min(audio.currentTime, duration); + if (sliderRef.current) { + sliderRef.current.max = String(duration); + sliderRef.current.value = String(currentTime); + sliderRef.current.style.setProperty( + "--audio-progress-fill", + `${duration > 0 ? (currentTime / duration) * 100 : 0}%`, + ); + } + if (timeRef.current) { + timeRef.current.textContent = `${formatTime(currentTime)} / ${formatTime(duration)}`; + } + }, []); + + const togglePlayback = React.useCallback(() => { + const audio = audioRef.current; + if (!audio) return; + if (audio.paused) { + void audio.play(); + } else { + audio.pause(); + } + }, []); + + return ( + + {/* biome-ignore lint/a11y/useMediaCaption: user-uploaded audio has no caption track */} + + ); +} + +export function MarkdownAudioAttachment({ + alt, + entry, + src, +}: { + alt?: string; + entry?: ImetaEntry; + src?: string; +}): React.ReactNode { + if (!src || !isAudioMedia(src, entry?.m)) return null; + return ( + + + + ); +} diff --git a/desktop/src/shared/ui/markdown/mediaEntry.test.mjs b/desktop/src/shared/ui/markdown/mediaEntry.test.mjs index bbd10898b..bb9556f89 100644 --- a/desktop/src/shared/ui/markdown/mediaEntry.test.mjs +++ b/desktop/src/shared/ui/markdown/mediaEntry.test.mjs @@ -1,11 +1,32 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { isRelayDownloadable, isVideoMedia } from "./mediaEntry.ts"; +import { + isAudioMedia, + isRelayDownloadable, + isVideoMedia, +} from "./mediaEntry.ts"; const RELAY = "https://relay.example.com"; const relayUrl = (name) => `${RELAY}/media/${name}`; +// ── isAudioMedia: MIME-first classification ────────────────────────────── + +test("isAudioMedia: supported MIME classifies regardless of extension", () => { + assert.equal(isAudioMedia(relayUrl("abc"), "audio/mpeg"), true); + assert.equal(isAudioMedia(relayUrl("abc.jpg"), "audio/ogg"), true); + assert.equal(isAudioMedia(relayUrl("abc.wav"), "audio/x-wav"), true); + assert.equal(isAudioMedia(relayUrl("abc.mp3"), "audio/aac"), false); + assert.equal(isAudioMedia(relayUrl("abc.mp3"), "image/png"), false); +}); + +test("isAudioMedia: wav/mp3/ogg extensions support legacy events", () => { + assert.equal(isAudioMedia(relayUrl("abc.wav")), true); + assert.equal(isAudioMedia(relayUrl("abc.MP3?v=2")), true); + assert.equal(isAudioMedia(relayUrl("abc.ogg#t=10")), true); + assert.equal(isAudioMedia(relayUrl("abc.m4a")), false); +}); + // ── isVideoMedia: MIME-first classification ────────────────────────────── test("isVideoMedia: video/* MIME classifies as video regardless of extension", () => { diff --git a/desktop/src/shared/ui/markdown/mediaEntry.ts b/desktop/src/shared/ui/markdown/mediaEntry.ts index 8cc0160de..7bfd0fbf2 100644 --- a/desktop/src/shared/ui/markdown/mediaEntry.ts +++ b/desktop/src/shared/ui/markdown/mediaEntry.ts @@ -13,7 +13,8 @@ * Kept DOM-free so the branch logic is unit-testable without a webview. */ -/** Legacy video extensions, used only when an imeta MIME type is absent. */ +/** Legacy media extensions, used only when an imeta MIME type is absent. */ +const AUDIO_EXTENSIONS = ["mp3", "ogg", "wav"] as const; const VIDEO_EXTENSIONS = ["mp4", "webm", "mov"] as const; /** The lowercased path extension of a URL, ignoring query strings and hashes. */ @@ -30,6 +31,27 @@ function urlPathExtension(src: string): string | undefined { return pathname.slice(lastDot + 1).toLowerCase(); } +/** Exact audio MIME types supported by the upload contract. */ +const AUDIO_MIME_TYPES = [ + "audio/mpeg", + "audio/ogg", + "audio/wav", + "audio/x-wav", +] as const; + +/** Whether `src` should render as audio. MIME is authoritative when present. */ +export function isAudioMedia(src: string, imetaMime?: string): boolean { + if (imetaMime) { + return (AUDIO_MIME_TYPES as readonly string[]).includes( + imetaMime.toLowerCase(), + ); + } + const ext = urlPathExtension(src); + return ( + ext !== undefined && (AUDIO_EXTENSIONS as readonly string[]).includes(ext) + ); +} + /** * Whether `src` should render as a video. * diff --git a/desktop/tests/e2e/audio-player-screenshots.spec.ts b/desktop/tests/e2e/audio-player-screenshots.spec.ts new file mode 100644 index 000000000..198c6d89d --- /dev/null +++ b/desktop/tests/e2e/audio-player-screenshots.spec.ts @@ -0,0 +1,123 @@ +import { expect, test } from "@playwright/test"; +import { installMockBridge } from "../helpers/bridge"; + +const CHANNEL = "general"; +const AUDIO_SHA = "a".repeat(64); + +test("audio attachments stay compact and avoid time-progression work", async ({ + page, +}, testInfo) => { + await page.addInitScript(() => { + Object.defineProperty(HTMLMediaElement.prototype, "play", { + configurable: true, + value() { + this.dispatchEvent(new Event("play")); + return Promise.resolve(); + }, + }); + Object.defineProperty(HTMLMediaElement.prototype, "pause", { + configurable: true, + value() { + this.dispatchEvent(new Event("pause")); + }, + }); + Object.defineProperty(HTMLMediaElement.prototype, "duration", { + configurable: true, + get() { + return 75; + }, + }); + }); + await installMockBridge(page); + await page.goto("/"); + await page.getByTestId(`channel-${CHANNEL}`).click(); + await expect(page.getByTestId("message-timeline")).toBeVisible(); + await expect + .poll( + () => + page.evaluate( + () => + window.__BUZZ_E2E_HAS_MOCK_LIVE_SUBSCRIPTION__?.({ + channelName: "general", + }) ?? false, + ), + { timeout: 5_000 }, + ) + .toBe(true); + + const attachments = Array.from({ length: 6 }, (_, index) => { + const url = `http://localhost:3000/media/${AUDIO_SHA}${index}.mp3`; + return { + line: `![Audio fixture ${index + 1}](${url})`, + tag: [ + "imeta", + `url ${url}`, + "m audio/mpeg", + `filename fixture-${index + 1}.mp3`, + ], + }; + }); + await page.evaluate((attachments) => { + window.__BUZZ_E2E_EMIT_MOCK_MESSAGE__?.({ + channelName: "general", + content: `Six compact audio attachments\n\n${attachments.map(({ line }) => line).join("\n\n")}`, + extraTags: attachments.map(({ tag }) => tag), + }); + }, attachments); + + const players = page.locator("[data-audio-player]"); + await expect(players).toHaveCount(6); + await expect(players.first()).toBeVisible(); + await expect(page.locator("[data-image-mosaic]")).toHaveCount(0); + + await page.evaluate(() => { + const hook = { activeMutations: 0, idleMutations: 0 }; + ( + window as typeof window & { + __AUDIO_PERF__?: typeof hook; + } + ).__AUDIO_PERF__ = hook; + document.querySelectorAll("[data-audio-player]").forEach((node, index) => { + new MutationObserver((records) => { + if (index === 0) hook.activeMutations += records.length; + else hook.idleMutations += records.length; + }).observe(node, { + attributes: true, + characterData: true, + childList: true, + subtree: true, + }); + }); + }); + + const firstPlayer = players.first(); + await firstPlayer.getByRole("button", { name: "Play audio" }).click(); + const pauseButton = firstPlayer.getByRole("button", { name: "Pause audio" }); + await expect(pauseButton).toBeVisible(); + await page.evaluate(() => { + const hook = ( + window as typeof window & { + __AUDIO_PERF__?: { activeMutations: number; idleMutations: number }; + } + ).__AUDIO_PERF__; + if (hook) hook.activeMutations = 0; + }); + const audio = firstPlayer.locator("audio"); + for (const currentTime of [15, 30, 45]) { + await audio.evaluate((element, value) => { + element.currentTime = value; + element.dispatchEvent(new Event("timeupdate")); + }, currentTime); + } + await expect(firstPlayer.getByText("0:45 / 1:15")).toBeVisible(); + const evidence = await page.evaluate( + () => + (window as typeof window & { __AUDIO_PERF__?: unknown }).__AUDIO_PERF__, + ); + expect(evidence).toEqual({ activeMutations: 9, idleMutations: 0 }); + + await page.screenshot({ + path: testInfo.outputPath("audio-player-final.png"), + fullPage: true, + }); +});