import type { AudioElement, LibraryAudioElement, RetimeConfig, SceneTracks, } from "@/lib/timeline"; import { shouldMaintainPitch } from "@/lib/retime/rate"; import type { MediaAsset } from "@/lib/media/types"; import { applyAudioMasteringToBuffer } from "@/lib/media/audio-mastering"; import type { AudioCapableElement } from "@/lib/timeline/audio-state"; import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; import { doesElementHaveEnabledAudio, } from "@/lib/timeline/audio-separation"; import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTrackHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; import { TICKS_PER_SECOND } from "@/lib/wasm"; const MAX_AUDIO_CHANNELS = 2; const EXPORT_SAMPLE_RATE = 44100; const COARSE_SAMPLE_COUNT = 2048; export interface CollectedAudioElement { timelineElement: AudioCapableElement; buffer: AudioBuffer; startTime: number; duration: number; trimStart: number; trimEnd: number; volume: number; muted: boolean; retime?: RetimeConfig; } export function createAudioContext({ sampleRate, }: { sampleRate?: number; } = {}): AudioContext { const AudioContextConstructor = window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }) .webkitAudioContext; return new AudioContextConstructor(sampleRate ? { sampleRate } : undefined); } export interface DecodedAudio { samples: Float32Array; sampleRate: number; } export async function decodeAudioToFloat32({ audioBlob, sampleRate, }: { audioBlob: Blob; sampleRate?: number; }): Promise { const audioContext = createAudioContext({ sampleRate }); const arrayBuffer = await audioBlob.arrayBuffer(); const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); // mix down to mono const numChannels = audioBuffer.numberOfChannels; const length = audioBuffer.length; const samples = new Float32Array(length); for (let i = 0; i < length; i++) { let sum = 0; for (let channel = 0; channel < numChannels; channel++) { sum += audioBuffer.getChannelData(channel)[i]; } samples[i] = sum / numChannels; } return { samples, sampleRate: audioBuffer.sampleRate }; } export async function collectAudioElements({ tracks, mediaAssets, audioContext, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; audioContext: AudioContext; }): Promise { const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; const mediaMap = new Map( mediaAssets.map((media) => [media.id, media]), ); const pendingElements: Array> = []; for (const track of orderedTracks) { if (canTrackHaveAudio(track) && track.muted) continue; for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; if (element.duration <= 0) continue; const mediaAsset = hasMediaId(element) ? (mediaMap.get(element.mediaId) ?? null) : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const isTrackMuted = canTrackHaveAudio(track) && track.muted; if (element.type === "audio") { pendingElements.push( resolveAudioBufferForElement({ element, mediaMap, audioContext, }).then((audioBuffer) => { if (!audioBuffer) return null; const muted = element.muted === true || isTrackMuted; return { timelineElement: element, buffer: audioBuffer, startTime: element.startTime / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, localTime: 0, }), muted, retime: element.retime, }; }), ); continue; } if (element.type === "video") { if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; pendingElements.push( resolveAudioBufferForVideoElement({ mediaAsset, audioContext, }).then((audioBuffer) => { if (!audioBuffer) return null; const muted = (element.muted ?? false) || isTrackMuted; return { timelineElement: element, buffer: audioBuffer, startTime: element.startTime / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, localTime: 0, }), muted, retime: element.retime, }; }), ); } } } const resolvedElements = await Promise.all(pendingElements); const audioElements: CollectedAudioElement[] = []; for (const element of resolvedElements) { if (element) audioElements.push(element); } return audioElements; } async function resolveAudioBufferForElement({ element, mediaMap, audioContext, }: { element: AudioElement; mediaMap: Map; audioContext: AudioContext; }): Promise { try { if (element.sourceType === "upload") { const asset = mediaMap.get(element.mediaId); if (!asset || asset.type !== "audio") return null; const arrayBuffer = await asset.file.arrayBuffer(); return await audioContext.decodeAudioData(arrayBuffer.slice(0)); } if (element.buffer) return element.buffer; const response = await fetch(element.sourceUrl); if (!response.ok) { throw new Error(`Library audio fetch failed: ${response.status}`); } const arrayBuffer = await response.arrayBuffer(); return await audioContext.decodeAudioData(arrayBuffer.slice(0)); } catch (error) { console.warn("Failed to decode audio:", error); return null; } } async function resolveAudioBufferForVideoElement({ mediaAsset, audioContext, }: { mediaAsset: MediaAsset; audioContext: AudioContext; }): Promise { const input = new Input({ source: new BlobSource(mediaAsset.file), formats: ALL_FORMATS, }); try { const audioTrack = await input.getPrimaryAudioTrack(); if (!audioTrack) return null; const sink = new AudioBufferSink(audioTrack); const targetSampleRate = audioContext.sampleRate; const chunks: AudioBuffer[] = []; let totalSamples = 0; for await (const { buffer } of sink.buffers(0)) { chunks.push(buffer); totalSamples += buffer.length; } if (chunks.length === 0) return null; const nativeSampleRate = chunks[0].sampleRate; const numChannels = Math.min( MAX_AUDIO_CHANNELS, chunks[0].numberOfChannels, ); const nativeChannels = Array.from( { length: numChannels }, () => new Float32Array(totalSamples), ); let offset = 0; for (const chunk of chunks) { for (let channel = 0; channel < numChannels; channel++) { const sourceData = chunk.getChannelData( Math.min(channel, chunk.numberOfChannels - 1), ); nativeChannels[channel].set(sourceData, offset); } offset += chunk.length; } // use OfflineAudioContext for high-quality resampling to target rate const outputSamples = Math.ceil( totalSamples * (targetSampleRate / nativeSampleRate), ); const offlineContext = new OfflineAudioContext( numChannels, outputSamples, targetSampleRate, ); const nativeBuffer = audioContext.createBuffer( numChannels, totalSamples, nativeSampleRate, ); for (let ch = 0; ch < numChannels; ch++) { nativeBuffer.copyToChannel(nativeChannels[ch], ch); } const sourceNode = offlineContext.createBufferSource(); sourceNode.buffer = nativeBuffer; sourceNode.connect(offlineContext.destination); sourceNode.start(0); return await offlineContext.startRendering(); } catch (error) { console.warn("Failed to decode video audio:", error); return null; } finally { input.dispose(); } } interface AudioMixSource { timelineElement: AudioCapableElement; file: File; startTime: number; duration: number; trimStart: number; trimEnd: number; volume: number; retime?: RetimeConfig; } export interface AudioClipSource { timelineElement: AudioCapableElement; id: string; sourceKey: string; file: File; startTime: number; duration: number; trimStart: number; trimEnd: number; volume: number; muted: boolean; retime?: RetimeConfig; } async function fetchLibraryAudioSource({ element, volume, }: { element: LibraryAudioElement; volume: number; }): Promise { try { const response = await fetch(element.sourceUrl); if (!response.ok) { throw new Error(`Library audio fetch failed: ${response.status}`); } const blob = await response.blob(); const file = new File([blob], `${element.name}.mp3`, { type: "audio/mpeg", }); return { timelineElement: element, file, startTime: element.startTime / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, retime: element.retime, }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; } } async function fetchLibraryAudioClip({ element, muted, volume, }: { element: LibraryAudioElement; muted: boolean; volume: number; }): Promise { try { const response = await fetch(element.sourceUrl); if (!response.ok) { throw new Error(`Library audio fetch failed: ${response.status}`); } const blob = await response.blob(); const file = new File([blob], `${element.name}.mp3`, { type: "audio/mpeg", }); return { timelineElement: element, id: element.id, sourceKey: element.id, file, startTime: element.startTime, duration: element.duration, trimStart: element.trimStart, trimEnd: element.trimEnd, volume, muted, retime: element.retime, }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; } } function collectMediaAudioSource({ element, mediaAsset, volume, }: { element: AudioCapableElement; mediaAsset: MediaAsset; volume: number; }): AudioMixSource { return { timelineElement: element, file: mediaAsset.file, startTime: element.startTime / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, retime: element.retime, }; } function collectMediaAudioClip({ element, mediaAsset, muted, volume, }: { element: AudioCapableElement; mediaAsset: MediaAsset; muted: boolean; volume: number; }): AudioClipSource { return { timelineElement: element, id: element.id, sourceKey: mediaAsset.id, file: mediaAsset.file, startTime: element.startTime / TICKS_PER_SECOND, duration: element.duration / TICKS_PER_SECOND, trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, muted, retime: element.retime, }; } export async function collectAudioMixSources({ tracks, mediaAssets, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; }): Promise { const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; const audioMixSources: AudioMixSource[] = []; const mediaMap = new Map( mediaAssets.map((asset) => [asset.id, asset]), ); const pendingLibrarySources: Array> = []; for (const track of orderedTracks) { if (canTrackHaveAudio(track) && track.muted) continue; for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; if (element.muted === true) continue; const mediaAsset = hasMediaId(element) ? (mediaMap.get(element.mediaId) ?? null) : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const volume = resolveEffectiveAudioGain({ element, localTime: 0, }); if (element.type === "audio") { if (element.sourceType === "upload") { const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset) continue; audioMixSources.push( collectMediaAudioSource({ element, mediaAsset, volume }), ); } else { pendingLibrarySources.push( fetchLibraryAudioSource({ element, volume }), ); } continue; } if (element.type === "video") { if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { audioMixSources.push( collectMediaAudioSource({ element, mediaAsset, volume }), ); } } } } const resolvedLibrarySources = await Promise.all(pendingLibrarySources); for (const source of resolvedLibrarySources) { if (source) audioMixSources.push(source); } return audioMixSources; } export async function collectAudioClips({ tracks, mediaAssets, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; }): Promise { const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; const clips: AudioClipSource[] = []; const mediaMap = new Map( mediaAssets.map((asset) => [asset.id, asset]), ); const pendingLibraryClips: Array> = []; for (const track of orderedTracks) { const isTrackMuted = canTrackHaveAudio(track) && track.muted; for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; const mediaAsset = hasMediaId(element) ? (mediaMap.get(element.mediaId) ?? null) : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const isElementMuted = "muted" in element ? (element.muted ?? false) : false; const muted = isTrackMuted || isElementMuted; const volume = resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, localTime: 0, }); if (element.type === "audio") { if (element.sourceType === "upload") { const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset) continue; clips.push( collectMediaAudioClip({ element, mediaAsset, muted, volume, }), ); } else { pendingLibraryClips.push( fetchLibraryAudioClip({ element, muted, volume }), ); } continue; } if (element.type === "video") { if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { clips.push( collectMediaAudioClip({ element, mediaAsset, muted, volume, }), ); } } } } const resolvedLibraryClips = await Promise.all(pendingLibraryClips); for (const clip of resolvedLibraryClips) { if (clip) clips.push(clip); } return clips; } export async function createTimelineAudioBuffer({ tracks, mediaAssets, duration, sampleRate = EXPORT_SAMPLE_RATE, audioContext, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; duration: number; sampleRate?: number; audioContext?: AudioContext; }): Promise { const context = audioContext ?? createAudioContext({ sampleRate }); const audioElements = await collectAudioElements({ tracks, mediaAssets, audioContext: context, }); if (audioElements.length === 0) return null; const outputChannels = 2; const durationSeconds = duration / TICKS_PER_SECOND; const outputLength = Math.ceil(durationSeconds * sampleRate); const outputBuffer = context.createBuffer( outputChannels, outputLength, sampleRate, ); for (const element of audioElements) { if (element.muted) continue; const renderedBuffer = shouldMaintainPitch({ rate: element.retime?.rate ?? 1, maintainPitch: element.retime?.maintainPitch, }) ? await renderRetimedBuffer({ audioContext: context, sourceBuffer: element.buffer, trimStart: element.trimStart, clipDuration: element.duration, retime: element.retime, maintainPitch: true, }) : undefined; mixAudioChannels({ element, buffer: renderedBuffer ?? element.buffer, trimStart: renderedBuffer ? 0 : element.trimStart, retime: renderedBuffer ? undefined : element.retime, outputBuffer, outputLength, sampleRate, }); } return await applyAudioMasteringToBuffer({ audioBuffer: outputBuffer }); } export function computeGlobalMaxRms({ buffer, }: { buffer: AudioBuffer; }): number { const channels = buffer.numberOfChannels; const step = Math.max(1, Math.floor(buffer.length / COARSE_SAMPLE_COUNT)); let globalMax = 0; for (let c = 0; c < channels; c++) { const data = buffer.getChannelData(c); for (let i = 0; i + step <= buffer.length; i += step) { for (let j = i; j < i + step; j++) { const abs = Math.abs(data[j]); if (abs > globalMax) globalMax = abs; } } } return globalMax || 1; } export function extractRmsRange({ buffer, count, startSample, endSample, globalMax, }: { buffer: AudioBuffer; count: number; startSample: number; endSample: number; globalMax: number; }): number[] { const channels = buffer.numberOfChannels; const rangeLength = endSample - startSample; const step = Math.max(1, Math.floor(rangeLength / count)); const peaks = new Float32Array(count); for (let c = 0; c < channels; c++) { const data = buffer.getChannelData(c); for (let i = 0; i < count; i++) { const start = startSample + i * step; const end = Math.min(start + step, endSample); for (let j = start; j < end; j++) { const abs = Math.abs(data[j]); if (abs > peaks[i]) peaks[i] = abs; } } } const norm = 1 / globalMax; const result = new Array(count); for (let i = 0; i < count; i++) result[i] = Math.min(1, peaks[i] * norm); return result; } function mixAudioChannels({ element, buffer, trimStart, retime, outputBuffer, outputLength, sampleRate, }: { element: CollectedAudioElement; buffer: AudioBuffer; trimStart: number; retime?: RetimeConfig; outputBuffer: AudioBuffer; outputLength: number; sampleRate: number; }): void { const { startTime, duration: elementDuration } = element; const outputStartSample = Math.floor(startTime * sampleRate); const renderedLength = Math.ceil(elementDuration * sampleRate); const outputChannels = 2; for (let channel = 0; channel < outputChannels; channel++) { const outputData = outputBuffer.getChannelData(channel); const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1); const sourceData = buffer.getChannelData(sourceChannel); for (let i = 0; i < renderedLength; i++) { const outputIndex = outputStartSample + i; if (outputIndex >= outputLength) break; const clipTime = i / sampleRate; const sourceTime = trimStart + getSourceTimeAtClipTime({ clipTime, retime }); const sourceIndex = sourceTime * buffer.sampleRate; if (sourceIndex >= sourceData.length) break; const lowerIndex = Math.floor(sourceIndex); const upperIndex = Math.min(sourceData.length - 1, lowerIndex + 1); const fraction = sourceIndex - lowerIndex; const gain = hasAnimatedVolume({ element: element.timelineElement }) ? resolveEffectiveAudioGain({ element: element.timelineElement, localTime: clipTime, }) : element.volume; outputData[outputIndex] += (sourceData[lowerIndex] * (1 - fraction) + sourceData[upperIndex] * fraction) * gain; } } }