import type { EditorCore } from "@/core"; import type { AudioClipSource } from "@/lib/media/audio"; import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; import { ALL_FORMATS, AudioBufferSink, BlobSource, Input, type WrappedAudioBuffer, } from "mediabunny"; export class AudioManager { private audioContext: AudioContext | null = null; private masterGain: GainNode | null = null; private playbackStartTime = 0; private playbackStartContextTime = 0; private scheduleTimer: number | null = null; private lookaheadSeconds = 2; private scheduleIntervalMs = 500; private clips: AudioClipSource[] = []; private sinks = new Map(); private inputs = new Map(); private activeClipIds = new Set(); private clipIterators = new Map< string, AsyncGenerator >(); private queuedSources = new Set(); private playbackSessionId = 0; private lastIsPlaying = false; private lastVolume = 1; private playbackLatencyCompensationSeconds = 0; private unsubscribers: Array<() => void> = []; constructor(private editor: EditorCore) { this.lastVolume = this.editor.playback.getVolume(); this.unsubscribers.push( this.editor.playback.subscribe(this.handlePlaybackChange), this.editor.timeline.subscribe(this.handleTimelineChange), this.editor.media.subscribe(this.handleTimelineChange), ); if (typeof window !== "undefined") { window.addEventListener("playback-seek", this.handleSeek); } } dispose(): void { this.stopPlayback(); for (const unsub of this.unsubscribers) { unsub(); } this.unsubscribers = []; if (typeof window !== "undefined") { window.removeEventListener("playback-seek", this.handleSeek); } this.disposeSinks(); if (this.audioContext) { void this.audioContext.close(); this.audioContext = null; this.masterGain = null; } } private handlePlaybackChange = (): void => { const isPlaying = this.editor.playback.getIsPlaying(); const volume = this.editor.playback.getVolume(); if (volume !== this.lastVolume) { this.lastVolume = volume; this.updateGain(); } if (isPlaying !== this.lastIsPlaying) { this.lastIsPlaying = isPlaying; if (isPlaying) { void this.startPlayback({ time: this.editor.playback.getCurrentTime(), }); } else { this.stopPlayback(); } } }; private handleSeek = (event: Event): void => { const detail = (event as CustomEvent<{ time: number }>).detail; if (!detail) return; if (this.editor.playback.getIsScrubbing()) { this.stopPlayback(); return; } if (this.editor.playback.getIsPlaying()) { void this.startPlayback({ time: detail.time }); return; } this.stopPlayback(); }; private handleTimelineChange = (): void => { this.disposeSinks(); if (!this.editor.playback.getIsPlaying()) return; void this.startPlayback({ time: this.editor.playback.getCurrentTime() }); }; private ensureAudioContext(): AudioContext | null { if (this.audioContext) return this.audioContext; if (typeof window === "undefined") return null; this.audioContext = createAudioContext(); this.masterGain = this.audioContext.createGain(); this.masterGain.gain.value = this.lastVolume; this.masterGain.connect(this.audioContext.destination); return this.audioContext; } private updateGain(): void { if (!this.masterGain) return; this.masterGain.gain.value = this.lastVolume; } private getPlaybackTime(): number { if (!this.audioContext) return this.playbackStartTime; const elapsed = this.audioContext.currentTime - this.playbackStartContextTime; return this.playbackStartTime + elapsed; } private async startPlayback({ time }: { time: number }): Promise { const audioContext = this.ensureAudioContext(); if (!audioContext) return; this.stopPlayback(); this.playbackSessionId++; this.playbackLatencyCompensationSeconds = 0; const tracks = this.editor.timeline.getTracks(); const mediaAssets = this.editor.media.getAssets(); const duration = this.editor.timeline.getTotalDuration(); if (duration <= 0) return; if (audioContext.state === "suspended") { await audioContext.resume(); } this.clips = await collectAudioClips({ tracks, mediaAssets }); if (!this.editor.playback.getIsPlaying()) return; this.playbackStartTime = time; this.playbackStartContextTime = audioContext.currentTime; this.scheduleUpcomingClips(); if (typeof window !== "undefined") { this.scheduleTimer = window.setInterval(() => { this.scheduleUpcomingClips(); }, this.scheduleIntervalMs); } } private scheduleUpcomingClips(): void { if (!this.editor.playback.getIsPlaying()) return; const currentTime = this.getPlaybackTime(); const windowEnd = currentTime + this.lookaheadSeconds; for (const clip of this.clips) { if (clip.muted) continue; if (this.activeClipIds.has(clip.id)) continue; const clipEnd = clip.startTime + clip.duration; if (clipEnd <= currentTime) continue; if (clip.startTime > windowEnd) continue; this.activeClipIds.add(clip.id); void this.runClipIterator({ clip, startTime: currentTime, sessionId: this.playbackSessionId, }); } } private stopPlayback(): void { if (this.scheduleTimer && typeof window !== "undefined") { window.clearInterval(this.scheduleTimer); } this.scheduleTimer = null; for (const iterator of this.clipIterators.values()) { void iterator.return(); } this.clipIterators.clear(); this.activeClipIds.clear(); for (const source of this.queuedSources) { try { source.stop(); } catch {} source.disconnect(); } this.queuedSources.clear(); } private async runClipIterator({ clip, startTime, sessionId, }: { clip: AudioClipSource; startTime: number; sessionId: number; }): Promise { const audioContext = this.ensureAudioContext(); if (!audioContext) return; const sink = await this.getAudioSink({ clip }); if (!sink || !this.editor.playback.getIsPlaying()) return; if (sessionId !== this.playbackSessionId) return; const clipStart = clip.startTime; const clipEnd = clip.startTime + clip.duration; const playbackTimeAfterSinkReady = this.getPlaybackTime(); const iteratorStartTime = Math.max( startTime, clipStart, playbackTimeAfterSinkReady, ); if (iteratorStartTime >= clipEnd) { return; } const sourceStartTime = clip.trimStart + (iteratorStartTime - clip.startTime); const iterator = sink.buffers(sourceStartTime); this.clipIterators.set(clip.id, iterator); let consecutiveDroppedBufferCount = 0; for await (const { buffer, timestamp } of iterator) { if (!this.editor.playback.getIsPlaying()) return; if (sessionId !== this.playbackSessionId) return; const timelineTime = clip.startTime + (timestamp - clip.trimStart); if (timelineTime >= clipEnd) break; const node = audioContext.createBufferSource(); node.buffer = buffer; node.connect(this.masterGain ?? audioContext.destination); const startTimestamp = this.playbackStartContextTime + this.playbackLatencyCompensationSeconds + (timelineTime - this.playbackStartTime); if (startTimestamp >= audioContext.currentTime) { node.start(startTimestamp); consecutiveDroppedBufferCount = 0; } else { const offset = audioContext.currentTime - startTimestamp; if (offset < buffer.duration) { node.start(audioContext.currentTime, offset); consecutiveDroppedBufferCount = 0; } else { consecutiveDroppedBufferCount += 1; if (consecutiveDroppedBufferCount >= 5) { const nextCompensationSeconds = Math.max( this.playbackLatencyCompensationSeconds, Math.min(0.25, offset + 0.01), ); if ( nextCompensationSeconds > this.playbackLatencyCompensationSeconds + 0.001 ) { this.playbackLatencyCompensationSeconds = nextCompensationSeconds; } const resyncStartTime = this.getPlaybackTime(); this.clipIterators.delete(clip.id); void this.runClipIterator({ clip, startTime: resyncStartTime, sessionId, }); return; } continue; } } this.queuedSources.add(node); node.addEventListener("ended", () => { node.disconnect(); this.queuedSources.delete(node); }); const aheadTime = timelineTime - this.getPlaybackTime(); if (aheadTime >= 1) { await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); if (sessionId !== this.playbackSessionId) return; } } this.clipIterators.delete(clip.id); // don't remove from activeClipIds - prevents scheduler from restarting this clip // the set is cleared on stopPlayback anyway } private waitUntilCaughtUp({ timelineTime, targetAhead, }: { timelineTime: number; targetAhead: number; }): Promise { return new Promise((resolve) => { const checkInterval = setInterval(() => { if (!this.editor.playback.getIsPlaying()) { clearInterval(checkInterval); resolve(); return; } const playbackTime = this.getPlaybackTime(); if (timelineTime - playbackTime < targetAhead) { clearInterval(checkInterval); resolve(); } }, 100); }); } private disposeSinks(): void { for (const iterator of this.clipIterators.values()) { void iterator.return(); } this.clipIterators.clear(); this.activeClipIds.clear(); for (const input of this.inputs.values()) { input.dispose(); } this.inputs.clear(); this.sinks.clear(); } private async getAudioSink({ clip, }: { clip: AudioClipSource; }): Promise { const existingSink = this.sinks.get(clip.sourceKey); if (existingSink) return existingSink; try { const input = new Input({ source: new BlobSource(clip.file), formats: ALL_FORMATS, }); const audioTrack = await input.getPrimaryAudioTrack(); if (!audioTrack) { input.dispose(); return null; } const sink = new AudioBufferSink(audioTrack); this.inputs.set(clip.sourceKey, input); this.sinks.set(clip.sourceKey, sink); return sink; } catch (error) { console.warn("Failed to initialize audio sink:", error); return null; } } }