i think we just fixed audio in the playback forever

apps/web/src/core/index.ts

@@ -6,6 +6,7 @@ import { MediaManager } from "./managers/media-manager";
 import { RendererManager } from "./managers/renderer-manager";
 import { CommandManager } from "./managers/commands";
 import { SaveManager } from "./managers/save-manager";
+import { AudioManager } from "./managers/audio-manager";
 
 export class EditorCore {
 	private static instance: EditorCore | null = null;
@@ -18,6 +19,7 @@ export class EditorCore {
 	public readonly media: MediaManager;
 	public readonly renderer: RendererManager;
 	public readonly save: SaveManager;
+	public readonly audio: AudioManager;
 
 	private constructor() {
 		this.command = new CommandManager();
@@ -28,6 +30,7 @@ export class EditorCore {
 		this.media = new MediaManager(this);
 		this.renderer = new RendererManager(this);
 		this.save = new SaveManager(this);
+		this.audio = new AudioManager(this);
 		this.save.start();
 	}
 

apps/web/src/core/managers/audio-manager.ts

@@ -0,0 +1,335 @@
+import type { EditorCore } from "@/core";
+import type { AudioClipSource } from "@/lib/media/audio";
+import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
+import {
+	ALL_FORMATS,
+	AudioBufferSink,
+	BlobSource,
+	Input,
+	type WrappedAudioBuffer,
+} from "mediabunny";
+
+export class AudioManager {
+	private audioContext: AudioContext | null = null;
+	private masterGain: GainNode | null = null;
+	private playbackStartTime = 0;
+	private playbackStartContextTime = 0;
+	private scheduleTimer: number | null = null;
+	private lookaheadSeconds = 2;
+	private scheduleIntervalMs = 500;
+	private clips: AudioClipSource[] = [];
+	private sinks = new Map<string, AudioBufferSink>();
+	private inputs = new Map<string, Input>();
+	private activeClipIds = new Set<string>();
+	private clipIterators = new Map<
+		string,
+		AsyncGenerator<WrappedAudioBuffer, void, unknown>
+	>();
+	private queuedSources = new Set<AudioBufferSourceNode>();
+	private playbackSessionId = 0;
+	private lastIsPlaying = false;
+	private lastVolume = 1;
+	private unsubscribers: Array<() => void> = [];
+
+	constructor(private editor: EditorCore) {
+		this.lastVolume = this.editor.playback.getVolume();
+
+		this.unsubscribers.push(
+			this.editor.playback.subscribe(this.handlePlaybackChange),
+			this.editor.timeline.subscribe(this.handleTimelineChange),
+			this.editor.media.subscribe(this.handleTimelineChange),
+		);
+		if (typeof window !== "undefined") {
+			window.addEventListener("playback-seek", this.handleSeek);
+		}
+	}
+
+	dispose(): void {
+		this.stopPlayback();
+		for (const unsub of this.unsubscribers) {
+			unsub();
+		}
+		this.unsubscribers = [];
+		if (typeof window !== "undefined") {
+			window.removeEventListener("playback-seek", this.handleSeek);
+		}
+		this.disposeSinks();
+		if (this.audioContext) {
+			void this.audioContext.close();
+			this.audioContext = null;
+			this.masterGain = null;
+		}
+	}
+
+	private handlePlaybackChange = (): void => {
+		const isPlaying = this.editor.playback.getIsPlaying();
+		const volume = this.editor.playback.getVolume();
+
+		if (volume !== this.lastVolume) {
+			this.lastVolume = volume;
+			this.updateGain();
+		}
+
+		if (isPlaying !== this.lastIsPlaying) {
+			this.lastIsPlaying = isPlaying;
+			if (isPlaying) {
+				void this.startPlayback({
+					time: this.editor.playback.getCurrentTime(),
+				});
+			} else {
+				this.stopPlayback();
+			}
+		}
+	};
+
+	private handleSeek = (event: Event): void => {
+		const detail = (event as CustomEvent<{ time: number }>).detail;
+		if (!detail) return;
+
+		if (this.editor.playback.getIsPlaying()) {
+			void this.startPlayback({ time: detail.time });
+			return;
+		}
+
+		this.stopPlayback();
+	};
+
+	private handleTimelineChange = (): void => {
+		this.disposeSinks();
+
+		if (!this.editor.playback.getIsPlaying()) return;
+
+		void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
+	};
+
+	private ensureAudioContext(): AudioContext | null {
+		if (this.audioContext) return this.audioContext;
+		if (typeof window === "undefined") return null;
+
+		this.audioContext = createAudioContext();
+		this.masterGain = this.audioContext.createGain();
+		this.masterGain.gain.value = this.lastVolume;
+		this.masterGain.connect(this.audioContext.destination);
+		return this.audioContext;
+	}
+
+	private updateGain(): void {
+		if (!this.masterGain) return;
+		this.masterGain.gain.value = this.lastVolume;
+	}
+
+	private getPlaybackTime(): number {
+		if (!this.audioContext) return this.playbackStartTime;
+		const elapsed = this.audioContext.currentTime - this.playbackStartContextTime;
+		return this.playbackStartTime + elapsed;
+	}
+
+	private async startPlayback({ time }: { time: number }): Promise<void> {
+		const audioContext = this.ensureAudioContext();
+		if (!audioContext) return;
+
+		this.stopPlayback();
+		this.playbackSessionId++;
+
+		const tracks = this.editor.timeline.getTracks();
+		const mediaAssets = this.editor.media.getAssets();
+		const duration = this.editor.timeline.getTotalDuration();
+
+		if (duration <= 0) return;
+
+		if (audioContext.state === "suspended") {
+			await audioContext.resume();
+		}
+
+		this.clips = await collectAudioClips({ tracks, mediaAssets });
+		if (!this.editor.playback.getIsPlaying()) return;
+
+		this.playbackStartTime = time;
+		this.playbackStartContextTime = audioContext.currentTime;
+
+		this.scheduleUpcomingClips();
+
+		if (typeof window !== "undefined") {
+			this.scheduleTimer = window.setInterval(() => {
+				this.scheduleUpcomingClips();
+			}, this.scheduleIntervalMs);
+		}
+	}
+
+	private scheduleUpcomingClips(): void {
+		if (!this.editor.playback.getIsPlaying()) return;
+
+		const currentTime = this.getPlaybackTime();
+		const windowEnd = currentTime + this.lookaheadSeconds;
+
+		for (const clip of this.clips) {
+			if (clip.muted) continue;
+			if (this.activeClipIds.has(clip.id)) continue;
+
+			const clipEnd = clip.startTime + clip.duration;
+			if (clipEnd <= currentTime) continue;
+			if (clip.startTime > windowEnd) continue;
+
+			this.activeClipIds.add(clip.id);
+			void this.runClipIterator({ clip, startTime: currentTime, sessionId: this.playbackSessionId });
+		}
+	}
+
+	private stopPlayback(): void {
+		if (this.scheduleTimer && typeof window !== "undefined") {
+			window.clearInterval(this.scheduleTimer);
+		}
+		this.scheduleTimer = null;
+
+		for (const iterator of this.clipIterators.values()) {
+			void iterator.return();
+		}
+		this.clipIterators.clear();
+		this.activeClipIds.clear();
+
+		for (const source of this.queuedSources) {
+			try {
+				source.stop();
+			} catch {}
+			source.disconnect();
+		}
+		this.queuedSources.clear();
+	}
+
+	private async runClipIterator({
+		clip,
+		startTime,
+		sessionId,
+	}: {
+		clip: AudioClipSource;
+		startTime: number;
+		sessionId: number;
+	}): Promise<void> {
+		const audioContext = this.ensureAudioContext();
+		if (!audioContext) return;
+
+		const sink = await this.getAudioSink({ clip });
+		if (!sink || !this.editor.playback.getIsPlaying()) return;
+		if (sessionId !== this.playbackSessionId) return;
+
+		const clipStart = clip.startTime;
+		const clipEnd = clip.startTime + clip.duration;
+
+		const iteratorStartTime = Math.max(startTime, clipStart);
+		const sourceStartTime =
+			clip.trimStart + (iteratorStartTime - clip.startTime);
+
+		const iterator = sink.buffers(sourceStartTime);
+		this.clipIterators.set(clip.id, iterator);
+
+		for await (const { buffer, timestamp } of iterator) {
+			if (!this.editor.playback.getIsPlaying()) return;
+			if (sessionId !== this.playbackSessionId) return;
+
+			const timelineTime = clip.startTime + (timestamp - clip.trimStart);
+			if (timelineTime >= clipEnd) break;
+
+			const node = audioContext.createBufferSource();
+			node.buffer = buffer;
+			node.connect(this.masterGain ?? audioContext.destination);
+
+			const startTimestamp =
+				this.playbackStartContextTime +
+				(timelineTime - this.playbackStartTime);
+
+			if (startTimestamp >= audioContext.currentTime) {
+				node.start(startTimestamp);
+			} else {
+				const offset = audioContext.currentTime - startTimestamp;
+				if (offset < buffer.duration) {
+					node.start(audioContext.currentTime, offset);
+				} else {
+					continue;
+				}
+			}
+
+			this.queuedSources.add(node);
+			node.addEventListener("ended", () => {
+				node.disconnect();
+				this.queuedSources.delete(node);
+			});
+
+			const aheadTime = timelineTime - this.getPlaybackTime();
+			if (aheadTime >= 1) {
+				await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
+				if (sessionId !== this.playbackSessionId) return;
+			}
+		}
+
+		this.clipIterators.delete(clip.id);
+		// don't remove from activeClipIds - prevents scheduler from restarting this clip
+		// the set is cleared on stopPlayback anyway
+	}
+
+	private waitUntilCaughtUp({
+		timelineTime,
+		targetAhead,
+	}: {
+		timelineTime: number;
+		targetAhead: number;
+	}): Promise<void> {
+		return new Promise((resolve) => {
+			const checkInterval = setInterval(() => {
+				if (!this.editor.playback.getIsPlaying()) {
+					clearInterval(checkInterval);
+					resolve();
+					return;
+				}
+
+				const playbackTime = this.getPlaybackTime();
+				if (timelineTime - playbackTime < targetAhead) {
+					clearInterval(checkInterval);
+					resolve();
+				}
+			}, 100);
+		});
+	}
+
+	private disposeSinks(): void {
+		for (const iterator of this.clipIterators.values()) {
+			void iterator.return();
+		}
+		this.clipIterators.clear();
+		this.activeClipIds.clear();
+
+		for (const input of this.inputs.values()) {
+			input.dispose();
+		}
+		this.inputs.clear();
+		this.sinks.clear();
+	}
+
+	private async getAudioSink({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): Promise<AudioBufferSink | null> {
+		const existingSink = this.sinks.get(clip.sourceKey);
+		if (existingSink) return existingSink;
+
+		try {
+			const input = new Input({
+				source: new BlobSource(clip.file),
+				formats: ALL_FORMATS,
+			});
+			const audioTrack = await input.getPrimaryAudioTrack();
+			if (!audioTrack) {
+				input.dispose();
+				return null;
+			}
+
+			const sink = new AudioBufferSink(audioTrack);
+			this.inputs.set(clip.sourceKey, input);
+			this.sinks.set(clip.sourceKey, sink);
+			return sink;
+		} catch (error) {
+			console.warn("Failed to initialize audio sink:", error);
+			return null;
+		}
+	}
+}

apps/web/src/core/managers/playback-manager.ts

@@ -6,7 +6,6 @@ export class PlaybackManager {
 	private volume = 1;
 	private muted = false;
 	private previousVolume = 1;
-	private speed = 1.0;
 	private listeners = new Set<() => void>();
 	private playbackTimer: number | null = null;
 	private lastUpdate = 0;
@@ -68,17 +67,6 @@ export class PlaybackManager {
 		this.notify();
 	}
 
-	setSpeed({ speed }: { speed: number }): void {
-		this.speed = Math.max(0.1, Math.min(2.0, speed));
-		this.notify();
-
-		window.dispatchEvent(
-			new CustomEvent("playback-speed", {
-				detail: { speed: this.speed },
-			}),
-		);
-	}
-
 	mute(): void {
 		if (this.volume > 0) {
 			this.previousVolume = this.volume;
@@ -118,10 +106,6 @@ export class PlaybackManager {
 		return this.muted;
 	}
 
-	getSpeed(): number {
-		return this.speed;
-	}
-
 	subscribe(listener: () => void): () => void {
 		this.listeners.add(listener);
 		return () => this.listeners.delete(listener);
@@ -154,7 +138,7 @@ export class PlaybackManager {
 		const delta = (now - this.lastUpdate) / 1000;
 		this.lastUpdate = now;
 
-		const newTime = this.currentTime + delta * this.speed;
+		const newTime = this.currentTime + delta;
 		const duration = this.editor.timeline.getTotalDuration();
 
 		if (duration > 0 && newTime >= duration) {

apps/web/src/core/managers/renderer-manager.ts

@@ -69,7 +69,7 @@ export class RendererManager {
 				fps: exportFps,
 				format,
 				quality,
-				includeAudio: !!includeAudio,
+				shouldIncludeAudio: !!includeAudio,
 				audioBuffer: audioBuffer || undefined,
 			});
 
@@ -91,7 +91,7 @@ export class RendererManager {
 			const cancelInterval = setInterval(checkCancel, 100);
 
 			try {
-				const buffer = await exporter.export(scene);
+				const buffer = await exporter.export({ rootNode: scene });
 				clearInterval(cancelInterval);
 
 				if (cancelled) {

apps/web/src/lib/media/audio.ts

@@ -144,6 +144,17 @@ interface AudioMixSource {
 	trimEnd: number;
 }
 
+export interface AudioClipSource {
+	id: string;
+	sourceKey: string;
+	file: File;
+	startTime: number;
+	duration: number;
+	trimStart: number;
+	trimEnd: number;
+	muted: boolean;
+}
+
 async function fetchLibraryAudioSource({
 	element,
 }: {
@@ -173,6 +184,40 @@ async function fetchLibraryAudioSource({
 	}
 }
 
+async function fetchLibraryAudioClip({
+	element,
+	muted,
+}: {
+	element: LibraryAudioElement;
+	muted: boolean;
+}): Promise<AudioClipSource | null> {
+	try {
+		const response = await fetch(element.sourceUrl);
+		if (!response.ok) {
+			throw new Error(`Library audio fetch failed: ${response.status}`);
+		}
+
+		const blob = await response.blob();
+		const file = new File([blob], `${element.name}.mp3`, {
+			type: "audio/mpeg",
+		});
+
+		return {
+			id: element.id,
+			sourceKey: element.id,
+			file,
+			startTime: element.startTime,
+			duration: element.duration,
+			trimStart: element.trimStart,
+			trimEnd: element.trimEnd,
+			muted,
+		};
+	} catch (error) {
+		console.warn("Failed to fetch library audio:", error);
+		return null;
+	}
+}
+
 function collectMediaAudioSource({
 	element,
 	mediaAsset,
@@ -189,6 +234,27 @@ function collectMediaAudioSource({
 	};
 }
 
+function collectMediaAudioClip({
+	element,
+	mediaAsset,
+	muted,
+}: {
+	element: TimelineElement;
+	mediaAsset: MediaAsset;
+	muted: boolean;
+}): AudioClipSource {
+	return {
+		id: element.id,
+		sourceKey: mediaAsset.id,
+		file: mediaAsset.file,
+		startTime: element.startTime,
+		duration: element.duration,
+		trimStart: element.trimStart,
+		trimEnd: element.trimEnd,
+		muted,
+	};
+}
+
 export async function collectAudioMixSources({
 	tracks,
 	mediaAssets,
@@ -243,30 +309,98 @@ export async function collectAudioMixSources({
 	return audioMixSources;
 }
 
+export async function collectAudioClips({
+	tracks,
+	mediaAssets,
+}: {
+	tracks: TimelineTrack[];
+	mediaAssets: MediaAsset[];
+}): Promise<AudioClipSource[]> {
+	const clips: AudioClipSource[] = [];
+	const mediaMap = new Map<string, MediaAsset>(
+		mediaAssets.map((asset) => [asset.id, asset]),
+	);
+	const pendingLibraryClips: Array<Promise<AudioClipSource | null>> = [];
+
+	for (const track of tracks) {
+		const isTrackMuted = canTracktHaveAudio(track) && track.muted;
+
+		for (const element of track.elements) {
+			if (!canElementHaveAudio(element)) continue;
+
+			const isElementMuted =
+				"muted" in element ? (element.muted ?? false) : false;
+			const muted = isTrackMuted || isElementMuted;
+
+			if (element.type === "audio") {
+				if (element.sourceType === "upload") {
+					const mediaAsset = mediaMap.get(element.mediaId);
+					if (!mediaAsset) continue;
+
+					clips.push(
+						collectMediaAudioClip({
+							element,
+							mediaAsset,
+							muted,
+						}),
+					);
+				} else {
+					pendingLibraryClips.push(fetchLibraryAudioClip({ element, muted }));
+				}
+				continue;
+			}
+
+			if (element.type === "video") {
+				const mediaAsset = mediaMap.get(element.mediaId);
+				if (!mediaAsset) continue;
+
+				if (mediaSupportsAudio({ media: mediaAsset })) {
+					clips.push(
+						collectMediaAudioClip({
+							element,
+							mediaAsset,
+							muted,
+						}),
+					);
+				}
+			}
+		}
+	}
+
+	const resolvedLibraryClips = await Promise.all(pendingLibraryClips);
+	for (const clip of resolvedLibraryClips) {
+		if (clip) clips.push(clip);
+	}
+
+	return clips;
+}
+
 export async function createTimelineAudioBuffer({
 	tracks,
 	mediaAssets,
 	duration,
 	sampleRate = 44100,
+	audioContext,
 }: {
 	tracks: TimelineTrack[];
 	mediaAssets: MediaAsset[];
 	duration: number;
 	sampleRate?: number;
+	audioContext?: AudioContext;
 }): Promise<AudioBuffer | null> {
-	const audioContext = createAudioContext();
+	const context = audioContext ?? createAudioContext();
 
 	const audioElements = await collectAudioElements({
 		tracks,
 		mediaAssets,
-		audioContext,
+		audioContext: context,
 	});
 
 	if (audioElements.length === 0) return null;
 
 	const outputChannels = 2;
 	const outputLength = Math.ceil(duration * sampleRate);
-	const outputBuffer = audioContext.createBuffer(
+	const outputBuffer = context.createBuffer(
 		outputChannels,
 		outputLength,
 		sampleRate,

apps/web/src/services/renderer/scene-exporter.ts

@@ -24,7 +24,7 @@ type ExportParams = {
 	fps: number;
 	format: ExportFormat;
 	quality: ExportQuality;
-	includeAudio?: boolean;
+	shouldIncludeAudio?: boolean;
 	audioBuffer?: AudioBuffer;
 };
 
@@ -46,30 +46,38 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
 	private renderer: CanvasRenderer;
 	private format: ExportFormat;
 	private quality: ExportQuality;
-	private includeAudio: boolean;
+	private shouldIncludeAudio: boolean;
 	private audioBuffer?: AudioBuffer;
 
-	private cancelled = false;
+	private isCancelled = false;
 
-	constructor(params: ExportParams) {
+	constructor({
+		width,
+		height,
+		fps,
+		format,
+		quality,
+		shouldIncludeAudio,
+		audioBuffer,
+	}: ExportParams) {
 		super();
 		this.renderer = new CanvasRenderer({
-			width: params.width,
-			height: params.height,
-			fps: params.fps,
+			width,
+			height,
+			fps,
 		});
 
-		this.format = params.format;
-		this.quality = params.quality;
-		this.includeAudio = params.includeAudio ?? false;
-		this.audioBuffer = params.audioBuffer;
+		this.format = format;
+		this.quality = quality;
+		this.shouldIncludeAudio = shouldIncludeAudio ?? false;
+		this.audioBuffer = audioBuffer;
 	}
 
-	cancel() {
-		this.cancelled = true;
+	cancel(): void {
+		this.isCancelled = true;
 	}
 
-	async export(rootNode: RootNode) {
+	async export({ rootNode }: { rootNode: RootNode }): Promise<ArrayBuffer | null> {
 		const { fps } = this.renderer;
 		const frameCount = Math.ceil(rootNode.duration * fps);
 
@@ -88,9 +96,8 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
 
 		output.addVideoTrack(videoSource, { frameRate: fps });
 
-		// Add audio track if requested
 		let audioSource: AudioBufferSource | null = null;
-		if (this.includeAudio && this.audioBuffer) {
+		if (this.shouldIncludeAudio && this.audioBuffer) {
 			audioSource = new AudioBufferSource({
 				codec: this.format === "webm" ? "opus" : "aac",
 				bitrate: qualityMap[this.quality],
@@ -100,15 +107,13 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
 
 		await output.start();
 
-		// Add audio data after starting
 		if (audioSource && this.audioBuffer) {
 			await audioSource.add(this.audioBuffer);
 			audioSource.close();
 		}
 
-		// Render video frames
 		for (let i = 0; i < frameCount; i++) {
-			if (this.cancelled) {
+			if (this.isCancelled) {
 				await output.cancel();
 				this.emit("cancelled");
 				return null;
@@ -121,7 +126,7 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
 			this.emit("progress", i / frameCount);
 		}
 
-		if (this.cancelled) {
+		if (this.isCancelled) {
 			await output.cancel();
 			this.emit("cancelled");
 			return null;

apps/web/src/types/timeline.ts

@@ -69,6 +69,8 @@ export interface LibraryAudioElement extends BaseAudioElement {
 	sourceUrl: string;
 }
 
+export type AudioElement = UploadAudioElement | LibraryAudioElement;
+
 interface BaseTimelineElement {
 	id: string;
 	name: string;
@@ -78,8 +80,6 @@ interface BaseTimelineElement {
 	trimEnd: number;
 }
 
-export type AudioElement = UploadAudioElement | LibraryAudioElement;
-
 export interface VideoElement extends BaseTimelineElement {
 	type: "video";
 	mediaId: string;