mirror of
https://github.com/OpenCut-app/OpenCut.git
synced 2026-07-13 21:52:53 +02:00
i think we just fixed audio in the playback forever
This commit is contained in:
Maze Winther
@@ -0,0 +1,335 @@
|
||||
import type { EditorCore } from "@/core";
|
||||
import type { AudioClipSource } from "@/lib/media/audio";
|
||||
import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
|
||||
import {
|
||||
ALL_FORMATS,
|
||||
AudioBufferSink,
|
||||
BlobSource,
|
||||
Input,
|
||||
type WrappedAudioBuffer,
|
||||
} from "mediabunny";
|
||||
|
||||
export class AudioManager {
|
||||
private audioContext: AudioContext | null = null;
|
||||
private masterGain: GainNode | null = null;
|
||||
private playbackStartTime = 0;
|
||||
private playbackStartContextTime = 0;
|
||||
private scheduleTimer: number | null = null;
|
||||
private lookaheadSeconds = 2;
|
||||
private scheduleIntervalMs = 500;
|
||||
private clips: AudioClipSource[] = [];
|
||||
private sinks = new Map<string, AudioBufferSink>();
|
||||
private inputs = new Map<string, Input>();
|
||||
private activeClipIds = new Set<string>();
|
||||
private clipIterators = new Map<
|
||||
string,
|
||||
AsyncGenerator<WrappedAudioBuffer, void, unknown>
|
||||
>();
|
||||
private queuedSources = new Set<AudioBufferSourceNode>();
|
||||
private playbackSessionId = 0;
|
||||
private lastIsPlaying = false;
|
||||
private lastVolume = 1;
|
||||
private unsubscribers: Array<() => void> = [];
|
||||
|
||||
constructor(private editor: EditorCore) {
|
||||
this.lastVolume = this.editor.playback.getVolume();
|
||||
|
||||
this.unsubscribers.push(
|
||||
this.editor.playback.subscribe(this.handlePlaybackChange),
|
||||
this.editor.timeline.subscribe(this.handleTimelineChange),
|
||||
this.editor.media.subscribe(this.handleTimelineChange),
|
||||
);
|
||||
if (typeof window !== "undefined") {
|
||||
window.addEventListener("playback-seek", this.handleSeek);
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.stopPlayback();
|
||||
for (const unsub of this.unsubscribers) {
|
||||
unsub();
|
||||
}
|
||||
this.unsubscribers = [];
|
||||
if (typeof window !== "undefined") {
|
||||
window.removeEventListener("playback-seek", this.handleSeek);
|
||||
}
|
||||
this.disposeSinks();
|
||||
if (this.audioContext) {
|
||||
void this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
this.masterGain = null;
|
||||
}
|
||||
}
|
||||
|
||||
private handlePlaybackChange = (): void => {
|
||||
const isPlaying = this.editor.playback.getIsPlaying();
|
||||
const volume = this.editor.playback.getVolume();
|
||||
|
||||
if (volume !== this.lastVolume) {
|
||||
this.lastVolume = volume;
|
||||
this.updateGain();
|
||||
}
|
||||
|
||||
if (isPlaying !== this.lastIsPlaying) {
|
||||
this.lastIsPlaying = isPlaying;
|
||||
if (isPlaying) {
|
||||
void this.startPlayback({
|
||||
time: this.editor.playback.getCurrentTime(),
|
||||
});
|
||||
} else {
|
||||
this.stopPlayback();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private handleSeek = (event: Event): void => {
|
||||
const detail = (event as CustomEvent<{ time: number }>).detail;
|
||||
if (!detail) return;
|
||||
|
||||
if (this.editor.playback.getIsPlaying()) {
|
||||
void this.startPlayback({ time: detail.time });
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopPlayback();
|
||||
};
|
||||
|
||||
private handleTimelineChange = (): void => {
|
||||
this.disposeSinks();
|
||||
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
|
||||
};
|
||||
|
||||
private ensureAudioContext(): AudioContext | null {
|
||||
if (this.audioContext) return this.audioContext;
|
||||
if (typeof window === "undefined") return null;
|
||||
|
||||
this.audioContext = createAudioContext();
|
||||
this.masterGain = this.audioContext.createGain();
|
||||
this.masterGain.gain.value = this.lastVolume;
|
||||
this.masterGain.connect(this.audioContext.destination);
|
||||
return this.audioContext;
|
||||
}
|
||||
|
||||
private updateGain(): void {
|
||||
if (!this.masterGain) return;
|
||||
this.masterGain.gain.value = this.lastVolume;
|
||||
}
|
||||
|
||||
private getPlaybackTime(): number {
|
||||
if (!this.audioContext) return this.playbackStartTime;
|
||||
const elapsed = this.audioContext.currentTime - this.playbackStartContextTime;
|
||||
return this.playbackStartTime + elapsed;
|
||||
}
|
||||
|
||||
private async startPlayback({ time }: { time: number }): Promise<void> {
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) return;
|
||||
|
||||
this.stopPlayback();
|
||||
this.playbackSessionId++;
|
||||
|
||||
const tracks = this.editor.timeline.getTracks();
|
||||
const mediaAssets = this.editor.media.getAssets();
|
||||
const duration = this.editor.timeline.getTotalDuration();
|
||||
|
||||
if (duration <= 0) return;
|
||||
|
||||
if (audioContext.state === "suspended") {
|
||||
await audioContext.resume();
|
||||
}
|
||||
|
||||
this.clips = await collectAudioClips({ tracks, mediaAssets });
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
this.playbackStartTime = time;
|
||||
this.playbackStartContextTime = audioContext.currentTime;
|
||||
|
||||
this.scheduleUpcomingClips();
|
||||
|
||||
if (typeof window !== "undefined") {
|
||||
this.scheduleTimer = window.setInterval(() => {
|
||||
this.scheduleUpcomingClips();
|
||||
}, this.scheduleIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleUpcomingClips(): void {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
|
||||
const currentTime = this.getPlaybackTime();
|
||||
const windowEnd = currentTime + this.lookaheadSeconds;
|
||||
|
||||
for (const clip of this.clips) {
|
||||
if (clip.muted) continue;
|
||||
if (this.activeClipIds.has(clip.id)) continue;
|
||||
|
||||
const clipEnd = clip.startTime + clip.duration;
|
||||
if (clipEnd <= currentTime) continue;
|
||||
if (clip.startTime > windowEnd) continue;
|
||||
|
||||
this.activeClipIds.add(clip.id);
|
||||
void this.runClipIterator({ clip, startTime: currentTime, sessionId: this.playbackSessionId });
|
||||
}
|
||||
}
|
||||
|
||||
private stopPlayback(): void {
|
||||
if (this.scheduleTimer && typeof window !== "undefined") {
|
||||
window.clearInterval(this.scheduleTimer);
|
||||
}
|
||||
this.scheduleTimer = null;
|
||||
|
||||
for (const iterator of this.clipIterators.values()) {
|
||||
void iterator.return();
|
||||
}
|
||||
this.clipIterators.clear();
|
||||
this.activeClipIds.clear();
|
||||
|
||||
for (const source of this.queuedSources) {
|
||||
try {
|
||||
source.stop();
|
||||
} catch {}
|
||||
source.disconnect();
|
||||
}
|
||||
this.queuedSources.clear();
|
||||
}
|
||||
|
||||
private async runClipIterator({
|
||||
clip,
|
||||
startTime,
|
||||
sessionId,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
startTime: number;
|
||||
sessionId: number;
|
||||
}): Promise<void> {
|
||||
const audioContext = this.ensureAudioContext();
|
||||
if (!audioContext) return;
|
||||
|
||||
const sink = await this.getAudioSink({ clip });
|
||||
if (!sink || !this.editor.playback.getIsPlaying()) return;
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
|
||||
const clipStart = clip.startTime;
|
||||
const clipEnd = clip.startTime + clip.duration;
|
||||
|
||||
const iteratorStartTime = Math.max(startTime, clipStart);
|
||||
const sourceStartTime =
|
||||
clip.trimStart + (iteratorStartTime - clip.startTime);
|
||||
|
||||
const iterator = sink.buffers(sourceStartTime);
|
||||
this.clipIterators.set(clip.id, iterator);
|
||||
|
||||
for await (const { buffer, timestamp } of iterator) {
|
||||
if (!this.editor.playback.getIsPlaying()) return;
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
|
||||
const timelineTime = clip.startTime + (timestamp - clip.trimStart);
|
||||
if (timelineTime >= clipEnd) break;
|
||||
|
||||
const node = audioContext.createBufferSource();
|
||||
node.buffer = buffer;
|
||||
node.connect(this.masterGain ?? audioContext.destination);
|
||||
|
||||
const startTimestamp =
|
||||
this.playbackStartContextTime +
|
||||
(timelineTime - this.playbackStartTime);
|
||||
|
||||
if (startTimestamp >= audioContext.currentTime) {
|
||||
node.start(startTimestamp);
|
||||
} else {
|
||||
const offset = audioContext.currentTime - startTimestamp;
|
||||
if (offset < buffer.duration) {
|
||||
node.start(audioContext.currentTime, offset);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
this.queuedSources.add(node);
|
||||
node.addEventListener("ended", () => {
|
||||
node.disconnect();
|
||||
this.queuedSources.delete(node);
|
||||
});
|
||||
|
||||
const aheadTime = timelineTime - this.getPlaybackTime();
|
||||
if (aheadTime >= 1) {
|
||||
await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
|
||||
if (sessionId !== this.playbackSessionId) return;
|
||||
}
|
||||
}
|
||||
|
||||
this.clipIterators.delete(clip.id);
|
||||
// don't remove from activeClipIds - prevents scheduler from restarting this clip
|
||||
// the set is cleared on stopPlayback anyway
|
||||
}
|
||||
|
||||
private waitUntilCaughtUp({
|
||||
timelineTime,
|
||||
targetAhead,
|
||||
}: {
|
||||
timelineTime: number;
|
||||
targetAhead: number;
|
||||
}): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const checkInterval = setInterval(() => {
|
||||
if (!this.editor.playback.getIsPlaying()) {
|
||||
clearInterval(checkInterval);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const playbackTime = this.getPlaybackTime();
|
||||
if (timelineTime - playbackTime < targetAhead) {
|
||||
clearInterval(checkInterval);
|
||||
resolve();
|
||||
}
|
||||
}, 100);
|
||||
});
|
||||
}
|
||||
|
||||
private disposeSinks(): void {
|
||||
for (const iterator of this.clipIterators.values()) {
|
||||
void iterator.return();
|
||||
}
|
||||
this.clipIterators.clear();
|
||||
this.activeClipIds.clear();
|
||||
|
||||
for (const input of this.inputs.values()) {
|
||||
input.dispose();
|
||||
}
|
||||
this.inputs.clear();
|
||||
this.sinks.clear();
|
||||
}
|
||||
|
||||
private async getAudioSink({
|
||||
clip,
|
||||
}: {
|
||||
clip: AudioClipSource;
|
||||
}): Promise<AudioBufferSink | null> {
|
||||
const existingSink = this.sinks.get(clip.sourceKey);
|
||||
if (existingSink) return existingSink;
|
||||
|
||||
try {
|
||||
const input = new Input({
|
||||
source: new BlobSource(clip.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) {
|
||||
input.dispose();
|
||||
return null;
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
this.inputs.set(clip.sourceKey, input);
|
||||
this.sinks.set(clip.sourceKey, sink);
|
||||
return sink;
|
||||
} catch (error) {
|
||||
console.warn("Failed to initialize audio sink:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user