Merge branch 'feature/separate-audio'

apps/web/src/lib/actions/definitions.ts

@@ -101,6 +101,10 @@ export const ACTIONS = {
 		description: "Toggle ripple editing",
 		category: "editing",
 	},
+	"toggle-source-audio": {
+		description: "Extract or recover source audio",
+		category: "editing",
+	},
 	"select-all": {
 		description: "Select all elements",
 		category: "selection",

apps/web/src/lib/commands/timeline/element/index.ts

@@ -8,6 +8,7 @@ export { SplitElementsCommand } from "./split-elements";
 export { UpdateElementCommand } from "./update-element";
 export { ToggleElementsVisibilityCommand } from "./toggle-elements-visibility";
 export { ToggleElementsMutedCommand } from "./toggle-elements-muted";
+export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation";
 export { MoveElementCommand } from "./move-elements";
 export * from "./keyframes";
 export * from "./effects";

apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts

@@ -0,0 +1,143 @@
+import { EditorCore } from "@/core";
+import { Command } from "@/lib/commands/base-command";
+import {
+	buildSeparatedAudioElement,
+	canExtractSourceAudio,
+	canRecoverSourceAudio,
+} from "@/lib/timeline/audio-separation";
+import {
+	applyPlacement,
+	resolveTrackPlacement,
+} from "@/lib/timeline/placement";
+import { updateElementInTracks } from "@/lib/timeline/track-element-update";
+import type { TimelineTrack, VideoElement } from "@/lib/timeline/types";
+import { generateUUID } from "@/utils/id";
+
+export class ToggleSourceAudioSeparationCommand extends Command {
+	private savedState: TimelineTrack[] | null = null;
+
+	constructor(
+		private readonly params: {
+			trackId: string;
+			elementId: string;
+		},
+	) {
+		super();
+	}
+
+	execute(): void {
+		const editor = EditorCore.getInstance();
+		this.savedState = editor.timeline.getTracks();
+
+		const trackIndex = this.savedState.findIndex(
+			(track) => track.id === this.params.trackId,
+		);
+		if (trackIndex < 0) {
+			return;
+		}
+
+		const sourceTrack = this.savedState[trackIndex];
+		const sourceElement = sourceTrack.elements.find(
+			(element) => element.id === this.params.elementId,
+		);
+		if (!sourceElement) {
+			return;
+		}
+
+		if (canRecoverSourceAudio({ element: sourceElement })) {
+			editor.timeline.updateTracks(
+				updateSourceAudioEnabled({
+					tracks: this.savedState,
+					trackId: this.params.trackId,
+					elementId: this.params.elementId,
+					isSourceAudioEnabled: true,
+				}),
+			);
+			return;
+		}
+
+		const mediaAsset = editor
+			.media
+			.getAssets()
+			.find((asset) =>
+				sourceElement.type === "video" ? asset.id === sourceElement.mediaId : false,
+			);
+		if (!canExtractSourceAudio({ element: sourceElement, mediaAsset })) {
+			return;
+		}
+		if (sourceElement.duration <= 0) {
+			return;
+		}
+
+		const separatedAudioElement = {
+			...buildSeparatedAudioElement({
+				sourceElement,
+			}),
+			id: generateUUID(),
+		};
+		const placementResult = resolveTrackPlacement({
+			tracks: this.savedState,
+			trackType: "audio",
+			timeSpans: [
+				{
+					startTime: separatedAudioElement.startTime,
+					duration: separatedAudioElement.duration,
+				},
+			],
+			strategy: { type: "aboveSource", sourceTrackIndex: trackIndex },
+		});
+		if (!placementResult) {
+			return;
+		}
+
+		const appliedPlacement = applyPlacement({
+			tracks: this.savedState,
+			placementResult,
+			elements: [separatedAudioElement],
+		});
+		if (!appliedPlacement) {
+			return;
+		}
+
+		editor.timeline.updateTracks(
+			updateSourceAudioEnabled({
+				tracks: appliedPlacement.updatedTracks,
+				trackId: this.params.trackId,
+				elementId: this.params.elementId,
+				isSourceAudioEnabled: false,
+			}),
+		);
+	}
+
+	undo(): void {
+		if (!this.savedState) {
+			return;
+		}
+
+		const editor = EditorCore.getInstance();
+		editor.timeline.updateTracks(this.savedState);
+	}
+}
+
+function updateSourceAudioEnabled({
+	tracks,
+	trackId,
+	elementId,
+	isSourceAudioEnabled,
+}: {
+	tracks: TimelineTrack[];
+	trackId: string;
+	elementId: string;
+	isSourceAudioEnabled: boolean;
+}): TimelineTrack[] {
+	return updateElementInTracks({
+		tracks,
+		trackId,
+		elementId,
+		elementPredicate: (element): element is VideoElement => element.type === "video",
+		update: (element) => ({
+			...element,
+			isSourceAudioEnabled,
+		}),
+	});
+}

apps/web/src/lib/media/audio.ts

@@ -12,7 +12,10 @@ import {
 	hasAnimatedVolume,
 	resolveEffectiveAudioGain,
 } from "@/lib/timeline/audio-state";
-import { canElementHaveAudio } from "@/lib/timeline/element-utils";
+import {
+	doesElementHaveEnabledAudio,
+} from "@/lib/timeline/audio-separation";
+import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";
 import { canTracktHaveAudio } from "@/lib/timeline";
 import { mediaSupportsAudio } from "@/lib/media/media-utils";
 import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime";
@@ -100,6 +103,11 @@ export async function collectAudioElements({
 			if (!canElementHaveAudio(element)) continue;
 			if (element.duration <= 0) continue;
 
+			const mediaAsset = hasMediaId(element)
+				? (mediaMap.get(element.mediaId) ?? null)
+				: null;
+			if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
+
 			const isTrackMuted = canTracktHaveAudio(track) && track.muted;
 
 			if (element.type === "audio") {
@@ -132,7 +140,6 @@ export async function collectAudioElements({
 			}
 
 			if (element.type === "video") {
-				const mediaAsset = mediaMap.get(element.mediaId);
 				if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
 
 				pendingElements.push(
@@ -451,6 +458,10 @@ export async function collectAudioMixSources({
 		for (const element of track.elements) {
 			if (!canElementHaveAudio(element)) continue;
 			if (element.muted === true) continue;
+			const mediaAsset = hasMediaId(element)
+				? (mediaMap.get(element.mediaId) ?? null)
+				: null;
+			if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
 			const volume = resolveEffectiveAudioGain({
 				element,
 				localTime: 0,
@@ -473,10 +484,7 @@ export async function collectAudioMixSources({
 			}
 
 			if (element.type === "video") {
-				const mediaAsset = mediaMap.get(element.mediaId);
-				if (!mediaAsset) continue;
-
-				if (mediaSupportsAudio({ media: mediaAsset })) {
+				if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) {
 					audioMixSources.push(
 						collectMediaAudioSource({ element, mediaAsset, volume }),
 					);
@@ -512,6 +520,11 @@ export async function collectAudioClips({
 		for (const element of track.elements) {
 			if (!canElementHaveAudio(element)) continue;
 
+			const mediaAsset = hasMediaId(element)
+				? (mediaMap.get(element.mediaId) ?? null)
+				: null;
+			if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
+
 			const isElementMuted =
 				"muted" in element ? (element.muted ?? false) : false;
 			const muted = isTrackMuted || isElementMuted;
@@ -543,10 +556,7 @@ export async function collectAudioClips({
 			}
 
 			if (element.type === "video") {
-				const mediaAsset = mediaMap.get(element.mediaId);
-				if (!mediaAsset) continue;
-
-				if (mediaSupportsAudio({ media: mediaAsset })) {
+				if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) {
 					clips.push(
 						collectMediaAudioClip({
 							element,

apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts

@@ -0,0 +1,147 @@
+import { describe, expect, test } from "bun:test";
+import type { AudioElement, VideoElement } from "@/lib/timeline";
+import {
+	buildSeparatedAudioElement,
+	doesElementHaveEnabledAudio,
+	isSourceAudioEnabled,
+	isSourceAudioSeparated,
+} from "..";
+
+describe("audio separation", () => {
+	test("treats missing source audio state as enabled", () => {
+		const element = buildVideoElement({});
+
+		expect(isSourceAudioEnabled({ element })).toBe(true);
+		expect(isSourceAudioSeparated({ element })).toBe(false);
+	});
+
+	test("builds a detached audio element from the source clip audio state", () => {
+		const element = buildVideoElement({
+			duration: 8,
+			startTime: 3,
+			trimStart: 1.5,
+			trimEnd: 0.5,
+			sourceDuration: 12,
+			volume: -6,
+			muted: true,
+			retime: { rate: 1.25, maintainPitch: true },
+			animations: {
+				channels: {
+					volume: {
+						valueKind: "number",
+						keyframes: [
+							{
+								id: "volume-keyframe",
+								time: 2,
+								value: -12,
+								interpolation: "linear",
+							},
+						],
+					},
+					opacity: {
+						valueKind: "number",
+						keyframes: [
+							{
+								id: "opacity-keyframe",
+								time: 1,
+								value: 0.5,
+								interpolation: "linear",
+							},
+						],
+					},
+				},
+			},
+		});
+
+		const separatedAudioElement = buildSeparatedAudioElement({
+			sourceElement: element,
+		});
+
+		expect(separatedAudioElement).toMatchObject({
+			type: "audio",
+			sourceType: "upload",
+			mediaId: element.mediaId,
+			name: element.name,
+			duration: 8,
+			startTime: 3,
+			trimStart: 1.5,
+			trimEnd: 0.5,
+			sourceDuration: 12,
+			volume: -6,
+			muted: true,
+			retime: { rate: 1.25, maintainPitch: true },
+		});
+		expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([
+			"volume",
+		]);
+		expect(
+			separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id,
+		).not.toBe("volume-keyframe");
+	});
+
+	test("skips source audio collection when the source clip is separated", () => {
+		const mediaAsset = {
+			id: "media-1",
+			type: "video",
+			name: "Clip",
+			size: 1,
+			lastModified: 1,
+			file: new File(["video"], "clip.mp4", { type: "video/mp4" }),
+			url: "blob:clip",
+			hasAudio: true,
+		};
+		const videoElement = buildVideoElement({
+			isSourceAudioEnabled: false,
+		});
+		const audioElement = {
+			id: "audio-1",
+			type: "audio",
+			sourceType: "upload",
+			mediaId: "audio-media-1",
+			name: "Detached audio",
+			duration: 5,
+			startTime: 0,
+			trimStart: 0,
+			trimEnd: 0,
+			volume: 0,
+		} as AudioElement;
+
+		expect(
+			doesElementHaveEnabledAudio({
+				element: videoElement,
+				mediaAsset,
+			}),
+		).toBe(false);
+		expect(
+			doesElementHaveEnabledAudio({
+				element: audioElement,
+			}),
+		).toBe(true);
+	});
+});
+
+function buildVideoElement(
+	overrides: Partial<VideoElement>,
+): VideoElement {
+	return {
+		id: "video-1",
+		type: "video",
+		mediaId: "media-1",
+		name: "Clip",
+		duration: 5,
+		startTime: 0,
+		trimStart: 0,
+		trimEnd: 0,
+		volume: 0,
+		muted: false,
+		isSourceAudioEnabled: true,
+		transform: {
+			scaleX: 1,
+			scaleY: 1,
+			position: { x: 0, y: 0 },
+			rotate: 0,
+		},
+		opacity: 1,
+		...overrides,
+	};
+}

apps/web/src/lib/timeline/audio-separation/index.ts

@@ -0,0 +1,133 @@
+import { cloneAnimations, getChannel } from "@/lib/animation";
+import type { ElementAnimations } from "@/lib/animation/types";
+import type { MediaAsset } from "@/lib/media/types";
+import { DEFAULTS } from "@/lib/timeline/defaults";
+import type {
+	CreateUploadAudioElement,
+	TimelineElement,
+	AudioElement,
+	VideoElement,
+} from "../types";
+
+export function isSourceAudioEnabled({
+	element,
+}: {
+	element: VideoElement;
+}): boolean {
+	return element.isSourceAudioEnabled !== false;
+}
+
+export function isSourceAudioSeparated({
+	element,
+}: {
+	element: VideoElement;
+}): boolean {
+	return !isSourceAudioEnabled({ element });
+}
+
+export function canExtractSourceAudio({
+	element,
+	mediaAsset,
+}: {
+	element: TimelineElement;
+	mediaAsset: MediaAsset | null | undefined;
+}): element is VideoElement {
+	return (
+		element.type === "video" &&
+		isSourceAudioEnabled({ element }) &&
+		!!mediaAsset &&
+		mediaAsset.hasAudio !== false
+	);
+}
+
+export function canRecoverSourceAudio({
+	element,
+}: {
+	element: TimelineElement;
+}): element is VideoElement {
+	return element.type === "video" && isSourceAudioSeparated({ element });
+}
+
+export function canToggleSourceAudio({
+	element,
+	mediaAsset,
+}: {
+	element: TimelineElement;
+	mediaAsset: MediaAsset | null | undefined;
+}): element is VideoElement {
+	return (
+		canRecoverSourceAudio({ element }) ||
+		canExtractSourceAudio({ element, mediaAsset })
+	);
+}
+
+export function doesElementHaveEnabledAudio({
+	element,
+	mediaAsset,
+}: {
+	element: AudioElement | VideoElement;
+	mediaAsset?: MediaAsset | null;
+}): boolean {
+	if (element.type === "audio") {
+		return true;
+	}
+
+	return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element });
+}
+
+export function buildSeparatedAudioElement({
+	sourceElement,
+}: {
+	sourceElement: VideoElement;
+}): CreateUploadAudioElement {
+	return {
+		type: "audio",
+		sourceType: "upload",
+		mediaId: sourceElement.mediaId,
+		name: sourceElement.name,
+		duration: sourceElement.duration,
+		startTime: sourceElement.startTime,
+		trimStart: sourceElement.trimStart,
+		trimEnd: sourceElement.trimEnd,
+		sourceDuration: sourceElement.sourceDuration,
+		volume: sourceElement.volume ?? DEFAULTS.element.volume,
+		muted: sourceElement.muted ?? false,
+		retime: sourceElement.retime
+			? {
+					rate: sourceElement.retime.rate,
+					maintainPitch: sourceElement.retime.maintainPitch,
+				}
+			: undefined,
+		animations: cloneVolumeAnimations({
+			animations: sourceElement.animations,
+		}),
+	};
+}
+
+export function getSourceAudioActionLabel({
+	element,
+}: {
+	element: VideoElement;
+}): "Extract audio" | "Recover audio" {
+	return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio";
+}
+
+function cloneVolumeAnimations({
+	animations,
+}: {
+	animations: ElementAnimations | undefined;
+}): ElementAnimations | undefined {
+	const volumeChannel = getChannel({ animations, propertyPath: "volume" });
+	if (!volumeChannel) {
+		return undefined;
+	}
+
+	return cloneAnimations({
+		animations: {
+			channels: {
+				volume: volumeChannel,
+			},
+		},
+		shouldRegenerateKeyframeIds: true,
+	});
+}

apps/web/src/lib/timeline/element-utils.ts

@@ -242,6 +242,7 @@ function buildVideoElement({
 		trimEnd: 0,
 		sourceDuration: duration,
 		muted: false,
+		isSourceAudioEnabled: true,
 		hidden: false,
 		transform: {
 			...DEFAULTS.element.transform,

apps/web/src/lib/timeline/index.ts

@@ -5,6 +5,7 @@ export * from "./drag";
 export * from "./track-utils";
 export * from "./track-element-update";
 export * from "./element-utils";
+export * from "./audio-separation";
 export * from "./zoom-utils";
 export * from "./ruler-utils";
 export * from "./ripple-utils";

apps/web/src/lib/timeline/types.ts

@@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement {
 	mediaId: string;
 	volume?: number;
 	muted?: boolean;
+	isSourceAudioEnabled?: boolean;
 	hidden?: boolean;
 	retime?: RetimeConfig;
 	transform: Transform;