mirror of
https://github.com/OpenCut-app/OpenCut.git
synced 2026-07-13 21:52:53 +02:00
shipping this slop
This commit is contained in:
Maze Winther
@@ -0,0 +1,224 @@
|
||||
import { FFmpeg } from "@ffmpeg/ffmpeg";
|
||||
import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
|
||||
import { collectAudioMixSources } from "@/lib/audio-utils";
|
||||
import { useEditor } from "@/hooks/use-editor";
|
||||
|
||||
let ffmpeg: FFmpeg | null = null;
|
||||
|
||||
export const initFFmpeg = async (): Promise<FFmpeg> => {
|
||||
if (ffmpeg) return ffmpeg;
|
||||
|
||||
ffmpeg = new FFmpeg();
|
||||
await ffmpeg.load(); // Use default config
|
||||
|
||||
return ffmpeg;
|
||||
};
|
||||
|
||||
export async function getVideoInfo({
|
||||
videoFile,
|
||||
}: {
|
||||
videoFile: File;
|
||||
}): Promise<{
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
}> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
// Get frame rate from packet statistics
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const fps = packetStats.averagePacketRate;
|
||||
|
||||
return {
|
||||
duration,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps,
|
||||
};
|
||||
}
|
||||
|
||||
// audio mixing for timeline - keeping ffmpeg for now due to complexity
|
||||
// TODO: Replace with Mediabunny audio processing when implementing canvas preview
|
||||
export const extractTimelineAudio = async (
|
||||
onProgress?: (progress: number) => void,
|
||||
): Promise<Blob> => {
|
||||
// Create fresh FFmpeg instance for this operation
|
||||
const ffmpeg = new FFmpeg();
|
||||
const editor = useEditor();
|
||||
|
||||
try {
|
||||
await ffmpeg.load();
|
||||
} catch (error) {
|
||||
console.error("Failed to load fresh FFmpeg instance:", error);
|
||||
throw new Error("Unable to initialize audio processing. Please try again.");
|
||||
}
|
||||
|
||||
const tracks = editor.timeline.getTracks();
|
||||
const mediaAssets = editor.media.getAssets();
|
||||
const totalDuration = editor.timeline.getTotalDuration();
|
||||
|
||||
if (totalDuration === 0) {
|
||||
const emptyAudioData = new ArrayBuffer(44);
|
||||
return new Blob([emptyAudioData], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
ffmpeg.on("progress", ({ progress }) => {
|
||||
onProgress(progress * 100);
|
||||
});
|
||||
}
|
||||
|
||||
const audioMixSources = await collectAudioMixSources({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
});
|
||||
|
||||
if (audioMixSources.length === 0) {
|
||||
// Return silent audio if no audio elements
|
||||
const silentDuration = Math.max(1, totalDuration); // At least 1 second
|
||||
try {
|
||||
const silentAudio = await generateSilentAudio(silentDuration);
|
||||
return silentAudio;
|
||||
} catch (error) {
|
||||
console.error("Failed to generate silent audio:", error);
|
||||
throw new Error("Unable to generate audio for empty timeline.");
|
||||
}
|
||||
}
|
||||
|
||||
// Create a complex filter to mix all audio sources
|
||||
const inputFiles: string[] = [];
|
||||
const filterInputs: string[] = [];
|
||||
|
||||
try {
|
||||
for (let i = 0; i < audioMixSources.length; i++) {
|
||||
const element = audioMixSources[i];
|
||||
const inputName = `input_${i}.${element.file.name.split(".").pop()}`;
|
||||
inputFiles.push(inputName);
|
||||
|
||||
try {
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await element.file.arrayBuffer()),
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`Failed to write file ${element.file.name}:`, error);
|
||||
throw new Error(
|
||||
`Unable to process file: ${element.file.name}. The file may be corrupted or in an unsupported format.`,
|
||||
);
|
||||
}
|
||||
|
||||
const actualStart = element.trimStart;
|
||||
const actualDuration = element.duration;
|
||||
|
||||
const filterName = `audio_${i}`;
|
||||
filterInputs.push(
|
||||
`[${i}:a]atrim=start=${actualStart}:duration=${actualDuration},asetpts=PTS-STARTPTS,adelay=${element.startTime * 1000}|${element.startTime * 1000}[${filterName}]`,
|
||||
);
|
||||
}
|
||||
|
||||
const mixFilter =
|
||||
audioMixSources.length === 1
|
||||
? `[audio_0]aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`
|
||||
: `${filterInputs.map((_, i) => `[audio_${i}]`).join("")}amix=inputs=${audioMixSources.length}:duration=longest:dropout_transition=2,aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`;
|
||||
|
||||
const complexFilter = [...filterInputs, mixFilter].join(";");
|
||||
const outputName = "timeline_audio.wav";
|
||||
|
||||
const ffmpegArgs = [
|
||||
...inputFiles.flatMap((name) => ["-i", name]),
|
||||
"-filter_complex",
|
||||
complexFilter,
|
||||
"-map",
|
||||
"[out]",
|
||||
"-t",
|
||||
totalDuration.toString(),
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-ar",
|
||||
"44100",
|
||||
outputName,
|
||||
];
|
||||
|
||||
try {
|
||||
await ffmpeg.exec(ffmpegArgs);
|
||||
} catch (error) {
|
||||
console.error("FFmpeg execution failed:", error);
|
||||
throw new Error(
|
||||
"Audio processing failed. Some audio files may be corrupted or incompatible.",
|
||||
);
|
||||
}
|
||||
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: "audio/wav" });
|
||||
|
||||
return blob;
|
||||
} catch (error) {
|
||||
for (const inputFile of inputFiles) {
|
||||
try {
|
||||
await ffmpeg.deleteFile(inputFile);
|
||||
} catch (cleanupError) {
|
||||
console.warn(`Failed to cleanup file ${inputFile}:`, cleanupError);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await ffmpeg.deleteFile("timeline_audio.wav");
|
||||
} catch (cleanupError) {
|
||||
console.warn("Failed to cleanup output file:", cleanupError);
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
for (const inputFile of inputFiles) {
|
||||
try {
|
||||
await ffmpeg.deleteFile(inputFile);
|
||||
} catch (cleanupError) {}
|
||||
}
|
||||
try {
|
||||
await ffmpeg.deleteFile("timeline_audio.wav");
|
||||
} catch (cleanupError) {}
|
||||
}
|
||||
};
|
||||
|
||||
const generateSilentAudio = async (durationSeconds: number): Promise<Blob> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
const outputName = "silent.wav";
|
||||
|
||||
try {
|
||||
await ffmpeg.exec([
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
`anullsrc=channel_layout=stereo:sample_rate=44100`,
|
||||
"-t",
|
||||
durationSeconds.toString(),
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
outputName,
|
||||
]);
|
||||
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: "audio/wav" });
|
||||
|
||||
return blob;
|
||||
} catch (error) {
|
||||
console.error("Failed to generate silent audio:", error);
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await ffmpeg.deleteFile(outputName);
|
||||
} catch (cleanupError) {
|
||||
// Silent cleanup
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,203 @@
|
||||
import { toast } from "sonner";
|
||||
import { MediaAsset } from "@/types/assets";
|
||||
import { getMediaTypeFromFile } from "@/lib/media-utils";
|
||||
import { getVideoInfo } from "./mediabunny";
|
||||
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
|
||||
|
||||
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> { }
|
||||
|
||||
export async function generateThumbnail({
|
||||
videoFile,
|
||||
timeInSeconds,
|
||||
}: {
|
||||
videoFile: File;
|
||||
timeInSeconds: number;
|
||||
}): Promise<string> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
|
||||
const frame = await sink.getSample(timeInSeconds);
|
||||
|
||||
if (!frame) {
|
||||
throw new Error("Could not get frame at specified time");
|
||||
}
|
||||
|
||||
const maxWidth = 1280;
|
||||
const maxHeight = 720;
|
||||
|
||||
const videoWidth = videoTrack.displayWidth;
|
||||
const videoHeight = videoTrack.displayHeight;
|
||||
const aspectRatio = videoWidth / videoHeight;
|
||||
|
||||
let width = videoWidth;
|
||||
let height = videoHeight;
|
||||
|
||||
if (width > maxWidth) {
|
||||
width = maxWidth;
|
||||
height = Math.round(width / aspectRatio);
|
||||
}
|
||||
if (height > maxHeight) {
|
||||
height = maxHeight;
|
||||
width = Math.round(height * aspectRatio);
|
||||
}
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const ctx = canvas.getContext("2d");
|
||||
|
||||
if (!ctx) {
|
||||
throw new Error("Could not get canvas context");
|
||||
}
|
||||
|
||||
frame.draw(ctx, 0, 0, width, height);
|
||||
const dataUrl = canvas.toDataURL("image/jpeg", 0.8);
|
||||
return dataUrl;
|
||||
}
|
||||
|
||||
export async function processMediaAssets({
|
||||
files,
|
||||
onProgress,
|
||||
}: {
|
||||
files: FileList | File[];
|
||||
onProgress?: ({ progress }: { progress: number }) => void;
|
||||
}): Promise<ProcessedMediaAsset[]> {
|
||||
const fileArray = Array.from(files);
|
||||
const processedAssets: ProcessedMediaAsset[] = [];
|
||||
|
||||
const total = fileArray.length;
|
||||
let completed = 0;
|
||||
|
||||
for (const file of fileArray) {
|
||||
const fileType = getMediaTypeFromFile({ file });
|
||||
|
||||
if (!fileType) {
|
||||
toast.error(`Unsupported file type: ${file.name}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = URL.createObjectURL(file);
|
||||
let thumbnailUrl: string | undefined;
|
||||
let duration: number | undefined;
|
||||
let width: number | undefined;
|
||||
let height: number | undefined;
|
||||
let fps: number | undefined;
|
||||
|
||||
try {
|
||||
if (fileType === "image") {
|
||||
const dimensions = await getImageDimensions({ file });
|
||||
width = dimensions.width;
|
||||
height = dimensions.height;
|
||||
} else if (fileType === "video") {
|
||||
try {
|
||||
const videoInfo = await getVideoInfo({ videoFile: file });
|
||||
duration = videoInfo.duration;
|
||||
width = videoInfo.width;
|
||||
height = videoInfo.height;
|
||||
fps = videoInfo.fps;
|
||||
|
||||
thumbnailUrl = await generateThumbnail({
|
||||
videoFile: file,
|
||||
timeInSeconds: 1,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("Video processing failed", error);
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
// For audio, we don't set width/height/fps (they'll be undefined)
|
||||
duration = await getMediaDuration({ file });
|
||||
}
|
||||
|
||||
processedAssets.push({
|
||||
name: file.name,
|
||||
type: fileType,
|
||||
file,
|
||||
url,
|
||||
thumbnailUrl,
|
||||
duration,
|
||||
width,
|
||||
height,
|
||||
fps,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
completed += 1;
|
||||
if (onProgress) {
|
||||
const percent = Math.round((completed / total) * 100);
|
||||
onProgress({ progress: percent });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error processing file:", file.name, error);
|
||||
toast.error(`Failed to process ${file.name}`);
|
||||
URL.revokeObjectURL(url); // Clean up on error
|
||||
}
|
||||
}
|
||||
|
||||
return processedAssets;
|
||||
}
|
||||
|
||||
const getImageDimensions = ({
|
||||
file,
|
||||
}: {
|
||||
file: File;
|
||||
}): Promise<{ width: number; height: number }> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new window.Image();
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
|
||||
img.addEventListener("load", () => {
|
||||
const width = img.naturalWidth;
|
||||
const height = img.naturalHeight;
|
||||
resolve({ width, height });
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
img.remove();
|
||||
});
|
||||
|
||||
img.addEventListener("error", () => {
|
||||
reject(new Error("Could not load image"));
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
img.remove();
|
||||
});
|
||||
|
||||
img.src = objectUrl;
|
||||
});
|
||||
};
|
||||
|
||||
const getMediaDuration = ({ file }: { file: File }): Promise<number> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const element = document.createElement(
|
||||
file.type.startsWith("video/") ? "video" : "audio",
|
||||
) as HTMLVideoElement;
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
|
||||
element.addEventListener("loadedmetadata", () => {
|
||||
resolve(element.duration);
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
element.remove();
|
||||
});
|
||||
|
||||
element.addEventListener("error", () => {
|
||||
reject(new Error("Could not load media"));
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
element.remove();
|
||||
});
|
||||
|
||||
element.src = objectUrl;
|
||||
element.load();
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user