mirror of
https://github.com/OpenCut-app/OpenCut.git
synced 2026-07-13 21:52:53 +02:00
feat: mediabunny + preview canvas refactor
This commit is contained in:
Maze Winther
@@ -6,7 +6,7 @@ import {
|
||||
getImageDimensions,
|
||||
type MediaItem,
|
||||
} from "@/stores/media-store";
|
||||
import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";
|
||||
import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";
|
||||
|
||||
export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}
|
||||
|
||||
@@ -37,33 +37,23 @@ export async function processMediaFiles(
|
||||
|
||||
try {
|
||||
if (fileType === "image") {
|
||||
// Get image dimensions
|
||||
const dimensions = await getImageDimensions(file);
|
||||
width = dimensions.width;
|
||||
height = dimensions.height;
|
||||
} else if (fileType === "video") {
|
||||
try {
|
||||
// Use FFmpeg for comprehensive video info extraction
|
||||
const videoInfo = await getVideoInfo(file);
|
||||
const videoInfo = await getVideoInfo({ videoFile: file });
|
||||
duration = videoInfo.duration;
|
||||
width = videoInfo.width;
|
||||
height = videoInfo.height;
|
||||
fps = videoInfo.fps;
|
||||
|
||||
// Generate thumbnail using FFmpeg
|
||||
thumbnailUrl = await generateThumbnail(file, 1);
|
||||
thumbnailUrl = await generateThumbnail({
|
||||
videoFile: file,
|
||||
timeInSeconds: 1,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"FFmpeg processing failed, falling back to basic processing:",
|
||||
error
|
||||
);
|
||||
// Fallback to basic processing
|
||||
const videoResult = await generateVideoThumbnail(file);
|
||||
thumbnailUrl = videoResult.thumbnailUrl;
|
||||
width = videoResult.width;
|
||||
height = videoResult.height;
|
||||
duration = await getMediaDuration(file);
|
||||
// FPS will remain undefined for fallback
|
||||
console.warn("Video processing failed", error);
|
||||
}
|
||||
} else if (fileType === "audio") {
|
||||
// For audio, we don't set width/height/fps (they'll be undefined)
|
||||
@@ -82,7 +72,6 @@ export async function processMediaFiles(
|
||||
fps,
|
||||
});
|
||||
|
||||
// Yield back to the event loop to keep the UI responsive
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
completed += 1;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { FFmpeg } from "@ffmpeg/ffmpeg";
|
||||
import { toBlobURL } from "@ffmpeg/util";
|
||||
import { useTimelineStore } from "@/stores/timeline-store";
|
||||
import { useMediaStore } from "@/stores/media-store";
|
||||
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
|
||||
|
||||
let ffmpeg: FFmpeg | null = null;
|
||||
|
||||
@@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
|
||||
return ffmpeg;
|
||||
};
|
||||
|
||||
export const generateThumbnail = async (
|
||||
videoFile: File,
|
||||
timeInSeconds = 1
|
||||
): Promise<string> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
export async function generateThumbnail({
|
||||
videoFile,
|
||||
timeInSeconds,
|
||||
}: {
|
||||
videoFile: File;
|
||||
timeInSeconds: number;
|
||||
}): Promise<string> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const inputName = "input.mp4";
|
||||
const outputName = "thumbnail.jpg";
|
||||
|
||||
// Write input file
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await videoFile.arrayBuffer())
|
||||
);
|
||||
|
||||
// Generate thumbnail at specific time
|
||||
await ffmpeg.exec([
|
||||
"-i",
|
||||
inputName,
|
||||
"-ss",
|
||||
timeInSeconds.toString(),
|
||||
"-vframes",
|
||||
"1",
|
||||
"-vf",
|
||||
"scale=320:240",
|
||||
"-q:v",
|
||||
"2",
|
||||
outputName,
|
||||
]);
|
||||
|
||||
// Read output file
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: "image/jpeg" });
|
||||
|
||||
// Cleanup
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
await ffmpeg.deleteFile(outputName);
|
||||
|
||||
return URL.createObjectURL(blob);
|
||||
};
|
||||
|
||||
export const trimVideo = async (
|
||||
videoFile: File,
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
onProgress?: (progress: number) => void
|
||||
): Promise<Blob> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
|
||||
const inputName = "input.mp4";
|
||||
const outputName = "output.mp4";
|
||||
|
||||
// Set up progress callback
|
||||
if (onProgress) {
|
||||
ffmpeg.on("progress", ({ progress }) => {
|
||||
onProgress(progress * 100);
|
||||
});
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
// Write input file
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await videoFile.arrayBuffer())
|
||||
);
|
||||
// Check if we can decode this video
|
||||
const canDecode = await videoTrack.canDecode();
|
||||
if (!canDecode) {
|
||||
throw new Error("Video codec not supported for decoding");
|
||||
}
|
||||
|
||||
const duration = endTime - startTime;
|
||||
const sink = new VideoSampleSink(videoTrack);
|
||||
|
||||
// Trim video
|
||||
await ffmpeg.exec([
|
||||
"-i",
|
||||
inputName,
|
||||
"-ss",
|
||||
startTime.toString(),
|
||||
"-t",
|
||||
duration.toString(),
|
||||
"-c",
|
||||
"copy", // Use stream copy for faster processing
|
||||
outputName,
|
||||
]);
|
||||
const frame = await sink.getSample(timeInSeconds);
|
||||
|
||||
// Read output file
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: "video/mp4" });
|
||||
if (!frame) {
|
||||
throw new Error("Could not get frame at specified time");
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
await ffmpeg.deleteFile(outputName);
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 320;
|
||||
canvas.height = 240;
|
||||
const ctx = canvas.getContext("2d");
|
||||
|
||||
return blob;
|
||||
};
|
||||
if (!ctx) {
|
||||
throw new Error("Could not get canvas context");
|
||||
}
|
||||
|
||||
export const getVideoInfo = async (
|
||||
videoFile: File
|
||||
): Promise<{
|
||||
frame.draw(ctx, 0, 0, 320, 240);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
canvas.toBlob(
|
||||
(blob) => {
|
||||
if (blob) {
|
||||
resolve(URL.createObjectURL(blob));
|
||||
} else {
|
||||
reject(new Error("Failed to create thumbnail blob"));
|
||||
}
|
||||
},
|
||||
"image/jpeg",
|
||||
0.8
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function getVideoInfo({
|
||||
videoFile,
|
||||
}: {
|
||||
videoFile: File;
|
||||
}): Promise<{
|
||||
duration: number;
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
}> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
}> {
|
||||
const input = new Input({
|
||||
source: new BlobSource(videoFile),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
const inputName = "input.mp4";
|
||||
const duration = await input.computeDuration();
|
||||
const videoTrack = await input.getPrimaryVideoTrack();
|
||||
|
||||
// Write input file
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await videoFile.arrayBuffer())
|
||||
);
|
||||
|
||||
// Capture FFmpeg stderr output with a one-time listener pattern
|
||||
let ffmpegOutput = "";
|
||||
let listening = true;
|
||||
const listener = (data: string) => {
|
||||
if (listening) ffmpegOutput += data;
|
||||
};
|
||||
ffmpeg.on("log", ({ message }) => listener(message));
|
||||
|
||||
// Run ffmpeg to get info (stderr will contain the info)
|
||||
try {
|
||||
await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);
|
||||
} catch (error) {
|
||||
listening = false;
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
console.error("FFmpeg execution failed:", error);
|
||||
throw new Error(
|
||||
"Failed to extract video info. The file may be corrupted or in an unsupported format."
|
||||
);
|
||||
if (!videoTrack) {
|
||||
throw new Error("No video track found in the file");
|
||||
}
|
||||
|
||||
// Disable listener after exec completes
|
||||
listening = false;
|
||||
|
||||
// Cleanup
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
|
||||
// Parse output for duration, resolution, and fps
|
||||
// Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s
|
||||
// Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc
|
||||
|
||||
const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);
|
||||
let duration = 0;
|
||||
if (durationMatch) {
|
||||
const [, h, m, s] = durationMatch;
|
||||
duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
|
||||
}
|
||||
|
||||
const videoStreamMatch = ffmpegOutput.match(
|
||||
/Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/
|
||||
);
|
||||
let width = 0,
|
||||
height = 0,
|
||||
fps = 0;
|
||||
if (videoStreamMatch) {
|
||||
width = parseInt(videoStreamMatch[1]);
|
||||
height = parseInt(videoStreamMatch[2]);
|
||||
fps = parseFloat(videoStreamMatch[3]);
|
||||
}
|
||||
// Get frame rate from packet statistics
|
||||
const packetStats = await videoTrack.computePacketStats(100);
|
||||
const fps = packetStats.averagePacketRate;
|
||||
|
||||
return {
|
||||
duration,
|
||||
width,
|
||||
height,
|
||||
width: videoTrack.displayWidth,
|
||||
height: videoTrack.displayHeight,
|
||||
fps,
|
||||
};
|
||||
};
|
||||
|
||||
export const convertToWebM = async (
|
||||
videoFile: File,
|
||||
onProgress?: (progress: number) => void
|
||||
): Promise<Blob> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
|
||||
const inputName = "input.mp4";
|
||||
const outputName = "output.webm";
|
||||
|
||||
// Set up progress callback
|
||||
if (onProgress) {
|
||||
ffmpeg.on("progress", ({ progress }) => {
|
||||
onProgress(progress * 100);
|
||||
});
|
||||
}
|
||||
|
||||
// Write input file
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await videoFile.arrayBuffer())
|
||||
);
|
||||
|
||||
// Convert to WebM
|
||||
await ffmpeg.exec([
|
||||
"-i",
|
||||
inputName,
|
||||
"-c:v",
|
||||
"libvpx-vp9",
|
||||
"-crf",
|
||||
"30",
|
||||
"-b:v",
|
||||
"0",
|
||||
"-c:a",
|
||||
"libopus",
|
||||
outputName,
|
||||
]);
|
||||
|
||||
// Read output file
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: "video/webm" });
|
||||
|
||||
// Cleanup
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
await ffmpeg.deleteFile(outputName);
|
||||
|
||||
return blob;
|
||||
};
|
||||
|
||||
export const extractAudio = async (
|
||||
videoFile: File,
|
||||
format: "mp3" | "wav" = "mp3"
|
||||
): Promise<Blob> => {
|
||||
const ffmpeg = await initFFmpeg();
|
||||
|
||||
const inputName = "input.mp4";
|
||||
const outputName = `output.${format}`;
|
||||
|
||||
// Write input file
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await videoFile.arrayBuffer())
|
||||
);
|
||||
|
||||
// Extract audio
|
||||
await ffmpeg.exec([
|
||||
"-i",
|
||||
inputName,
|
||||
"-vn", // Disable video
|
||||
"-acodec",
|
||||
format === "mp3" ? "libmp3lame" : "pcm_s16le",
|
||||
outputName,
|
||||
]);
|
||||
|
||||
// Read output file
|
||||
const data = await ffmpeg.readFile(outputName);
|
||||
const blob = new Blob([data], { type: `audio/${format}` });
|
||||
|
||||
// Cleanup
|
||||
await ffmpeg.deleteFile(inputName);
|
||||
await ffmpeg.deleteFile(outputName);
|
||||
|
||||
return blob;
|
||||
};
|
||||
}
|
||||
|
||||
// Audio mixing for timeline - keeping FFmpeg for now due to complexity
|
||||
// TODO: Replace with Mediabunny audio processing when implementing canvas preview
|
||||
export const extractTimelineAudio = async (
|
||||
onProgress?: (progress: number) => void
|
||||
): Promise<Blob> => {
|
||||
@@ -0,0 +1,179 @@
|
||||
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
|
||||
import type { TimelineTrack } from "@/types/timeline";
|
||||
import type { MediaItem } from "@/stores/media-store";
|
||||
|
||||
export interface RenderContext {
|
||||
ctx: CanvasRenderingContext2D;
|
||||
time: number;
|
||||
canvasWidth: number;
|
||||
canvasHeight: number;
|
||||
tracks: TimelineTrack[];
|
||||
mediaItems: MediaItem[];
|
||||
backgroundColor?: string;
|
||||
useCache?: boolean;
|
||||
cache?: Map<
|
||||
string,
|
||||
Map<
|
||||
number,
|
||||
{
|
||||
draw: (
|
||||
c: CanvasRenderingContext2D,
|
||||
x: number,
|
||||
y: number,
|
||||
w: number,
|
||||
h: number
|
||||
) => void;
|
||||
}
|
||||
>
|
||||
>;
|
||||
fps: number;
|
||||
}
|
||||
|
||||
export async function renderTimelineFrame({
|
||||
ctx,
|
||||
time,
|
||||
canvasWidth,
|
||||
canvasHeight,
|
||||
tracks,
|
||||
mediaItems,
|
||||
backgroundColor,
|
||||
useCache = true,
|
||||
cache,
|
||||
fps,
|
||||
}: RenderContext): Promise<void> {
|
||||
// Background
|
||||
ctx.clearRect(0, 0, canvasWidth, canvasHeight);
|
||||
if (backgroundColor && backgroundColor !== "transparent") {
|
||||
ctx.fillStyle = backgroundColor;
|
||||
ctx.fillRect(0, 0, canvasWidth, canvasHeight);
|
||||
}
|
||||
|
||||
const scaleX = 1;
|
||||
const scaleY = 1;
|
||||
const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));
|
||||
const active: Array<{
|
||||
track: TimelineTrack;
|
||||
element: TimelineTrack["elements"][number];
|
||||
mediaItem: MediaItem | null;
|
||||
}> = [];
|
||||
|
||||
for (let t = tracks.length - 1; t >= 0; t -= 1) {
|
||||
const track = tracks[t];
|
||||
for (const element of track.elements) {
|
||||
if ((element as any).hidden) continue;
|
||||
const elementStart = element.startTime;
|
||||
const elementEnd =
|
||||
element.startTime +
|
||||
(element.duration - element.trimStart - element.trimEnd);
|
||||
if (time >= elementStart && time < elementEnd) {
|
||||
let mediaItem: MediaItem | null = null;
|
||||
if (element.type === "media") {
|
||||
mediaItem =
|
||||
element.mediaId === "test"
|
||||
? null
|
||||
: idToMedia.get(element.mediaId) || null;
|
||||
}
|
||||
active.push({ track, element, mediaItem });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const { element, mediaItem } of active) {
|
||||
if (element.type === "media" && mediaItem) {
|
||||
if (mediaItem.type === "video") {
|
||||
const input = new Input({
|
||||
source: new BlobSource(mediaItem.file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
const track = await input.getPrimaryVideoTrack();
|
||||
if (!track) continue;
|
||||
const decodable = await track.canDecode();
|
||||
if (!decodable) continue;
|
||||
const sink = new VideoSampleSink(track);
|
||||
|
||||
const localTime = time - element.startTime + element.trimStart;
|
||||
const sample = await sink.getSample(localTime);
|
||||
if (!sample) continue;
|
||||
|
||||
const mediaW = Math.max(1, mediaItem.width || canvasWidth);
|
||||
const mediaH = Math.max(1, mediaItem.height || canvasHeight);
|
||||
const containScale = Math.min(
|
||||
canvasWidth / mediaW,
|
||||
canvasHeight / mediaH
|
||||
);
|
||||
const drawW = mediaW * containScale;
|
||||
const drawH = mediaH * containScale;
|
||||
const drawX = (canvasWidth - drawW) / 2;
|
||||
const drawY = (canvasHeight - drawH) / 2;
|
||||
sample.draw(ctx, drawX, drawY, drawW, drawH);
|
||||
}
|
||||
if (mediaItem.type === "image") {
|
||||
const img = new Image();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
img.onload = () => resolve();
|
||||
img.onerror = () => reject(new Error("Image load failed"));
|
||||
img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);
|
||||
});
|
||||
const mediaW = Math.max(
|
||||
1,
|
||||
mediaItem.width || img.naturalWidth || canvasWidth
|
||||
);
|
||||
const mediaH = Math.max(
|
||||
1,
|
||||
mediaItem.height || img.naturalHeight || canvasHeight
|
||||
);
|
||||
const containScale = Math.min(
|
||||
canvasWidth / mediaW,
|
||||
canvasHeight / mediaH
|
||||
);
|
||||
const drawW = mediaW * containScale;
|
||||
const drawH = mediaH * containScale;
|
||||
const drawX = (canvasWidth - drawW) / 2;
|
||||
const drawY = (canvasHeight - drawH) / 2;
|
||||
ctx.drawImage(img, drawX, drawY, drawW, drawH);
|
||||
}
|
||||
}
|
||||
if (element.type === "text") {
|
||||
const posX = canvasWidth / 2 + (element as any).x * scaleX;
|
||||
const posY = canvasHeight / 2 + (element as any).y * scaleY;
|
||||
ctx.save();
|
||||
ctx.translate(posX, posY);
|
||||
ctx.rotate(((element as any).rotation * Math.PI) / 180);
|
||||
ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));
|
||||
const px = (element as any).fontSize * scaleX;
|
||||
const weight = (element as any).fontWeight === "bold" ? "bold " : "";
|
||||
const style = (element as any).fontStyle === "italic" ? "italic " : "";
|
||||
ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;
|
||||
ctx.fillStyle = (element as any).color;
|
||||
ctx.textAlign = (element as any).textAlign;
|
||||
ctx.textBaseline = "middle";
|
||||
const metrics = ctx.measureText((element as any).content);
|
||||
const ascent =
|
||||
(metrics as unknown as { actualBoundingBoxAscent?: number })
|
||||
.actualBoundingBoxAscent ?? px * 0.8;
|
||||
const descent =
|
||||
(metrics as unknown as { actualBoundingBoxDescent?: number })
|
||||
.actualBoundingBoxDescent ?? px * 0.2;
|
||||
const textW = metrics.width;
|
||||
const textH = ascent + descent;
|
||||
const padX = 8 * scaleX;
|
||||
const padY = 4 * scaleX;
|
||||
if ((element as any).backgroundColor) {
|
||||
ctx.save();
|
||||
ctx.fillStyle = (element as any).backgroundColor;
|
||||
let bgLeft = -textW / 2;
|
||||
if (ctx.textAlign === "left") bgLeft = 0;
|
||||
if (ctx.textAlign === "right") bgLeft = -textW;
|
||||
ctx.fillRect(
|
||||
bgLeft - padX,
|
||||
-textH / 2 - padY,
|
||||
textW + padX * 2,
|
||||
textH + padY * 2
|
||||
);
|
||||
ctx.restore();
|
||||
}
|
||||
ctx.fillText((element as any).content, 0, 0);
|
||||
ctx.restore();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user