feat: mediabunny + preview canvas refactor

apps/web/src/components/editor/media-panel/views/captions.tsx

@@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item";
 import { BaseView } from "./base-view";
 import { Language, LanguageSelect } from "@/components/language-select";
 import { useState, useRef, useEffect } from "react";
-import { extractTimelineAudio } from "@/lib/ffmpeg-utils";
+import { extractTimelineAudio } from "@/lib/mediabunny-utils";
 import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption";
 import { useTimelineStore } from "@/stores/timeline-store";
 import { Loader2, Shield, Trash2, Upload } from "lucide-react";

apps/web/src/components/editor/preview-panel.tsx

@@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline";
 import { useMediaStore, type MediaItem } from "@/stores/media-store";
 import { usePlaybackStore } from "@/stores/playback-store";
 import { useEditorStore } from "@/stores/editor-store";
-import { VideoPlayer } from "@/components/ui/video-player";
-import { AudioPlayer } from "@/components/ui/audio-player";
 import { Button } from "@/components/ui/button";
 import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react";
 import { useState, useRef, useEffect, useCallback } from "react";
+import { VideoSampleSink } from "mediabunny";
+import { renderTimelineFrame } from "@/lib/timeline-renderer";
 import { cn } from "@/lib/utils";
 import { formatTimeCode } from "@/lib/time";
 import { EditableTimecode } from "@/components/ui/editable-timecode";
-import { FONT_CLASS_MAP } from "@/lib/font-config";
+import {
-import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store";
+  DEFAULT_CANVAS_SIZE,
+  DEFAULT_FPS,
+  useProjectStore,
+} from "@/stores/project-store";
 import { TextElementDragState } from "@/types/editor";
 import {
   Popover,
@@ -37,8 +40,20 @@ export function PreviewPanel() {
   const { tracks, getTotalDuration, updateTextElement } = useTimelineStore();
   const { mediaItems } = useMediaStore();
   const { currentTime, toggle, setCurrentTime } = usePlaybackStore();
+  const { isPlaying, volume, muted } = usePlaybackStore();
   const { activeProject } = useProjectStore();
   const previewRef = useRef<HTMLDivElement>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const lastFrameTimeRef = useRef(0);
+  const renderSeqRef = useRef(0);
+  const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>(
+    null
+  );
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const audioGainRef = useRef<GainNode | null>(null);
+  const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());
+  const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
   const containerRef = useRef<HTMLDivElement>(null);
   const [previewDimensions, setPreviewDimensions] = useState({
     width: 0,
@@ -262,6 +277,296 @@ export function PreviewPanel() {
 
   const activeElements = getActiveElements();
 
+  // Ensure first frame after mount/seek renders immediately
+  useEffect(() => {
+    const onSeek = () => {
+      lastFrameTimeRef.current = -Infinity;
+      renderSeqRef.current++;
+    };
+    window.addEventListener("playback-seek", onSeek as EventListener);
+    lastFrameTimeRef.current = -Infinity;
+    return () => {
+      window.removeEventListener("playback-seek", onSeek as EventListener);
+    };
+  }, []);
+
+  // Web Audio: schedule only on play/pause/seek/volume/mute changes
+  useEffect(() => {
+    const stopAll = () => {
+      for (const src of playingSourcesRef.current) {
+        try {
+          src.stop();
+        } catch {}
+      }
+      playingSourcesRef.current.clear();
+    };
+
+    type WebAudioWindow = Window & {
+      AudioContext?: typeof AudioContext;
+      webkitAudioContext?: typeof AudioContext;
+    };
+    const ensureAudioGraph = async () => {
+      if (!audioContextRef.current) {
+        const win = window as WebAudioWindow;
+        const Ctx = win.AudioContext ?? win.webkitAudioContext;
+        if (!Ctx) return;
+        audioContextRef.current = new Ctx();
+      }
+      if (!audioGainRef.current) {
+        audioGainRef.current = audioContextRef.current!.createGain();
+        audioGainRef.current.connect(audioContextRef.current!.destination);
+      }
+      if (audioContextRef.current!.state === "suspended") {
+        try {
+          await audioContextRef.current!.resume();
+        } catch {}
+      }
+      const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume));
+      audioGainRef.current!.gain.setValueAtTime(
+        gainValue,
+        audioContextRef.current!.currentTime
+      );
+    };
+
+    const scheduleNow = async () => {
+      await ensureAudioGraph();
+      const audioCtx = audioContextRef.current!;
+      const gain = audioGainRef.current!;
+
+      const tracksSnapshot = useTimelineStore.getState().tracks;
+      const mediaList = useMediaStore.getState().mediaItems;
+      const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const));
+      const playbackNow = usePlaybackStore.getState().currentTime;
+
+      const audible: Array<{
+        id: string;
+        elementStart: number;
+        trimStart: number;
+        trimEnd: number;
+        duration: number;
+        muted: boolean;
+        trackMuted: boolean;
+      }> = [];
+      const uniqueIds = new Set<string>();
+      for (const track of tracksSnapshot) {
+        for (const element of track.elements) {
+          if (element.type !== "media") continue;
+          const media = idToMedia.get(element.mediaId);
+          if (!media || media.type !== "audio") continue;
+          const visibleDuration =
+            element.duration - element.trimStart - element.trimEnd;
+          if (visibleDuration <= 0) continue;
+          const localTime = playbackNow - element.startTime + element.trimStart;
+          if (localTime < 0 || localTime >= visibleDuration) continue;
+          audible.push({
+            id: media.id,
+            elementStart: element.startTime,
+            trimStart: element.trimStart,
+            trimEnd: element.trimEnd,
+            duration: element.duration,
+            muted: !!element.muted,
+            trackMuted: !!track.muted,
+          });
+          uniqueIds.add(media.id);
+        }
+      }
+
+      if (audible.length === 0) return;
+
+      // Decode buffers as needed
+      const decodePromises: Array<Promise<void>> = [];
+      for (const id of uniqueIds) {
+        if (!audioBuffersRef.current.has(id)) {
+          const mediaItem = idToMedia.get(id);
+          if (!mediaItem) continue;
+          const p = (async () => {
+            const arr = await mediaItem.file.arrayBuffer();
+            const buf = await audioCtx.decodeAudioData(arr.slice(0));
+            audioBuffersRef.current.set(id, buf);
+          })();
+          decodePromises.push(p);
+        }
+      }
+      await Promise.all(decodePromises);
+
+      const startAt = audioCtx.currentTime + 0.02;
+      for (const entry of audible) {
+        if (entry.muted || entry.trackMuted) continue;
+        const buffer = audioBuffersRef.current.get(entry.id);
+        if (!buffer) continue;
+        const visibleDuration =
+          entry.duration - entry.trimStart - entry.trimEnd;
+        const localTime = Math.max(
+          0,
+          playbackNow - entry.elementStart + entry.trimStart
+        );
+        const playDuration = Math.max(0, visibleDuration - localTime);
+        if (playDuration <= 0) continue;
+        const src = audioCtx.createBufferSource();
+        src.buffer = buffer;
+        src.connect(gain);
+        try {
+          src.start(startAt, localTime, playDuration);
+          playingSourcesRef.current.add(src);
+        } catch {}
+      }
+    };
+
+    const onSeek = () => {
+      if (!isPlaying) return;
+      for (const src of playingSourcesRef.current) {
+        try {
+          src.stop();
+        } catch {}
+      }
+      playingSourcesRef.current.clear();
+      void scheduleNow();
+    };
+
+    // Apply volume/mute changes immediately
+    void ensureAudioGraph();
+
+    // Start/stop on play state changes
+    for (const src of playingSourcesRef.current) {
+      try {
+        src.stop();
+      } catch {}
+    }
+    playingSourcesRef.current.clear();
+    if (isPlaying) {
+      void scheduleNow();
+    }
+
+    window.addEventListener("playback-seek", onSeek as EventListener);
+    return () => {
+      window.removeEventListener("playback-seek", onSeek as EventListener);
+      for (const src of playingSourcesRef.current) {
+        try {
+          src.stop();
+        } catch {}
+      }
+      playingSourcesRef.current.clear();
+    };
+  }, [isPlaying, volume, muted, mediaItems]);
+
+  // Canvas: draw current frame for visible elements using offscreen compositing
+  useEffect(() => {
+    const draw = async () => {
+      const canvas = canvasRef.current;
+      if (!canvas) return;
+      const mainCtx = canvas.getContext("2d");
+      if (!mainCtx) return;
+
+      // Set canvas internal resolution to avoid blurry scaling
+      const displayWidth = Math.max(1, Math.floor(previewDimensions.width));
+      const displayHeight = Math.max(1, Math.floor(previewDimensions.height));
+      if (canvas.width !== displayWidth || canvas.height !== displayHeight) {
+        canvas.width = displayWidth;
+        canvas.height = displayHeight;
+      }
+
+      // Throttle rendering to project FPS during playback only
+      const fps = activeProject?.fps || DEFAULT_FPS;
+      const minDelta = 1 / fps;
+      if (isPlaying) {
+        if (currentTime - lastFrameTimeRef.current < minDelta) {
+          return;
+        }
+        lastFrameTimeRef.current = currentTime;
+      }
+
+      // Invalidate older async renders when user scrubs rapidly
+      const mySeq = (renderSeqRef.current += 1);
+
+      // Offscreen buffer to avoid flicker (reuse canvas)
+      if (!offscreenCanvasRef.current) {
+        const hasOffscreen =
+          typeof (globalThis as unknown as { OffscreenCanvas?: unknown })
+            .OffscreenCanvas !== "undefined";
+        if (hasOffscreen) {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas(
+            displayWidth,
+            displayHeight
+          ) as OffscreenCanvas;
+        } else {
+          const c = document.createElement("canvas");
+          c.width = displayWidth;
+          c.height = displayHeight;
+          offscreenCanvasRef.current = c;
+        }
+      }
+      // Ensure size matches
+      if (
+        offscreenCanvasRef.current &&
+        (offscreenCanvasRef.current as HTMLCanvasElement).getContext
+      ) {
+        const c = offscreenCanvasRef.current as HTMLCanvasElement;
+        if (c.width !== displayWidth || c.height !== displayHeight) {
+          c.width = displayWidth;
+          c.height = displayHeight;
+        }
+      } else {
+        const c = offscreenCanvasRef.current as OffscreenCanvas;
+        // @ts-ignore width/height exist on OffscreenCanvas in modern browsers
+        if (
+          (c as unknown as { width: number }).width !== displayWidth ||
+          (c as unknown as { height: number }).height !== displayHeight
+        ) {
+          // @ts-ignore
+          (c as unknown as { width: number }).width = displayWidth;
+          // @ts-ignore
+          (c as unknown as { height: number }).height = displayHeight;
+        }
+      }
+      const offscreenCanvas = offscreenCanvasRef.current as
+        | HTMLCanvasElement
+        | OffscreenCanvas;
+      const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext
+        ? (offscreenCanvas as HTMLCanvasElement).getContext("2d")
+        : (offscreenCanvas as OffscreenCanvas).getContext("2d");
+      if (!offCtx) return;
+
+      await renderTimelineFrame({
+        ctx: offCtx as CanvasRenderingContext2D,
+        time: currentTime,
+        canvasWidth: displayWidth,
+        canvasHeight: displayHeight,
+        tracks,
+        mediaItems,
+        backgroundColor:
+          activeProject?.backgroundType === "blur"
+            ? "transparent"
+            : activeProject?.backgroundColor || "#000000",
+        useCache: false,
+        fps: activeProject?.fps || DEFAULT_FPS,
+      });
+
+      // Blit offscreen to visible canvas
+      mainCtx.clearRect(0, 0, displayWidth, displayHeight);
+      if ((offscreenCanvas as HTMLCanvasElement).getContext) {
+        mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0);
+      } else {
+        mainCtx.drawImage(
+          offscreenCanvas as unknown as CanvasImageSource,
+          0,
+          0
+        );
+      }
+    };
+
+    void draw();
+  }, [
+    activeElements,
+    currentTime,
+    previewDimensions.width,
+    previewDimensions.height,
+    canvasSize.width,
+    canvasSize.height,
+    activeProject?.backgroundType,
+    activeProject?.backgroundColor,
+  ]);
+
   // Get media elements for blur background (video/image only)
   const getBlurBackgroundElements = (): ActiveElement[] => {
     return activeElements.filter(
@@ -275,213 +580,11 @@ export function PreviewPanel() {
 
   const blurBackgroundElements = getBlurBackgroundElements();
 
-  // Render blur background layer
+  // Render blur background layer (handled by canvas now)
-  const renderBlurBackground = () => {
+  const renderBlurBackground = () => null;
-    if (
-      !activeProject?.backgroundType ||
-      activeProject.backgroundType !== "blur" ||
-      blurBackgroundElements.length === 0
-    ) {
-      return null;
-    }
 
-    // Use the first media element for background (could be enhanced to use primary/focused element)
+  // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio.
-    const backgroundElement = blurBackgroundElements[0];
+  const renderElement = (_elementData: ActiveElement) => null;
-    const { element, mediaItem } = backgroundElement;
-
-    if (!mediaItem) return null;
-
-    const blurIntensity = activeProject.blurIntensity || 8;
-
-    if (mediaItem.type === "video") {
-      return (
-        <div
-          key={`blur-${element.id}`}
-          className="absolute inset-0 overflow-hidden"
-          style={{
-            filter: `blur(${blurIntensity}px)`,
-            transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
-            transformOrigin: "center",
-          }}
-        >
-          <VideoPlayer
-            src={mediaItem.url!}
-            poster={mediaItem.thumbnailUrl}
-            clipStartTime={element.startTime}
-            trimStart={element.trimStart}
-            trimEnd={element.trimEnd}
-            clipDuration={element.duration}
-            className="w-full h-full object-cover"
-            trackMuted={true}
-          />
-        </div>
-      );
-    }
-
-    if (mediaItem.type === "image") {
-      return (
-        <div
-          key={`blur-${element.id}`}
-          className="absolute inset-0 overflow-hidden"
-          style={{
-            filter: `blur(${blurIntensity}px)`,
-            transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
-            transformOrigin: "center",
-          }}
-        >
-          <img
-            src={mediaItem.url!}
-            alt={mediaItem.name}
-            className="w-full h-full object-cover"
-            draggable={false}
-          />
-        </div>
-      );
-    }
-
-    return null;
-  };
-
-  // Render an element
-  const renderElement = (elementData: ActiveElement, index: number) => {
-    const { element, mediaItem } = elementData;
-
-    // Text elements
-    if (element.type === "text") {
-      const fontClassName =
-        FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || "";
-
-      const scaleRatio = previewDimensions.width / canvasSize.width;
-
-      return (
-        <div
-          key={element.id}
-          className="absolute cursor-grab"
-          onMouseDown={(e) =>
-            handleTextMouseDown(e, element, elementData.track.id)
-          }
-          style={{
-            left: `${
-              50 +
-              ((dragState.isDragging && dragState.elementId === element.id
-                ? dragState.currentX
-                : element.x) /
-                canvasSize.width) *
-                100
-            }%`,
-            top: `${
-              50 +
-              ((dragState.isDragging && dragState.elementId === element.id
-                ? dragState.currentY
-                : element.y) /
-                canvasSize.height) *
-                100
-            }%`,
-            transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`,
-            opacity: element.opacity,
-            zIndex: 100 + index, // Text elements on top
-          }}
-        >
-          <div
-            className={fontClassName}
-            style={{
-              fontSize: `${element.fontSize * scaleRatio}px`,
-              color: element.color,
-              backgroundColor: element.backgroundColor,
-              textAlign: element.textAlign,
-              fontWeight: element.fontWeight,
-              fontStyle: element.fontStyle,
-              textDecoration: element.textDecoration,
-              padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`,
-              borderRadius: `${2 * scaleRatio}px`,
-              whiteSpace: "nowrap",
-              // Fallback for system fonts that don't have classes
-              ...(fontClassName === "" && { fontFamily: element.fontFamily }),
-            }}
-          >
-            {element.content}
-          </div>
-        </div>
-      );
-    }
-
-    // Media elements
-    if (element.type === "media") {
-      // Test elements
-      if (!mediaItem || element.mediaId === "test") {
-        return (
-          <div
-            key={element.id}
-            className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center"
-          >
-            <div className="text-center">
-              <div className="text-2xl mb-2">🎬</div>
-              <p className="text-xs text-foreground">{element.name}</p>
-            </div>
-          </div>
-        );
-      }
-
-      // Video elements
-      if (mediaItem.type === "video") {
-        return (
-          <div
-            key={element.id}
-            className="absolute inset-0 flex items-center justify-center"
-          >
-            <VideoPlayer
-              src={mediaItem.url!}
-              poster={mediaItem.thumbnailUrl}
-              clipStartTime={element.startTime}
-              trimStart={element.trimStart}
-              trimEnd={element.trimEnd}
-              clipDuration={element.duration}
-              trackMuted={element.muted || elementData.track.muted}
-            />
-          </div>
-        );
-      }
-
-      // Image elements
-      if (mediaItem.type === "image") {
-        return (
-          <div
-            key={element.id}
-            className="absolute inset-0 flex items-center justify-center"
-          >
-            <img
-              src={mediaItem.url!}
-              alt={mediaItem.name}
-              className="max-w-full max-h-full object-contain"
-              draggable={false}
-            />
-          </div>
-        );
-      }
-
-      // Audio elements (no visual representation)
-      if (mediaItem.type === "audio") {
-        return (
-          <div
-            key={element.id}
-            className="absolute inset-0"
-            style={{ pointerEvents: "none" }}
-          >
-            <AudioPlayer
-              src={mediaItem.url!}
-              clipStartTime={element.startTime}
-              trimStart={element.trimStart}
-              trimEnd={element.trimEnd}
-              clipDuration={element.duration}
-              trackMuted={element.muted || elementData.track.muted}
-            />
-          </div>
-        );
-      }
-    }
-
-    return null;
-  };
 
   return (
     <>
@@ -505,14 +608,23 @@ export function PreviewPanel() {
               }}
             >
               {renderBlurBackground()}
+              <canvas
+                ref={canvasRef}
+                style={{
+                  position: "absolute",
+                  left: 0,
+                  top: 0,
+                  width: previewDimensions.width,
+                  height: previewDimensions.height,
+                }}
+                aria-label="Video preview canvas"
+              />
               {activeElements.length === 0 ? (
                 <div className="absolute inset-0 flex items-center justify-center text-muted-foreground">
                   No elements at current time
                 </div>
               ) : (
-                activeElements.map((elementData, index) =>
+                activeElements.map((elementData) => renderElement(elementData))
-                  renderElement(elementData, index)
-                )
               )}
               <LayoutGuideOverlay />
             </div>

apps/web/src/lib/media-processing.ts

@@ -6,7 +6,7 @@ import {
   getImageDimensions,
   type MediaItem,
 } from "@/stores/media-store";
-import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";
+import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";
 
 export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}
 
@@ -37,33 +37,23 @@ export async function processMediaFiles(
 
     try {
       if (fileType === "image") {
-        // Get image dimensions
         const dimensions = await getImageDimensions(file);
         width = dimensions.width;
         height = dimensions.height;
       } else if (fileType === "video") {
         try {
-          // Use FFmpeg for comprehensive video info extraction
+          const videoInfo = await getVideoInfo({ videoFile: file });
-          const videoInfo = await getVideoInfo(file);
           duration = videoInfo.duration;
           width = videoInfo.width;
           height = videoInfo.height;
           fps = videoInfo.fps;
 
-          // Generate thumbnail using FFmpeg
+          thumbnailUrl = await generateThumbnail({
-          thumbnailUrl = await generateThumbnail(file, 1);
+            videoFile: file,
+            timeInSeconds: 1,
+          });
         } catch (error) {
-          console.warn(
+          console.warn("Video processing failed", error);
-            "FFmpeg processing failed, falling back to basic processing:",
-            error
-          );
-          // Fallback to basic processing
-          const videoResult = await generateVideoThumbnail(file);
-          thumbnailUrl = videoResult.thumbnailUrl;
-          width = videoResult.width;
-          height = videoResult.height;
-          duration = await getMediaDuration(file);
-          // FPS will remain undefined for fallback
         }
       } else if (fileType === "audio") {
         // For audio, we don't set width/height/fps (they'll be undefined)
@@ -82,7 +72,6 @@ export async function processMediaFiles(
         fps,
       });
 
-      // Yield back to the event loop to keep the UI responsive
       await new Promise((resolve) => setTimeout(resolve, 0));
 
       completed += 1;

apps/web/src/lib/mediabunny-utils.ts

@@ -1,7 +1,7 @@
 import { FFmpeg } from "@ffmpeg/ffmpeg";
-import { toBlobURL } from "@ffmpeg/util";
 import { useTimelineStore } from "@/stores/timeline-store";
 import { useMediaStore } from "@/stores/media-store";
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
 
 let ffmpeg: FFmpeg | null = null;
 
@@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
   return ffmpeg;
 };
 
-export const generateThumbnail = async (
+export async function generateThumbnail({
-  videoFile: File,
+  videoFile,
-  timeInSeconds = 1
+  timeInSeconds,
-): Promise<string> => {
+}: {
-  const ffmpeg = await initFFmpeg();
+  videoFile: File;
+  timeInSeconds: number;
+}): Promise<string> {
+  const input = new Input({
+    source: new BlobSource(videoFile),
+    formats: ALL_FORMATS,
+  });
 
-  const inputName = "input.mp4";
+  const videoTrack = await input.getPrimaryVideoTrack();
-  const outputName = "thumbnail.jpg";
+  if (!videoTrack) {
-
+    throw new Error("No video track found in the file");
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Generate thumbnail at specific time
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-ss",
-    timeInSeconds.toString(),
-    "-vframes",
-    "1",
-    "-vf",
-    "scale=320:240",
-    "-q:v",
-    "2",
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: "image/jpeg" });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
-
-  return URL.createObjectURL(blob);
-};
-
-export const trimVideo = async (
-  videoFile: File,
-  startTime: number,
-  endTime: number,
-  onProgress?: (progress: number) => void
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = "output.mp4";
-
-  // Set up progress callback
-  if (onProgress) {
-    ffmpeg.on("progress", ({ progress }) => {
-      onProgress(progress * 100);
-    });
   }
 
-  // Write input file
+  // Check if we can decode this video
-  await ffmpeg.writeFile(
+  const canDecode = await videoTrack.canDecode();
-    inputName,
+  if (!canDecode) {
-    new Uint8Array(await videoFile.arrayBuffer())
+    throw new Error("Video codec not supported for decoding");
-  );
+  }
 
-  const duration = endTime - startTime;
+  const sink = new VideoSampleSink(videoTrack);
 
-  // Trim video
+  const frame = await sink.getSample(timeInSeconds);
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-ss",
-    startTime.toString(),
-    "-t",
-    duration.toString(),
-    "-c",
-    "copy", // Use stream copy for faster processing
-    outputName,
-  ]);
 
-  // Read output file
+  if (!frame) {
-  const data = await ffmpeg.readFile(outputName);
+    throw new Error("Could not get frame at specified time");
-  const blob = new Blob([data], { type: "video/mp4" });
+  }
 
-  // Cleanup
+  const canvas = document.createElement("canvas");
-  await ffmpeg.deleteFile(inputName);
+  canvas.width = 320;
-  await ffmpeg.deleteFile(outputName);
+  canvas.height = 240;
+  const ctx = canvas.getContext("2d");
 
-  return blob;
+  if (!ctx) {
-};
+    throw new Error("Could not get canvas context");
+  }
 
-export const getVideoInfo = async (
+  frame.draw(ctx, 0, 0, 320, 240);
-  videoFile: File
+
-): Promise<{
+  return new Promise((resolve, reject) => {
+    canvas.toBlob(
+      (blob) => {
+        if (blob) {
+          resolve(URL.createObjectURL(blob));
+        } else {
+          reject(new Error("Failed to create thumbnail blob"));
+        }
+      },
+      "image/jpeg",
+      0.8
+    );
+  });
+}
+
+export async function getVideoInfo({
+  videoFile,
+}: {
+  videoFile: File;
+}): Promise<{
   duration: number;
   width: number;
   height: number;
   fps: number;
-}> => {
+}> {
-  const ffmpeg = await initFFmpeg();
+  const input = new Input({
+    source: new BlobSource(videoFile),
+    formats: ALL_FORMATS,
+  });
 
-  const inputName = "input.mp4";
+  const duration = await input.computeDuration();
+  const videoTrack = await input.getPrimaryVideoTrack();
 
-  // Write input file
+  if (!videoTrack) {
-  await ffmpeg.writeFile(
+    throw new Error("No video track found in the file");
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Capture FFmpeg stderr output with a one-time listener pattern
-  let ffmpegOutput = "";
-  let listening = true;
-  const listener = (data: string) => {
-    if (listening) ffmpegOutput += data;
-  };
-  ffmpeg.on("log", ({ message }) => listener(message));
-
-  // Run ffmpeg to get info (stderr will contain the info)
-  try {
-    await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);
-  } catch (error) {
-    listening = false;
-    await ffmpeg.deleteFile(inputName);
-    console.error("FFmpeg execution failed:", error);
-    throw new Error(
-      "Failed to extract video info. The file may be corrupted or in an unsupported format."
-    );
   }
 
-  // Disable listener after exec completes
+  // Get frame rate from packet statistics
-  listening = false;
+  const packetStats = await videoTrack.computePacketStats(100);
-
+  const fps = packetStats.averagePacketRate;
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-
-  // Parse output for duration, resolution, and fps
-  // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s
-  // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc
-
-  const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);
-  let duration = 0;
-  if (durationMatch) {
-    const [, h, m, s] = durationMatch;
-    duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
-  }
-
-  const videoStreamMatch = ffmpegOutput.match(
-    /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/
-  );
-  let width = 0,
-    height = 0,
-    fps = 0;
-  if (videoStreamMatch) {
-    width = parseInt(videoStreamMatch[1]);
-    height = parseInt(videoStreamMatch[2]);
-    fps = parseFloat(videoStreamMatch[3]);
-  }
 
   return {
     duration,
-    width,
+    width: videoTrack.displayWidth,
-    height,
+    height: videoTrack.displayHeight,
     fps,
   };
-};
+}
-
-export const convertToWebM = async (
-  videoFile: File,
-  onProgress?: (progress: number) => void
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = "output.webm";
-
-  // Set up progress callback
-  if (onProgress) {
-    ffmpeg.on("progress", ({ progress }) => {
-      onProgress(progress * 100);
-    });
-  }
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Convert to WebM
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-c:v",
-    "libvpx-vp9",
-    "-crf",
-    "30",
-    "-b:v",
-    "0",
-    "-c:a",
-    "libopus",
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: "video/webm" });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
-
-  return blob;
-};
-
-export const extractAudio = async (
-  videoFile: File,
-  format: "mp3" | "wav" = "mp3"
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = `output.${format}`;
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Extract audio
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-vn", // Disable video
-    "-acodec",
-    format === "mp3" ? "libmp3lame" : "pcm_s16le",
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: `audio/${format}` });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
-
-  return blob;
-};
 
+// Audio mixing for timeline - keeping FFmpeg for now due to complexity
+// TODO: Replace with Mediabunny audio processing when implementing canvas preview
 export const extractTimelineAudio = async (
   onProgress?: (progress: number) => void
 ): Promise<Blob> => {

apps/web/src/lib/timeline-renderer.ts

@@ -0,0 +1,179 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+import type { TimelineTrack } from "@/types/timeline";
+import type { MediaItem } from "@/stores/media-store";
+
+export interface RenderContext {
+  ctx: CanvasRenderingContext2D;
+  time: number;
+  canvasWidth: number;
+  canvasHeight: number;
+  tracks: TimelineTrack[];
+  mediaItems: MediaItem[];
+  backgroundColor?: string;
+  useCache?: boolean;
+  cache?: Map<
+    string,
+    Map<
+      number,
+      {
+        draw: (
+          c: CanvasRenderingContext2D,
+          x: number,
+          y: number,
+          w: number,
+          h: number
+        ) => void;
+      }
+    >
+  >;
+  fps: number;
+}
+
+export async function renderTimelineFrame({
+  ctx,
+  time,
+  canvasWidth,
+  canvasHeight,
+  tracks,
+  mediaItems,
+  backgroundColor,
+  useCache = true,
+  cache,
+  fps,
+}: RenderContext): Promise<void> {
+  // Background
+  ctx.clearRect(0, 0, canvasWidth, canvasHeight);
+  if (backgroundColor && backgroundColor !== "transparent") {
+    ctx.fillStyle = backgroundColor;
+    ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+  }
+
+  const scaleX = 1;
+  const scaleY = 1;
+  const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));
+  const active: Array<{
+    track: TimelineTrack;
+    element: TimelineTrack["elements"][number];
+    mediaItem: MediaItem | null;
+  }> = [];
+
+  for (let t = tracks.length - 1; t >= 0; t -= 1) {
+    const track = tracks[t];
+    for (const element of track.elements) {
+      if ((element as any).hidden) continue;
+      const elementStart = element.startTime;
+      const elementEnd =
+        element.startTime +
+        (element.duration - element.trimStart - element.trimEnd);
+      if (time >= elementStart && time < elementEnd) {
+        let mediaItem: MediaItem | null = null;
+        if (element.type === "media") {
+          mediaItem =
+            element.mediaId === "test"
+              ? null
+              : idToMedia.get(element.mediaId) || null;
+        }
+        active.push({ track, element, mediaItem });
+      }
+    }
+  }
+
+  for (const { element, mediaItem } of active) {
+    if (element.type === "media" && mediaItem) {
+      if (mediaItem.type === "video") {
+        const input = new Input({
+          source: new BlobSource(mediaItem.file),
+          formats: ALL_FORMATS,
+        });
+        const track = await input.getPrimaryVideoTrack();
+        if (!track) continue;
+        const decodable = await track.canDecode();
+        if (!decodable) continue;
+        const sink = new VideoSampleSink(track);
+
+        const localTime = time - element.startTime + element.trimStart;
+        const sample = await sink.getSample(localTime);
+        if (!sample) continue;
+
+        const mediaW = Math.max(1, mediaItem.width || canvasWidth);
+        const mediaH = Math.max(1, mediaItem.height || canvasHeight);
+        const containScale = Math.min(
+          canvasWidth / mediaW,
+          canvasHeight / mediaH
+        );
+        const drawW = mediaW * containScale;
+        const drawH = mediaH * containScale;
+        const drawX = (canvasWidth - drawW) / 2;
+        const drawY = (canvasHeight - drawH) / 2;
+        sample.draw(ctx, drawX, drawY, drawW, drawH);
+      }
+      if (mediaItem.type === "image") {
+        const img = new Image();
+        await new Promise<void>((resolve, reject) => {
+          img.onload = () => resolve();
+          img.onerror = () => reject(new Error("Image load failed"));
+          img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);
+        });
+        const mediaW = Math.max(
+          1,
+          mediaItem.width || img.naturalWidth || canvasWidth
+        );
+        const mediaH = Math.max(
+          1,
+          mediaItem.height || img.naturalHeight || canvasHeight
+        );
+        const containScale = Math.min(
+          canvasWidth / mediaW,
+          canvasHeight / mediaH
+        );
+        const drawW = mediaW * containScale;
+        const drawH = mediaH * containScale;
+        const drawX = (canvasWidth - drawW) / 2;
+        const drawY = (canvasHeight - drawH) / 2;
+        ctx.drawImage(img, drawX, drawY, drawW, drawH);
+      }
+    }
+    if (element.type === "text") {
+      const posX = canvasWidth / 2 + (element as any).x * scaleX;
+      const posY = canvasHeight / 2 + (element as any).y * scaleY;
+      ctx.save();
+      ctx.translate(posX, posY);
+      ctx.rotate(((element as any).rotation * Math.PI) / 180);
+      ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));
+      const px = (element as any).fontSize * scaleX;
+      const weight = (element as any).fontWeight === "bold" ? "bold " : "";
+      const style = (element as any).fontStyle === "italic" ? "italic " : "";
+      ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;
+      ctx.fillStyle = (element as any).color;
+      ctx.textAlign = (element as any).textAlign;
+      ctx.textBaseline = "middle";
+      const metrics = ctx.measureText((element as any).content);
+      const ascent =
+        (metrics as unknown as { actualBoundingBoxAscent?: number })
+          .actualBoundingBoxAscent ?? px * 0.8;
+      const descent =
+        (metrics as unknown as { actualBoundingBoxDescent?: number })
+          .actualBoundingBoxDescent ?? px * 0.2;
+      const textW = metrics.width;
+      const textH = ascent + descent;
+      const padX = 8 * scaleX;
+      const padY = 4 * scaleX;
+      if ((element as any).backgroundColor) {
+        ctx.save();
+        ctx.fillStyle = (element as any).backgroundColor;
+        let bgLeft = -textW / 2;
+        if (ctx.textAlign === "left") bgLeft = 0;
+        if (ctx.textAlign === "right") bgLeft = -textW;
+        ctx.fillRect(
+          bgLeft - padX,
+          -textH / 2 - padY,
+          textW + padX * 2,
+          textH + padY * 2
+        );
+        ctx.restore();
+      }
+      ctx.fillText((element as any).content, 0, 0);
+      ctx.restore();
+    }
+  }
+}

bun.lock

@@ -4,6 +4,7 @@
     "": {
       "name": "opencut",
       "dependencies": {
+        "mediabunny": "^1.9.3",
         "next": "^15.3.4",
         "react-country-flag": "^3.1.0",
         "wavesurfer.js": "^7.9.8",
@@ -558,6 +559,10 @@
 
     "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],
 
+    "@types/dom-mediacapture-transform": ["@types/dom-mediacapture-transform@0.1.11", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="],
+
+    "@types/dom-webcodecs": ["@types/dom-webcodecs@0.1.13", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="],
+
     "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
 
     "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="],
@@ -896,6 +901,8 @@
 
     "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
 
+    "mediabunny": ["mediabunny@1.9.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ=="],
+
     "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],
 
     "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],
@@ -1320,10 +1327,10 @@
 
     "motion/framer-motion/motion-utils": ["motion-utils@12.23.6", "", {}, "sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ=="],
 
-    "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
-
     "next/postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
 
+    "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
+
     "opencut/next/@next/env": ["@next/env@15.4.5", "", {}, "sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ=="],
 
     "opencut/next/@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@15.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-84dAN4fkfdC7nX6udDLz9GzQlMUwEMKD7zsseXrl7FTeIItF8vpk1lhLEnsotiiDt+QFu3O1FVWnqwcRD2U3KA=="],

package.json

@@ -20,6 +20,7 @@
     "format": "npx ultracite@latest format"
   },
   "dependencies": {
+    "mediabunny": "^1.9.3",
     "next": "^15.3.4",
     "react-country-flag": "^3.1.0",
     "wavesurfer.js": "^7.9.8"