refactor not done

apps/web/src/services/renderer/canvas-renderer.ts

@@ -0,0 +1,87 @@
+import { BaseNode } from "./nodes/base-node";
+
+export type CanvasRendererParams = {
+  width: number;
+  height: number;
+  fps: number;
+};
+
+export class CanvasRenderer {
+  canvas: OffscreenCanvas | HTMLCanvasElement;
+  context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
+  width: number;
+  height: number;
+  fps: number;
+
+  constructor({ width, height, fps }: CanvasRendererParams) {
+    this.width = width;
+    this.height = height;
+    this.fps = fps;
+
+    try {
+      this.canvas = new OffscreenCanvas(width, height);
+    } catch {
+      this.canvas = document.createElement("canvas");
+      this.canvas.width = width;
+      this.canvas.height = height;
+    }
+
+    const context = this.canvas.getContext("2d");
+    if (!context) {
+      throw new Error("Failed to get canvas context");
+    }
+
+    this.context = context as
+      | OffscreenCanvasRenderingContext2D
+      | CanvasRenderingContext2D;
+  }
+
+  setSize({ width, height }: { width: number; height: number }) {
+    this.width = width;
+    this.height = height;
+
+    if (this.canvas instanceof OffscreenCanvas) {
+      this.canvas = new OffscreenCanvas(width, height);
+    } else {
+      this.canvas.width = width;
+      this.canvas.height = height;
+    }
+
+    const context = this.canvas.getContext("2d");
+    if (!context) {
+      throw new Error("Failed to get canvas context");
+    }
+    this.context = context as
+      | OffscreenCanvasRenderingContext2D
+      | CanvasRenderingContext2D;
+  }
+
+  private clear() {
+    this.context.fillStyle = "black";
+    this.context.fillRect(0, 0, this.canvas.width, this.canvas.height);
+  }
+
+  async render({ node, time }: { node: BaseNode; time: number }) {
+    this.clear();
+    await node.render({ renderer: this, time });
+  }
+
+  async renderToCanvas({
+    node,
+    time,
+    targetCanvas,
+  }: {
+    node: BaseNode;
+    time: number;
+    targetCanvas: HTMLCanvasElement;
+  }) {
+    await this.render({ node, time });
+
+    const ctx = targetCanvas.getContext("2d");
+    if (!ctx) {
+      throw new Error("Failed to get target canvas context");
+    }
+
+    ctx.drawImage(this.canvas, 0, 0, targetCanvas.width, targetCanvas.height);
+  }
+}

apps/web/src/services/renderer/nodes/base-node.ts

@@ -0,0 +1,29 @@
+import { CanvasRenderer } from "../canvas-renderer";
+
+export type BaseNodeParams = object | undefined;
+
+export class BaseNode<Params extends BaseNodeParams = BaseNodeParams> {
+  params: Params;
+
+  constructor(params?: Params) {
+    this.params = params ?? ({} as Params);
+  }
+
+  children: BaseNode[] = [];
+
+  add(child: BaseNode) {
+    this.children.push(child);
+    return this;
+  }
+
+  remove(child: BaseNode) {
+    this.children = this.children.filter((c) => c !== child);
+    return this;
+  }
+
+  async render({ renderer, time }: { renderer: CanvasRenderer; time: number }): Promise<void> {
+    for (const child of this.children) {
+      await child.render({ renderer, time });
+    }
+  }
+}

apps/web/src/services/renderer/nodes/blur-background-node.ts

@@ -0,0 +1,84 @@
+import type { CanvasRenderer } from "../canvas-renderer";
+import { BaseNode } from "./base-node";
+
+export type BlurBackgroundNodeParams = {
+  blurIntensity: number;
+  contentNodes: BaseNode[];
+};
+
+export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> {
+  private blurIntensity: number;
+  private contentNodes: BaseNode[];
+
+  constructor(params: BlurBackgroundNodeParams) {
+    super(params);
+    this.blurIntensity = params.blurIntensity;
+    this.contentNodes = params.contentNodes;
+  }
+
+  async render({
+    renderer,
+    time,
+  }: {
+    renderer: CanvasRenderer;
+    time: number;
+  }): Promise<void> {
+    let offscreen: OffscreenCanvas | HTMLCanvasElement;
+    let offscreenCtx:
+      | OffscreenCanvasRenderingContext2D
+      | CanvasRenderingContext2D;
+
+    try {
+      offscreen = new OffscreenCanvas(renderer.width, renderer.height);
+      const ctx = offscreen.getContext("2d");
+      if (!ctx) {
+        throw new Error("Failed to get offscreen canvas context");
+      }
+      offscreenCtx = ctx;
+    } catch {
+      offscreen = document.createElement("canvas");
+      offscreen.width = renderer.width;
+      offscreen.height = renderer.height;
+      const ctx = offscreen.getContext("2d");
+      if (!ctx) {
+        throw new Error("Failed to get canvas context");
+      }
+      offscreenCtx = ctx;
+    }
+
+    const originalContext = renderer.context;
+    renderer.context = offscreenCtx;
+
+    for (const node of this.contentNodes) {
+      await node.render({ renderer, time });
+    }
+
+    renderer.context = originalContext;
+
+    renderer.context.save();
+    renderer.context.filter = `blur(${this.blurIntensity}px)`;
+
+    const scale = Math.max(
+      renderer.width / offscreen.width,
+      renderer.height / offscreen.height,
+    );
+    const scaledWidth = offscreen.width * scale;
+    const scaledHeight = offscreen.height * scale;
+    const x = (renderer.width - scaledWidth) / 2;
+    const y = (renderer.height - scaledHeight) / 2;
+
+    if (offscreen instanceof OffscreenCanvas) {
+      renderer.context.drawImage(
+        offscreen as unknown as CanvasImageSource,
+        x,
+        y,
+        scaledWidth,
+        scaledHeight,
+      );
+    } else {
+      renderer.context.drawImage(offscreen, x, y, scaledWidth, scaledHeight);
+    }
+
+    renderer.context.restore();
+  }
+}

apps/web/src/services/renderer/nodes/color-node.ts

@@ -0,0 +1,20 @@
+import { CanvasRenderer } from "../canvas-renderer";
+import { BaseNode } from "./base-node";
+
+export type ColorNodeParams = {
+  color: string;
+};
+
+export class ColorNode extends BaseNode<ColorNodeParams> {
+  private color: string;
+
+  constructor(params: ColorNodeParams) {
+    super(params);
+    this.color = params.color;
+  }
+
+  async render({ renderer }: { renderer: CanvasRenderer }) {
+    renderer.context.fillStyle = this.color;
+    renderer.context.fillRect(0, 0, renderer.width, renderer.height);
+  }
+}

apps/web/src/services/renderer/nodes/image-node.ts

@@ -0,0 +1,92 @@
+import { CanvasRenderer } from "../canvas-renderer";
+import { BaseNode } from "./base-node";
+import { BaseMediaNodeParams } from "./video-node";
+
+const IMAGE_EPSILON = 1 / 1000;
+
+export type ImageNodeParams = BaseMediaNodeParams;
+
+export class ImageNode extends BaseNode<ImageNodeParams> {
+  private image?: HTMLImageElement;
+  private readyPromise: Promise<void>;
+
+  constructor(params: ImageNodeParams) {
+    super(params);
+    this.readyPromise = this.load();
+  }
+
+  private async load() {
+    this.image = new Image();
+    const url = URL.createObjectURL(this.params.file);
+
+    await new Promise<void>((resolve, reject) => {
+      this.image!.onload = () => resolve();
+      this.image!.onerror = () => reject(new Error("Image load failed"));
+      this.image!.src = url;
+    });
+
+    URL.revokeObjectURL(url);
+  }
+
+  private getImageTime(time: number) {
+    return time - this.params.timeOffset + this.params.trimStart;
+  }
+
+  private isInRange(time: number) {
+    const imageTime = this.getImageTime(time);
+    return (
+      imageTime >= this.params.trimStart - IMAGE_EPSILON &&
+      imageTime < this.params.duration - this.params.trimEnd
+    );
+  }
+
+  async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
+    await super.render({ renderer, time });
+
+    if (!this.isInRange(time)) {
+      return;
+    }
+
+    await this.readyPromise;
+
+    if (!this.image) {
+      return;
+    }
+
+    renderer.context.save();
+
+    if (this.params.opacity !== undefined) {
+      renderer.context.globalAlpha = this.params.opacity;
+    }
+
+    if (
+      this.params.x !== undefined &&
+      this.params.y !== undefined &&
+      this.params.width !== undefined &&
+      this.params.height !== undefined
+    ) {
+      renderer.context.drawImage(
+        this.image,
+        this.params.x,
+        this.params.y,
+        this.params.width,
+        this.params.height,
+      );
+    } else {
+      const mediaW = this.image.naturalWidth || renderer.width;
+      const mediaH = this.image.naturalHeight || renderer.height;
+      const containScale = Math.min(
+        renderer.width / mediaW,
+        renderer.height / mediaH,
+      );
+      const drawW = mediaW * containScale;
+      const drawH = mediaH * containScale;
+      const drawX = (renderer.width - drawW) / 2;
+      const drawY = (renderer.height - drawH) / 2;
+
+      renderer.context.drawImage(this.image, drawX, drawY, drawW, drawH);
+    }
+
+    renderer.context.restore();
+  }
+}

apps/web/src/services/renderer/nodes/root-node.ts

@@ -0,0 +1,11 @@
+import { BaseNode } from "./base-node";
+
+export type RootNodeParams = {
+  duration: number;
+};
+
+export class RootNode extends BaseNode<RootNodeParams> {
+  get duration() {
+    return this.params.duration ?? 0;
+  }
+}

apps/web/src/services/renderer/nodes/text-node.ts

@@ -0,0 +1,72 @@
+import { CanvasRenderer } from "../canvas-renderer";
+import { BaseNode } from "./base-node";
+import { TextElement } from "@/types/timeline";
+
+export type TextNodeParams = TextElement & {
+  textBaseline?: CanvasTextBaseline;
+};
+
+export class TextNode extends BaseNode<TextNodeParams> {
+  isInRange({ time }: { time: number }) {
+    const visibleDuration =
+      this.params.duration - this.params.trimStart - this.params.trimEnd;
+    return (
+      time >= this.params.startTime &&
+      time < this.params.startTime + visibleDuration
+    );
+  }
+
+  async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
+    if (!this.isInRange({ time })) {
+      return;
+    }
+
+    renderer.context.save();
+
+    renderer.context.translate(this.params.x, this.params.y);
+    if (this.params.rotation) {
+      renderer.context.rotate((this.params.rotation * Math.PI) / 180);
+    }
+
+    const fontWeight = this.params.fontWeight === "bold" ? "bold" : "normal";
+    const fontStyle = this.params.fontStyle === "italic" ? "italic" : "normal";
+    renderer.context.font = `${fontStyle} ${fontWeight} ${this.params.fontSize}px ${this.params.fontFamily}`;
+    renderer.context.textAlign = this.params.textAlign;
+    renderer.context.textBaseline = this.params.textBaseline || "middle";
+    renderer.context.fillStyle = this.params.color;
+
+    const prevAlpha = renderer.context.globalAlpha;
+    renderer.context.globalAlpha = this.params.opacity;
+
+    if (this.params.backgroundColor) {
+      const metrics = renderer.context.measureText(this.params.content);
+      const ascent =
+        metrics.actualBoundingBoxAscent ?? this.params.fontSize * 0.8;
+      const descent =
+        metrics.actualBoundingBoxDescent ?? this.params.fontSize * 0.2;
+      const textW = metrics.width;
+      const textH = ascent + descent;
+      const padX = 8;
+      const padY = 4;
+
+      renderer.context.fillStyle = this.params.backgroundColor;
+      let bgLeft = -textW / 2;
+      if (renderer.context.textAlign === "left") bgLeft = 0;
+      if (renderer.context.textAlign === "right") bgLeft = -textW;
+
+      renderer.context.fillRect(
+        bgLeft - padX,
+        -textH / 2 - padY,
+        textW + padX * 2,
+        textH + padY * 2,
+      );
+
+      renderer.context.fillStyle = this.params.color;
+    }
+
+    renderer.context.fillText(this.params.content, 0, 0);
+
+    renderer.context.globalAlpha = prevAlpha;
+    renderer.context.restore();
+  }
+}

apps/web/src/services/renderer/nodes/video-node.ts

@@ -0,0 +1,108 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+import { CanvasRenderer } from "../canvas-renderer";
+import { BaseNode } from "./base-node";
+
+const VIDEO_EPSILON = 1 / 1000;
+
+export interface BaseMediaNodeParams {
+  file: File;
+  duration: number;
+  timeOffset: number;
+  trimStart: number;
+  trimEnd: number;
+  x?: number;
+  y?: number;
+  width?: number;
+  height?: number;
+  opacity?: number;
+}
+
+export type VideoNodeParams = BaseMediaNodeParams;
+
+export class VideoNode extends BaseNode<VideoNodeParams> {
+  private sink?: VideoSampleSink;
+  private readyPromise: Promise<void>;
+
+  constructor(params: VideoNodeParams) {
+    super(params);
+    this.readyPromise = this.load();
+  }
+
+  private async load() {
+    const input = new Input({
+      source: new BlobSource(this.params.file),
+      formats: ALL_FORMATS,
+    });
+
+    const videoTrack = await input.getPrimaryVideoTrack();
+    if (!videoTrack) {
+      throw new Error("No video track found");
+    }
+
+    if (!(await videoTrack.canDecode())) {
+      throw new Error("Unable to decode the video track.");
+    }
+
+    this.sink = new VideoSampleSink(videoTrack);
+  }
+
+  private getVideoTime(time: number) {
+    return time - this.params.timeOffset + this.params.trimStart;
+  }
+
+  private isInRange(time: number) {
+    const videoTime = this.getVideoTime(time);
+    return (
+      videoTime >= this.params.trimStart - VIDEO_EPSILON &&
+      videoTime < this.params.duration - this.params.trimEnd
+    );
+  }
+
+  async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
+    await super.render({ renderer, time });
+
+    if (!this.isInRange(time)) {
+      return;
+    }
+
+    await this.readyPromise;
+
+    if (!this.sink) {
+      throw new Error("Sink not initialized");
+    }
+
+    const videoTime = this.getVideoTime(time);
+    const sample = await this.sink.getSample(videoTime);
+
+    if (sample) {
+      try {
+        renderer.context.save();
+
+        if (this.params.opacity !== undefined) {
+          renderer.context.globalAlpha = this.params.opacity;
+        }
+
+        if (
+          this.params.x !== undefined &&
+          this.params.y !== undefined &&
+          this.params.width !== undefined &&
+          this.params.height !== undefined
+        ) {
+          sample.draw(
+            renderer.context,
+            this.params.x,
+            this.params.y,
+            this.params.width,
+            this.params.height,
+          );
+        } else {
+          sample.draw(renderer.context, 0, 0, renderer.width, renderer.height);
+        }
+
+        renderer.context.restore();
+      } finally {
+        sample.close();
+      }
+    }
+  }
+}

apps/web/src/services/renderer/scene-builder.ts

@@ -0,0 +1,102 @@
+import { type TimelineTrack } from "@/types/timeline";
+import { type MediaFile } from "@/types/media";
+import { RootNode } from "./nodes/root-node";
+import { VideoNode } from "./nodes/video-node";
+import { ImageNode } from "./nodes/image-node";
+import { TextNode } from "./nodes/text-node";
+import { ColorNode } from "./nodes/color-node";
+import { BlurBackgroundNode } from "./nodes/blur-background-node";
+import { TBackgroundType } from "@/types/project";
+import { DEFAULT_BLUR_INTENSITY } from "@/constants/editor-constants";
+
+export type BuildSceneParams = {
+  canvasSize: { width: number; height: number };
+  tracks: TimelineTrack[];
+  mediaFiles: MediaFile[];
+  duration: number;
+  backgroundColor?: string;
+  backgroundType?: TBackgroundType;
+  blurIntensity?: number;
+};
+
+export function buildScene(params: BuildSceneParams) {
+  const {
+    tracks,
+    mediaFiles,
+    duration,
+    canvasSize,
+    backgroundColor,
+    backgroundType,
+    blurIntensity,
+  } = params;
+
+  const rootNode = new RootNode({ duration });
+  const mediaMap = new Map(mediaFiles.map((m) => [m.id, m]));
+
+  const elements = tracks
+    .slice()
+    .reverse()
+    .filter((track) => !track.muted)
+    .flatMap((track) => track.elements);
+
+  const contentNodes = [];
+
+  for (const element of elements) {
+    if (element.type === "media") {
+      const media = mediaMap.get(element.mediaId);
+      if (media && media.file) {
+        if (media.type === "video") {
+          contentNodes.push(
+            new VideoNode({
+              file: media.file,
+              duration: element.duration,
+              timeOffset: element.startTime,
+              trimStart: element.trimStart,
+              trimEnd: element.trimEnd,
+            }),
+          );
+        } else if (media.type === "image") {
+          contentNodes.push(
+            new ImageNode({
+              file: media.file,
+              duration: element.duration,
+              timeOffset: element.startTime,
+              trimStart: element.trimStart,
+              trimEnd: element.trimEnd,
+            }),
+          );
+        }
+        // TODO: Add AudioNode for audio files
+      }
+    }
+
+    if (element.type === "text") {
+      const textElement = element;
+      contentNodes.push(
+        new TextNode({
+          ...textElement,
+          x: textElement.x + canvasSize.width / 2,
+          y: textElement.y + canvasSize.height / 2,
+          textBaseline: "middle",
+        }),
+      );
+    }
+  }
+
+  if (backgroundType === "blur") {
+    rootNode.add(
+      new BlurBackgroundNode({
+        blurIntensity: blurIntensity ?? DEFAULT_BLUR_INTENSITY,
+        contentNodes,
+      }),
+    );
+  } else if (backgroundColor && backgroundColor !== "transparent") {
+    rootNode.add(new ColorNode({ color: backgroundColor }));
+  }
+
+  for (const node of contentNodes) {
+    rootNode.add(node);
+  }
+
+  return rootNode;
+}

apps/web/src/services/renderer/scene-exporter.ts

@@ -0,0 +1,143 @@
+import EventEmitter from "eventemitter3";
+
+import {
+  Output,
+  Mp4OutputFormat,
+  WebMOutputFormat,
+  BufferTarget,
+  CanvasSource,
+  AudioBufferSource,
+  QUALITY_LOW,
+  QUALITY_MEDIUM,
+  QUALITY_HIGH,
+  QUALITY_VERY_HIGH,
+} from "mediabunny";
+import { RootNode } from "./nodes/root-node";
+import { CanvasRenderer } from "./canvas-renderer";
+
+export type ExportFormat = "mp4" | "webm";
+export type ExportQuality = "low" | "medium" | "high" | "very_high";
+
+type ExportParams = {
+  width: number;
+  height: number;
+  fps: number;
+  format: ExportFormat;
+  quality: ExportQuality;
+  includeAudio?: boolean;
+  audioBuffer?: AudioBuffer;
+};
+
+const qualityMap = {
+  low: QUALITY_LOW,
+  medium: QUALITY_MEDIUM,
+  high: QUALITY_HIGH,
+  very_high: QUALITY_VERY_HIGH,
+};
+
+export type SceneExporterEvents = {
+  progress: [progress: number];
+  complete: [buffer: ArrayBuffer];
+  error: [error: Error];
+  cancelled: [];
+};
+
+export class SceneExporter extends EventEmitter<SceneExporterEvents> {
+  private renderer: CanvasRenderer;
+  private format: ExportFormat;
+  private quality: ExportQuality;
+  private includeAudio: boolean;
+  private audioBuffer?: AudioBuffer;
+
+  private cancelled = false;
+
+  constructor(params: ExportParams) {
+    super();
+    this.renderer = new CanvasRenderer({
+      width: params.width,
+      height: params.height,
+      fps: params.fps,
+    });
+
+    this.format = params.format;
+    this.quality = params.quality;
+    this.includeAudio = params.includeAudio ?? false;
+    this.audioBuffer = params.audioBuffer;
+  }
+
+  cancel() {
+    this.cancelled = true;
+  }
+
+  async export(rootNode: RootNode) {
+    const { fps } = this.renderer;
+    const frameCount = Math.ceil(rootNode.duration * fps);
+
+    const outputFormat =
+      this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
+
+    const output = new Output({
+      format: outputFormat,
+      target: new BufferTarget(),
+    });
+
+    const videoSource = new CanvasSource(this.renderer.canvas as any, {
+      codec: this.format === "webm" ? "vp9" : "avc",
+      bitrate: qualityMap[this.quality],
+    });
+
+    output.addVideoTrack(videoSource, { frameRate: fps });
+
+    // Add audio track if requested
+    let audioSource: AudioBufferSource | null = null;
+    if (this.includeAudio && this.audioBuffer) {
+      audioSource = new AudioBufferSource({
+        codec: this.format === "webm" ? "opus" : "aac",
+        bitrate: qualityMap[this.quality],
+      });
+      output.addAudioTrack(audioSource);
+    }
+
+    await output.start();
+
+    // Add audio data after starting
+    if (audioSource && this.audioBuffer) {
+      await audioSource.add(this.audioBuffer);
+      audioSource.close();
+    }
+
+    // Render video frames
+    for (let i = 0; i < frameCount; i++) {
+      if (this.cancelled) {
+        await output.cancel();
+        this.emit("cancelled");
+        return null;
+      }
+
+      const time = i / fps;
+      await this.renderer.render({ node: rootNode, time });
+      await videoSource.add(time, 1 / fps);
+
+      this.emit("progress", i / frameCount);
+    }
+
+    if (this.cancelled) {
+      await output.cancel();
+      this.emit("cancelled");
+      return null;
+    }
+
+    videoSource.close();
+    await output.finalize();
+    this.emit("progress", 1);
+
+    const buffer = output.target.buffer;
+    if (!buffer) {
+      this.emit("error", new Error("Failed to export video"));
+      return null;
+    }
+
+    this.emit("complete", buffer);
+    return buffer;
+  }
+}