codebase overhaul (#697)

This commit is contained in:
Maze
2026-01-31 00:20:04 +01:00
committed by GitHub
parent 0173db9944
commit 7bf0984698
469 changed files with 36184 additions and 32931 deletions
@@ -0,0 +1,87 @@
import type { BaseNode } from "./nodes/base-node";
export type CanvasRendererParams = {
width: number;
height: number;
fps: number;
};
export class CanvasRenderer {
canvas: OffscreenCanvas | HTMLCanvasElement;
context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
width: number;
height: number;
fps: number;
constructor({ width, height, fps }: CanvasRendererParams) {
this.width = width;
this.height = height;
this.fps = fps;
try {
this.canvas = new OffscreenCanvas(width, height);
} catch {
this.canvas = document.createElement("canvas");
this.canvas.width = width;
this.canvas.height = height;
}
const context = this.canvas.getContext("2d");
if (!context) {
throw new Error("Failed to get canvas context");
}
this.context = context as
| OffscreenCanvasRenderingContext2D
| CanvasRenderingContext2D;
}
setSize({ width, height }: { width: number; height: number }) {
this.width = width;
this.height = height;
if (this.canvas instanceof OffscreenCanvas) {
this.canvas = new OffscreenCanvas(width, height);
} else {
this.canvas.width = width;
this.canvas.height = height;
}
const context = this.canvas.getContext("2d");
if (!context) {
throw new Error("Failed to get canvas context");
}
this.context = context as
| OffscreenCanvasRenderingContext2D
| CanvasRenderingContext2D;
}
private clear() {
this.context.fillStyle = "black";
this.context.fillRect(0, 0, this.canvas.width, this.canvas.height);
}
async render({ node, time }: { node: BaseNode; time: number }) {
this.clear();
await node.render({ renderer: this, time });
}
async renderToCanvas({
node,
time,
targetCanvas,
}: {
node: BaseNode;
time: number;
targetCanvas: HTMLCanvasElement;
}) {
await this.render({ node, time });
const ctx = targetCanvas.getContext("2d");
if (!ctx) {
throw new Error("Failed to get target canvas context");
}
ctx.drawImage(this.canvas, 0, 0, targetCanvas.width, targetCanvas.height);
}
}
@@ -0,0 +1,35 @@
import type { CanvasRenderer } from "../canvas-renderer";
export type BaseNodeParams = object | undefined;
export class BaseNode<Params extends BaseNodeParams = BaseNodeParams> {
params: Params;
constructor(params?: Params) {
this.params = params ?? ({} as Params);
}
children: BaseNode[] = [];
add(child: BaseNode) {
this.children.push(child);
return this;
}
remove(child: BaseNode) {
this.children = this.children.filter((c) => c !== child);
return this;
}
async render({
renderer,
time,
}: {
renderer: CanvasRenderer;
time: number;
}): Promise<void> {
for (const child of this.children) {
await child.render({ renderer, time });
}
}
}
@@ -0,0 +1,75 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
export type BlurBackgroundNodeParams = {
blurIntensity: number;
contentNodes: BaseNode[];
};
export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> {
private blurIntensity: number;
private contentNodes: BaseNode[];
constructor(params: BlurBackgroundNodeParams) {
super(params);
this.blurIntensity = params.blurIntensity;
this.contentNodes = params.contentNodes;
}
async render({
renderer,
time,
}: {
renderer: CanvasRenderer;
time: number;
}): Promise<void> {
let offscreen: OffscreenCanvas | HTMLCanvasElement;
let offscreenCtx:
| OffscreenCanvasRenderingContext2D
| CanvasRenderingContext2D;
try {
offscreen = new OffscreenCanvas(renderer.width, renderer.height);
const ctx = offscreen.getContext("2d");
if (!ctx) {
throw new Error("failed to get offscreen canvas context");
}
offscreenCtx = ctx;
} catch {
offscreen = document.createElement("canvas");
offscreen.width = renderer.width;
offscreen.height = renderer.height;
const ctx = offscreen.getContext("2d");
if (!ctx) {
throw new Error("failed to get canvas context");
}
offscreenCtx = ctx;
}
const originalContext = renderer.context;
renderer.context = offscreenCtx;
for (const node of this.contentNodes) {
await node.render({ renderer, time });
}
renderer.context = originalContext;
const zoomScale = 1.4;
const scaledWidth = renderer.width * zoomScale;
const scaledHeight = renderer.height * zoomScale;
const offsetX = (renderer.width - scaledWidth) / 2;
const offsetY = (renderer.height - scaledHeight) / 2;
renderer.context.save();
renderer.context.filter = `blur(${this.blurIntensity}px)`;
renderer.context.drawImage(
offscreen as CanvasImageSource,
offsetX,
offsetY,
scaledWidth,
scaledHeight,
);
renderer.context.restore();
}
}
@@ -0,0 +1,31 @@
import { drawCssBackground } from "@/lib/gradients";
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
export type ColorNodeParams = {
color: string;
};
export class ColorNode extends BaseNode<ColorNodeParams> {
private color: string;
constructor(params: ColorNodeParams) {
super(params);
this.color = params.color;
}
async render({ renderer }: { renderer: CanvasRenderer }) {
if (/gradient\(/i.test(this.color)) {
drawCssBackground({
ctx: renderer.context,
width: renderer.width,
height: renderer.height,
css: this.color,
});
return;
}
renderer.context.fillStyle = this.color;
renderer.context.fillRect(0, 0, renderer.width, renderer.height);
}
}
@@ -0,0 +1,93 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
import type { BaseMediaNodeParams } from "./video-node";
const IMAGE_EPSILON = 1 / 1000;
export type ImageNodeParams = BaseMediaNodeParams;
export class ImageNode extends BaseNode<ImageNodeParams> {
private image?: HTMLImageElement;
private readyPromise: Promise<void>;
constructor(params: ImageNodeParams) {
super(params);
this.readyPromise = this.load();
}
private async load() {
const image = new Image();
this.image = image;
const url = URL.createObjectURL(this.params.file);
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () => reject(new Error("Image load failed"));
image.src = url;
});
URL.revokeObjectURL(url);
}
private getImageTime(time: number) {
return time - this.params.timeOffset + this.params.trimStart;
}
private isInRange(time: number) {
const imageTime = this.getImageTime(time);
return (
imageTime >= this.params.trimStart - IMAGE_EPSILON &&
imageTime < this.params.trimStart + this.params.duration
);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange(time)) {
return;
}
await this.readyPromise;
if (!this.image) {
return;
}
renderer.context.save();
if (this.params.opacity !== undefined) {
renderer.context.globalAlpha = this.params.opacity;
}
if (
this.params.x !== undefined &&
this.params.y !== undefined &&
this.params.width !== undefined &&
this.params.height !== undefined
) {
renderer.context.drawImage(
this.image,
this.params.x,
this.params.y,
this.params.width,
this.params.height,
);
} else {
const mediaW = this.image.naturalWidth || renderer.width;
const mediaH = this.image.naturalHeight || renderer.height;
const containScale = Math.min(
renderer.width / mediaW,
renderer.height / mediaH,
);
const drawW = mediaW * containScale;
const drawH = mediaH * containScale;
const drawX = (renderer.width - drawW) / 2;
const drawY = (renderer.height - drawH) / 2;
renderer.context.drawImage(this.image, drawX, drawY, drawW, drawH);
}
renderer.context.restore();
}
}
@@ -0,0 +1,11 @@
import { BaseNode } from "./base-node";
export type RootNodeParams = {
duration: number;
};
export class RootNode extends BaseNode<RootNodeParams> {
get duration() {
return this.params.duration ?? 0;
}
}
@@ -0,0 +1,87 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode, type BaseNodeParams } from "./base-node";
import type { Transform } from "@/types/timeline";
const STICKER_EPSILON = 1 / 1000;
export type StickerNodeParams = BaseNodeParams & {
iconName: string;
duration: number;
timeOffset: number;
trimStart: number;
trimEnd: number;
transform: Transform;
opacity: number;
color?: string;
};
export class StickerNode extends BaseNode<StickerNodeParams> {
private image?: HTMLImageElement;
private readyPromise: Promise<void>;
constructor(params: StickerNodeParams) {
super(params);
this.readyPromise = this.load();
}
private async load() {
const image = new Image();
this.image = image;
const color = this.params.color
? `&color=${encodeURIComponent(this.params.color)}`
: "";
const url = `https://api.iconify.design/${this.params.iconName}.svg?width=200&height=200${color}`;
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () =>
reject(new Error(`Failed to load sticker: ${this.params.iconName}`));
image.src = url;
});
}
private getStickerTime(time: number) {
return time - this.params.timeOffset + this.params.trimStart;
}
private isInRange(time: number) {
const stickerTime = this.getStickerTime(time);
return (
stickerTime >= this.params.trimStart - STICKER_EPSILON &&
stickerTime < this.params.trimStart + this.params.duration
);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange(time)) {
return;
}
await this.readyPromise;
if (!this.image) {
return;
}
const { transform, opacity } = this.params;
const size = 200 * transform.scale;
const x = renderer.width / 2 + transform.position.x - size / 2;
const y = renderer.height / 2 + transform.position.y - size / 2;
renderer.context.save();
renderer.context.globalAlpha = opacity;
if (transform.rotate !== 0) {
const centerX = x + size / 2;
const centerY = y + size / 2;
renderer.context.translate(centerX, centerY);
renderer.context.rotate((transform.rotate * Math.PI) / 180);
renderer.context.translate(-centerX, -centerY);
}
renderer.context.drawImage(this.image, x, y, size, size);
renderer.context.restore();
}
}
@@ -0,0 +1,74 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
import type { TextElement } from "@/types/timeline";
export type TextNodeParams = TextElement & {
canvasCenter: { x: number; y: number };
textBaseline?: CanvasTextBaseline;
};
export class TextNode extends BaseNode<TextNodeParams> {
isInRange({ time }: { time: number }) {
return (
time >= this.params.startTime &&
time < this.params.startTime + this.params.duration
);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
if (!this.isInRange({ time })) {
return;
}
renderer.context.save();
const x = this.params.transform.position.x + this.params.canvasCenter.x;
const y = this.params.transform.position.y + this.params.canvasCenter.y;
renderer.context.translate(x, y);
if (this.params.transform.rotate) {
renderer.context.rotate((this.params.transform.rotate * Math.PI) / 180);
}
const fontWeight = this.params.fontWeight === "bold" ? "bold" : "normal";
const fontStyle = this.params.fontStyle === "italic" ? "italic" : "normal";
renderer.context.font = `${fontStyle} ${fontWeight} ${this.params.fontSize}px ${this.params.fontFamily}`;
renderer.context.textAlign = this.params.textAlign;
renderer.context.textBaseline = this.params.textBaseline || "middle";
renderer.context.fillStyle = this.params.color;
const prevAlpha = renderer.context.globalAlpha;
renderer.context.globalAlpha = this.params.opacity;
if (this.params.backgroundColor) {
const metrics = renderer.context.measureText(this.params.content);
const ascent =
metrics.actualBoundingBoxAscent ?? this.params.fontSize * 0.8;
const descent =
metrics.actualBoundingBoxDescent ?? this.params.fontSize * 0.2;
const textW = metrics.width;
const textH = ascent + descent;
const padX = 8;
const padY = 4;
renderer.context.fillStyle = this.params.backgroundColor;
let bgLeft = -textW / 2;
if (renderer.context.textAlign === "left") bgLeft = 0;
if (renderer.context.textAlign === "right") bgLeft = -textW;
renderer.context.fillRect(
bgLeft - padX,
-textH / 2 - padY,
textW + padX * 2,
textH + padY * 2,
);
renderer.context.fillStyle = this.params.color;
}
renderer.context.fillText(this.params.content, 0, 0);
renderer.context.globalAlpha = prevAlpha;
renderer.context.restore();
}
}
@@ -0,0 +1,84 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
import { videoCache } from "@/services/media/video-cache";
const VIDEO_EPSILON = 1 / 1000;
export interface BaseMediaNodeParams {
file: File;
duration: number;
timeOffset: number;
trimStart: number;
trimEnd: number;
x?: number;
y?: number;
width?: number;
height?: number;
opacity?: number;
}
export interface VideoNodeParams extends BaseMediaNodeParams {
mediaId: string;
}
export class VideoNode extends BaseNode<VideoNodeParams> {
private getVideoTime(time: number) {
return time - this.params.timeOffset + this.params.trimStart;
}
private isInRange(time: number) {
const videoTime = this.getVideoTime(time);
return (
videoTime >= this.params.trimStart - VIDEO_EPSILON &&
videoTime < this.params.trimStart + this.params.duration
);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange(time)) {
return;
}
const videoTime = this.getVideoTime(time);
const frame = await videoCache.getFrameAt({
mediaId: this.params.mediaId,
file: this.params.file,
time: videoTime,
});
if (frame) {
renderer.context.save();
if (this.params.opacity !== undefined) {
renderer.context.globalAlpha = this.params.opacity;
}
if (
this.params.x !== undefined &&
this.params.y !== undefined &&
this.params.width !== undefined &&
this.params.height !== undefined
) {
renderer.context.drawImage(
frame.canvas,
this.params.x,
this.params.y,
this.params.width,
this.params.height,
);
} else {
renderer.context.drawImage(
frame.canvas,
0,
0,
renderer.width,
renderer.height,
);
}
renderer.context.restore();
}
}
}
@@ -0,0 +1,130 @@
import type { TimelineTrack } from "@/types/timeline";
import type { MediaAsset } from "@/types/assets";
import { RootNode } from "./nodes/root-node";
import { VideoNode } from "./nodes/video-node";
import { ImageNode } from "./nodes/image-node";
import { TextNode } from "./nodes/text-node";
import { StickerNode } from "./nodes/sticker-node";
import { ColorNode } from "./nodes/color-node";
import { BlurBackgroundNode } from "./nodes/blur-background-node";
import type { TBackground, TCanvasSize } from "@/types/project";
import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants";
import { isMainTrack } from "@/lib/timeline";
export type BuildSceneParams = {
canvasSize: TCanvasSize;
tracks: TimelineTrack[];
mediaAssets: MediaAsset[];
duration: number;
background: TBackground;
};
export function buildScene(params: BuildSceneParams) {
const { tracks, mediaAssets, duration, canvasSize, background } = params;
const rootNode = new RootNode({ duration });
const mediaMap = new Map(mediaAssets.map((m) => [m.id, m]));
const visibleTracks = tracks.filter(
(track) => !("hidden" in track && track.hidden),
);
const mainTrack = visibleTracks.find((track) => isMainTrack(track)) ?? null;
const orderedTracksTopToBottom = [
...visibleTracks.filter((track) => !isMainTrack(track)),
...(mainTrack ? [mainTrack] : []),
];
const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse();
const contentNodes = [];
for (const track of orderedTracksBottomToTop) {
const elements = track.elements
.filter((element) => !("hidden" in element && element.hidden))
.slice()
.sort((a, b) => {
if (a.startTime !== b.startTime) return a.startTime - b.startTime;
return a.id.localeCompare(b.id);
});
for (const element of elements) {
if (element.type === "video" || element.type === "image") {
const mediaAsset = mediaMap.get(element.mediaId);
if (!mediaAsset?.file) {
continue;
}
if (mediaAsset.type === "video") {
contentNodes.push(
new VideoNode({
mediaId: mediaAsset.id,
file: mediaAsset.file,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
}),
);
}
if (mediaAsset.type === "image") {
contentNodes.push(
new ImageNode({
file: mediaAsset.file,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
}),
);
}
}
if (element.type === "text") {
contentNodes.push(
new TextNode({
...element,
canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 },
textBaseline: "middle",
}),
);
}
if (element.type === "sticker") {
contentNodes.push(
new StickerNode({
iconName: element.iconName,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
opacity: element.opacity,
color: element.color,
}),
);
}
}
}
if (background.type === "blur") {
rootNode.add(
new BlurBackgroundNode({
blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY,
contentNodes,
}),
);
for (const node of contentNodes) {
rootNode.add(node);
}
} else {
if (background.type === "color" && background.color !== "transparent") {
rootNode.add(new ColorNode({ color: background.color }));
}
for (const node of contentNodes) {
rootNode.add(node);
}
}
return rootNode;
}
@@ -0,0 +1,152 @@
import EventEmitter from "eventemitter3";
import {
Output,
Mp4OutputFormat,
WebMOutputFormat,
BufferTarget,
CanvasSource,
AudioBufferSource,
QUALITY_LOW,
QUALITY_MEDIUM,
QUALITY_HIGH,
QUALITY_VERY_HIGH,
} from "mediabunny";
import type { RootNode } from "./nodes/root-node";
import { CanvasRenderer } from "./canvas-renderer";
export type ExportFormat = "mp4" | "webm";
export type ExportQuality = "low" | "medium" | "high" | "very_high";
type ExportParams = {
width: number;
height: number;
fps: number;
format: ExportFormat;
quality: ExportQuality;
shouldIncludeAudio?: boolean;
audioBuffer?: AudioBuffer;
};
const qualityMap = {
low: QUALITY_LOW,
medium: QUALITY_MEDIUM,
high: QUALITY_HIGH,
very_high: QUALITY_VERY_HIGH,
};
export type SceneExporterEvents = {
progress: [progress: number];
complete: [buffer: ArrayBuffer];
error: [error: Error];
cancelled: [];
};
export class SceneExporter extends EventEmitter<SceneExporterEvents> {
private renderer: CanvasRenderer;
private format: ExportFormat;
private quality: ExportQuality;
private shouldIncludeAudio: boolean;
private audioBuffer?: AudioBuffer;
private isCancelled = false;
constructor({
width,
height,
fps,
format,
quality,
shouldIncludeAudio,
audioBuffer,
}: ExportParams) {
super();
this.renderer = new CanvasRenderer({
width,
height,
fps,
});
this.format = format;
this.quality = quality;
this.shouldIncludeAudio = shouldIncludeAudio ?? false;
this.audioBuffer = audioBuffer;
}
cancel(): void {
this.isCancelled = true;
}
async export({
rootNode,
}: {
rootNode: RootNode;
}): Promise<ArrayBuffer | null> {
const { fps } = this.renderer;
const frameCount = Math.ceil(rootNode.duration * fps);
const outputFormat =
this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
const output = new Output({
format: outputFormat,
target: new BufferTarget(),
});
const videoSource = new CanvasSource(this.renderer.canvas, {
codec: this.format === "webm" ? "vp9" : "avc",
bitrate: qualityMap[this.quality],
});
output.addVideoTrack(videoSource, { frameRate: fps });
let audioSource: AudioBufferSource | null = null;
if (this.shouldIncludeAudio && this.audioBuffer) {
audioSource = new AudioBufferSource({
codec: this.format === "webm" ? "opus" : "aac",
bitrate: qualityMap[this.quality],
});
output.addAudioTrack(audioSource);
}
await output.start();
if (audioSource && this.audioBuffer) {
await audioSource.add(this.audioBuffer);
audioSource.close();
}
for (let i = 0; i < frameCount; i++) {
if (this.isCancelled) {
await output.cancel();
this.emit("cancelled");
return null;
}
const time = i / fps;
await this.renderer.render({ node: rootNode, time });
await videoSource.add(time, 1 / fps);
this.emit("progress", i / frameCount);
}
if (this.isCancelled) {
await output.cancel();
this.emit("cancelled");
return null;
}
videoSource.close();
await output.finalize();
this.emit("progress", 1);
const buffer = output.target.buffer;
if (!buffer) {
this.emit("error", new Error("Failed to export video"));
return null;
}
this.emit("complete", buffer);
return buffer;
}
}