mirror of
https://github.com/OpenCut-app/OpenCut.git
synced 2026-07-13 21:52:53 +02:00
feat: introduce WebGL effects system and Blur effect
This implements the foundational architecture for video effects, starting with a multi-pass WebGL rendering pipeline and a customizable Gaussian Blur effect. Key changes: - WebGL Engine: Added `raw-loader` for `.glsl` shaders, multi-pass framebuffer rendering, and live offscreen canvas previews. - Node Architecture: Replaced hardcoded background blur with `CompositeEffectNode` and added `EffectLayerNode` to apply effects to specific visual elements. - Timeline & DND: Added a new `effect` track type. Upgraded drag-and-drop to support dropping effects directly onto the timeline. Consolidated track constants into a cleaner `TRACK_CONFIG`. - UI/UX: Added an Effects tab in the assets panel with live previews. Added an Effect Properties panel with sliders and inputs for fine-tuning parameters. - Data Model: Added `sourceDuration` to video and audio elements, and wrote a v8 storage migration to update existing projects to the new schema. - Docs: Added `CHANGELOG.md` tracking v0.1.0 and v0.2.0, plus `docs/effects-renderer.md` to document the new WebGL pipeline.
This commit is contained in:
Maze Winther
@@ -6,12 +6,137 @@ import { ImageNode } from "./nodes/image-node";
|
||||
import { TextNode } from "./nodes/text-node";
|
||||
import { StickerNode } from "./nodes/sticker-node";
|
||||
import { ColorNode } from "./nodes/color-node";
|
||||
import { BlurBackgroundNode } from "./nodes/blur-background-node";
|
||||
import { CompositeEffectNode } from "./nodes/composite-effect-node";
|
||||
import { EffectLayerNode } from "./nodes/effect-layer-node";
|
||||
import type { BaseNode } from "./nodes/base-node";
|
||||
import type { TBackground, TCanvasSize } from "@/types/project";
|
||||
import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants";
|
||||
import { isMainTrack } from "@/lib/timeline";
|
||||
|
||||
const PREVIEW_MAX_IMAGE_SIZE = 2048;
|
||||
const BLUR_BACKGROUND_ZOOM_SCALE = 1.4;
|
||||
|
||||
function getVisibleSortedElements({
|
||||
track,
|
||||
}: {
|
||||
track: TimelineTrack;
|
||||
}) {
|
||||
return track.elements
|
||||
.filter((element) => !("hidden" in element && element.hidden))
|
||||
.slice()
|
||||
.sort((a, b) => {
|
||||
if (a.startTime !== b.startTime) return a.startTime - b.startTime;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
|
||||
function buildTrackNodes({
|
||||
tracks,
|
||||
mediaMap,
|
||||
canvasSize,
|
||||
isPreview,
|
||||
}: {
|
||||
tracks: TimelineTrack[];
|
||||
mediaMap: Map<string, MediaAsset>;
|
||||
canvasSize: TCanvasSize;
|
||||
isPreview?: boolean;
|
||||
}): BaseNode[] {
|
||||
const nodes: BaseNode[] = [];
|
||||
|
||||
for (const track of tracks) {
|
||||
const elements = getVisibleSortedElements({ track });
|
||||
|
||||
for (const element of elements) {
|
||||
if (element.type === "effect") {
|
||||
nodes.push(
|
||||
new EffectLayerNode({
|
||||
effectType: element.effectType,
|
||||
effectParams: element.params,
|
||||
timeOffset: element.startTime,
|
||||
duration: element.duration,
|
||||
}),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (element.type === "video" || element.type === "image") {
|
||||
const mediaAsset = mediaMap.get(element.mediaId);
|
||||
if (!mediaAsset?.file || !mediaAsset?.url) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mediaAsset.type === "video") {
|
||||
nodes.push(
|
||||
new VideoNode({
|
||||
mediaId: mediaAsset.id,
|
||||
url: mediaAsset.url,
|
||||
file: mediaAsset.file,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
effects: element.effects,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (mediaAsset.type === "image") {
|
||||
nodes.push(
|
||||
new ImageNode({
|
||||
url: mediaAsset.url,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
effects: element.effects,
|
||||
...(isPreview && {
|
||||
maxSourceSize: PREVIEW_MAX_IMAGE_SIZE,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (element.type === "text") {
|
||||
nodes.push(
|
||||
new TextNode({
|
||||
...element,
|
||||
canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 },
|
||||
canvasHeight: canvasSize.height,
|
||||
textBaseline: "middle",
|
||||
effects: element.effects,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (element.type === "sticker") {
|
||||
nodes.push(
|
||||
new StickerNode({
|
||||
stickerId: element.stickerId,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
effects: element.effects,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
export type BuildSceneParams = {
|
||||
canvasSize: TCanvasSize;
|
||||
@@ -22,9 +147,14 @@ export type BuildSceneParams = {
|
||||
isPreview?: boolean;
|
||||
};
|
||||
|
||||
export function buildScene(params: BuildSceneParams) {
|
||||
const { tracks, mediaAssets, duration, canvasSize, background } = params;
|
||||
|
||||
export function buildScene({
|
||||
canvasSize,
|
||||
tracks,
|
||||
mediaAssets,
|
||||
duration,
|
||||
background,
|
||||
isPreview,
|
||||
}: BuildSceneParams) {
|
||||
const rootNode = new RootNode({ duration });
|
||||
const mediaMap = new Map(mediaAssets.map((m) => [m.id, m]));
|
||||
|
||||
@@ -39,107 +169,33 @@ export function buildScene(params: BuildSceneParams) {
|
||||
|
||||
const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse();
|
||||
|
||||
const contentNodes = [];
|
||||
|
||||
for (const track of orderedTracksBottomToTop) {
|
||||
const elements = track.elements
|
||||
.filter((element) => !("hidden" in element && element.hidden))
|
||||
.slice()
|
||||
.sort((a, b) => {
|
||||
if (a.startTime !== b.startTime) return a.startTime - b.startTime;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
|
||||
for (const element of elements) {
|
||||
if (element.type === "video" || element.type === "image") {
|
||||
const mediaAsset = mediaMap.get(element.mediaId);
|
||||
if (!mediaAsset?.file || !mediaAsset?.url) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mediaAsset.type === "video") {
|
||||
contentNodes.push(
|
||||
new VideoNode({
|
||||
mediaId: mediaAsset.id,
|
||||
url: mediaAsset.url,
|
||||
file: mediaAsset.file,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (mediaAsset.type === "image") {
|
||||
contentNodes.push(
|
||||
new ImageNode({
|
||||
url: mediaAsset.url,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
...(params.isPreview && {
|
||||
maxSourceSize: PREVIEW_MAX_IMAGE_SIZE,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (element.type === "text") {
|
||||
contentNodes.push(
|
||||
new TextNode({
|
||||
...element,
|
||||
canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 },
|
||||
canvasHeight: canvasSize.height,
|
||||
textBaseline: "middle",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (element.type === "sticker") {
|
||||
contentNodes.push(
|
||||
new StickerNode({
|
||||
stickerId: element.stickerId,
|
||||
duration: element.duration,
|
||||
timeOffset: element.startTime,
|
||||
trimStart: element.trimStart,
|
||||
trimEnd: element.trimEnd,
|
||||
transform: element.transform,
|
||||
animations: element.animations,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const allNodes = buildTrackNodes({
|
||||
tracks: orderedTracksBottomToTop,
|
||||
mediaMap,
|
||||
canvasSize,
|
||||
isPreview,
|
||||
});
|
||||
|
||||
if (background.type === "blur") {
|
||||
rootNode.add(
|
||||
new BlurBackgroundNode({
|
||||
blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY,
|
||||
contentNodes,
|
||||
new CompositeEffectNode({
|
||||
contentNodes: allNodes.filter(
|
||||
(node) => !(node instanceof EffectLayerNode),
|
||||
),
|
||||
effectType: "blur",
|
||||
effectParams: {
|
||||
intensity:
|
||||
background.blurIntensity ?? DEFAULT_BLUR_INTENSITY,
|
||||
},
|
||||
scale: BLUR_BACKGROUND_ZOOM_SCALE,
|
||||
}),
|
||||
);
|
||||
for (const node of contentNodes) {
|
||||
rootNode.add(node);
|
||||
}
|
||||
} else {
|
||||
if (background.type === "color" && background.color !== "transparent") {
|
||||
rootNode.add(new ColorNode({ color: background.color }));
|
||||
}
|
||||
for (const node of contentNodes) {
|
||||
rootNode.add(node);
|
||||
}
|
||||
} else if (background.type === "color" && background.color !== "transparent") {
|
||||
rootNode.add(new ColorNode({ color: background.color }));
|
||||
}
|
||||
|
||||
for (const node of allNodes) {
|
||||
rootNode.add(node);
|
||||
}
|
||||
|
||||
return rootNode;
|
||||
|
||||
Reference in New Issue
Block a user