not complete

This commit is contained in:
Maze Winther
2026-02-02 01:47:45 +01:00
parent df84bb2388
commit 47bc479551
9 changed files with 1357 additions and 10 deletions
@@ -0,0 +1,80 @@
"use client";
import { useFilmstrip } from "@/hooks/timeline/use-filmstrip";
import { TIMELINE_CONSTANTS } from "@/constants/timeline-constants";
import { getThumbnailWidth } from "@/lib/timeline/filmstrip-utils";
interface FilmstripProps {
mediaId: string;
file: File | null;
duration: number;
trimStart: number;
trackHeight: number;
zoomLevel: number;
fallbackThumbnailUrl?: string;
}
export function Filmstrip({
mediaId,
file,
duration,
trimStart,
trackHeight,
zoomLevel,
fallbackThumbnailUrl,
}: FilmstripProps) {
const thumbnailWidth = getThumbnailWidth({ trackHeight });
const pixelsPerSecond = TIMELINE_CONSTANTS.PIXELS_PER_SECOND * zoomLevel;
const visibleStartTime = trimStart;
const visibleEndTime = trimStart + duration;
const { frames } = useFilmstrip({
mediaId,
file,
duration,
visibleStartTime,
visibleEndTime,
zoomLevel,
});
if (frames.length === 0 && fallbackThumbnailUrl) {
return (
<div className="absolute inset-0 flex items-center justify-center">
<div
className="absolute right-0 left-0"
style={{
backgroundImage: `url(${fallbackThumbnailUrl})`,
backgroundRepeat: "repeat-x",
backgroundSize: `${thumbnailWidth}px ${trackHeight}px`,
backgroundPosition: "left center",
pointerEvents: "none",
top: 0,
bottom: 0,
}}
/>
</div>
);
}
return (
<div className="absolute inset-0 overflow-hidden">
{frames.map((frame) => {
const left = (frame.timestamp - visibleStartTime) * pixelsPerSecond;
return (
<div
key={`${frame.mediaId}-${frame.timestamp}-${frame.tier}`}
className="absolute top-0 h-full bg-cover bg-center"
style={{
left: `${left}px`,
width: `${thumbnailWidth}px`,
backgroundImage: `url(${frame.dataUrl})`,
pointerEvents: "none",
}}
/>
);
})}
</div>
);
}
@@ -30,6 +30,7 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils";
import { getActionDefinition, type TAction, invokeAction } from "@/lib/actions";
import { useElementSelection } from "@/hooks/timeline/element/use-element-selection";
import Image from "next/image";
import { Filmstrip } from "./filmstrip";
import {
ScissorIcon,
Delete02Icon,
@@ -159,6 +160,7 @@ export function TimelineElement({
hasAudio={hasAudio}
isMuted={isMuted}
mediaAssets={mediaAssets}
zoomLevel={zoomLevel}
onElementClick={onElementClick}
onElementMouseDown={onElementMouseDown}
handleResizeStart={handleResizeStart}
@@ -166,7 +168,10 @@ export function TimelineElement({
</div>
</ContextMenuTrigger>
<ContextMenuContent className="z-200 w-64">
<ActionMenuItem action="split" icon={<HugeiconsIcon icon={ScissorIcon} />}>
<ActionMenuItem
action="split"
icon={<HugeiconsIcon icon={ScissorIcon} />}
>
Split
</ActionMenuItem>
<CopyMenuItem />
@@ -185,7 +190,10 @@ export function TimelineElement({
/>
)}
{selectedElements.length === 1 && (
<ActionMenuItem action="duplicate-selected" icon={<HugeiconsIcon icon={Copy01Icon} />}>
<ActionMenuItem
action="duplicate-selected"
icon={<HugeiconsIcon icon={Copy01Icon} />}
>
Duplicate
</ActionMenuItem>
)}
@@ -225,6 +233,7 @@ function ElementInner({
hasAudio,
isMuted,
mediaAssets,
zoomLevel,
onElementClick,
onElementMouseDown,
handleResizeStart,
@@ -236,6 +245,7 @@ function ElementInner({
hasAudio: boolean;
isMuted: boolean;
mediaAssets: MediaAsset[];
zoomLevel: number;
onElementClick: (e: React.MouseEvent, element: TimelineElementType) => void;
onElementMouseDown: (
e: React.MouseEvent,
@@ -267,6 +277,7 @@ function ElementInner({
track={track}
isSelected={isSelected}
mediaAssets={mediaAssets}
zoomLevel={zoomLevel}
/>
</div>
@@ -338,11 +349,13 @@ function ElementContent({
track,
isSelected,
mediaAssets,
zoomLevel,
}: {
element: TimelineElementType;
track: TimelineTrack;
isSelected: boolean;
mediaAssets: MediaAsset[];
zoomLevel: number;
}) {
if (element.type === "text") {
return (
@@ -408,14 +421,10 @@ function ElementContent({
);
}
if (
mediaAsset.type === "image" ||
(mediaAsset.type === "video" && mediaAsset.thumbnailUrl)
) {
if (mediaAsset.type === "image") {
const trackHeight = getTrackHeight({ type: track.type });
const tileWidth = trackHeight * (16 / 9);
const imageUrl =
mediaAsset.type === "image" ? mediaAsset.url : mediaAsset.thumbnailUrl;
const imageUrl = mediaAsset.url;
return (
<div className="flex size-full items-center justify-center">
@@ -439,6 +448,29 @@ function ElementContent({
);
}
if (mediaAsset.type === "video") {
const trackHeight = getTrackHeight({ type: track.type });
const trimStart = element.trimStart ?? 0;
return (
<div className="flex size-full items-center justify-center">
<div
className={`relative size-full ${isSelected ? "bg-primary" : "bg-transparent"}`}
>
<Filmstrip
mediaId={element.mediaId}
file={mediaAsset.file}
duration={element.duration}
trimStart={trimStart}
trackHeight={trackHeight}
zoomLevel={zoomLevel}
fallbackThumbnailUrl={mediaAsset.thumbnailUrl}
/>
</div>
</div>
);
}
return (
<span className="text-foreground/80 truncate text-xs">{element.name}</span>
);
@@ -446,7 +478,10 @@ function ElementContent({
function CopyMenuItem() {
return (
<ActionMenuItem action="copy-selected" icon={<HugeiconsIcon icon={Copy01Icon} />}>
<ActionMenuItem
action="copy-selected"
icon={<HugeiconsIcon icon={Copy01Icon} />}
>
Copy
</ActionMenuItem>
);
@@ -502,7 +537,10 @@ function VisibilityMenuItem({
};
return (
<ActionMenuItem action="toggle-elements-visibility-selected" icon={getIcon()}>
<ActionMenuItem
action="toggle-elements-visibility-selected"
icon={getIcon()}
>
{isHidden ? "Show" : "Hide"}
</ActionMenuItem>
);
@@ -3,6 +3,7 @@ import type { MediaAsset } from "@/types/assets";
import { storageService } from "@/services/storage/service";
import { generateUUID } from "@/utils/id";
import { videoCache } from "@/services/video-cache/service";
import { filmstripService } from "@/services/filmstrip/service";
import { hasMediaId } from "@/lib/timeline/element-utils";
export class MediaManager {
@@ -46,6 +47,7 @@ export class MediaManager {
const asset = this.assets.find((asset) => asset.id === id);
videoCache.clearVideo({ mediaId: id });
filmstripService.clearMedia({ mediaId: id });
if (asset?.url) {
URL.revokeObjectURL(asset.url);
@@ -124,6 +126,7 @@ export class MediaManager {
clearAllAssets(): void {
videoCache.clearAll();
filmstripService.destroy();
this.assets.forEach((asset) => {
if (asset.url) {
@@ -0,0 +1,146 @@
import { useEffect, useState, useRef } from "react";
import { filmstripService } from "@/services/filmstrip/service";
import type {
FilmstripFrame,
FilmstripStatus,
} from "@/services/filmstrip/types";
import {
getTierForZoom,
getTimestampsForRange,
} from "@/lib/timeline/filmstrip-utils";
interface UseFilmstripOptions {
mediaId: string;
file: File | null;
duration: number;
visibleStartTime: number;
visibleEndTime: number;
zoomLevel: number;
}
interface UseFilmstripReturn {
frames: FilmstripFrame[];
status: FilmstripStatus;
progress: number;
}
export function useFilmstrip({
mediaId,
file,
duration,
visibleStartTime,
visibleEndTime,
zoomLevel,
}: UseFilmstripOptions): UseFilmstripReturn {
const [frames, setFrames] = useState<FilmstripFrame[]>([]);
const [status, setStatus] = useState<FilmstripStatus>("idle");
const [progress, setProgress] = useState(0);
const tierRef = useRef<number>(0);
const requestedTimestampsRef = useRef<Set<number>>(new Set());
useEffect(() => {
if (!file || duration <= 0) {
setFrames([]);
setStatus("idle");
setProgress(0);
return;
}
const tier = getTierForZoom({ zoomLevel });
tierRef.current = tier;
const timestamps = getTimestampsForRange({
startTime: visibleStartTime,
endTime: visibleEndTime,
tier: tier as 0 | 1 | 2 | 3,
});
const cachedFrames = filmstripService.getFrames({
mediaId,
tier,
startTime: visibleStartTime,
endTime: visibleEndTime,
});
setFrames(cachedFrames);
const missingTimestamps = timestamps.filter(
(ts) => !cachedFrames.some((f) => f.timestamp === ts),
);
if (missingTimestamps.length === 0) {
setStatus("ready");
setProgress(100);
return;
}
setStatus("generating");
setProgress((cachedFrames.length / timestamps.length) * 100);
const newTimestamps = missingTimestamps.filter(
(ts) => !requestedTimestampsRef.current.has(ts),
);
if (newTimestamps.length > 0) {
for (const ts of newTimestamps) {
requestedTimestampsRef.current.add(ts);
}
filmstripService.requestFrames({
mediaId,
file,
timestamps: newTimestamps,
tier,
});
}
const handleFrame = ({
mediaId: frameMediaId,
}: {
mediaId: string;
timestamp: number;
dataUrl: string;
}) => {
if (frameMediaId !== mediaId) {
return;
}
if (tierRef.current !== tier) {
return;
}
const updatedFrames = filmstripService.getFrames({
mediaId,
tier,
startTime: visibleStartTime,
endTime: visibleEndTime,
});
setFrames(updatedFrames);
setProgress((updatedFrames.length / timestamps.length) * 100);
if (updatedFrames.length === timestamps.length) {
setStatus("ready");
setProgress(100);
}
};
const handleError = (errorMediaId: string, _error: string) => {
if (errorMediaId === mediaId) {
setStatus("error");
}
};
filmstripService.on("frame", handleFrame);
filmstripService.on("error", handleError);
return () => {
filmstripService.off("frame", handleFrame);
filmstripService.off("error", handleError);
filmstripService.cancelPending({ mediaId });
};
}, [mediaId, file, duration, visibleStartTime, visibleEndTime, zoomLevel]);
return { frames, status, progress };
}
@@ -0,0 +1,61 @@
export type FilmstripTier = 0 | 1 | 2 | 3;
const TIER_INTERVALS: Record<FilmstripTier, number> = {
0: 5,
1: 1,
2: 0.5,
3: 0.25,
} as const;
export function getTierForZoom({
zoomLevel,
}: {
zoomLevel: number;
}): FilmstripTier {
if (zoomLevel < 0.5) {
return 0;
}
if (zoomLevel < 2) {
return 1;
}
if (zoomLevel < 5) {
return 2;
}
return 3;
}
export function getIntervalForTier({ tier }: { tier: FilmstripTier }): number {
return TIER_INTERVALS[tier];
}
export function getTimestampsForRange({
startTime,
endTime,
tier,
}: {
startTime: number;
endTime: number;
tier: FilmstripTier;
}): number[] {
const interval = getIntervalForTier({ tier });
const timestamps: number[] = [];
let current = Math.floor(startTime / interval) * interval;
while (current <= endTime) {
if (current >= startTime) {
timestamps.push(current);
}
current += interval;
}
return timestamps;
}
export function getThumbnailWidth({
trackHeight,
}: {
trackHeight: number;
}): number {
return Math.round(trackHeight * (16 / 9));
}
+250
View File
@@ -0,0 +1,250 @@
import EventEmitter from "eventemitter3";
import type { FilmstripFrame, FilmstripGenerationProgress } from "./types";
import { getTimestampsForRange } from "@/lib/timeline/filmstrip-utils";
const MAX_CACHE_SIZE = 150;
type CacheKey = string;
function createCacheKey({
mediaId,
timestamp,
tier,
}: {
mediaId: string;
timestamp: number;
tier: number;
}): CacheKey {
return `${mediaId}:${tier}:${timestamp}`;
}
interface LRUCacheEntry {
key: CacheKey;
frame: FilmstripFrame;
}
export type FilmstripServiceEvents = {
frame: [FilmstripGenerationProgress];
error: [string, string];
};
export class FilmstripService extends EventEmitter<FilmstripServiceEvents> {
private cache = new Map<CacheKey, LRUCacheEntry>();
private cacheOrder: CacheKey[] = [];
private worker: Worker | null = null;
private pendingRequests = new Map<string, Map<number, number>>();
private getWorker(): Worker {
if (!this.worker) {
this.worker = new Worker(new URL("./worker.ts", import.meta.url), {
type: "module",
});
this.worker.onmessage = (event) => {
const response = event.data;
if (response.type === "frame") {
const pending = this.pendingRequests.get(response.mediaId);
const tier = pending?.get(response.timestamp);
if (tier !== undefined) {
this.handleFrame({
mediaId: response.mediaId,
timestamp: response.timestamp,
dataUrl: response.dataUrl,
tier,
});
}
} else if (response.type === "error") {
this.emit("error", response.mediaId, response.error);
} else if (response.type === "cancelled") {
this.pendingRequests.delete(response.mediaId);
}
};
}
return this.worker;
}
private handleFrame({
mediaId,
timestamp,
dataUrl,
tier,
}: {
mediaId: string;
timestamp: number;
dataUrl: string;
tier: number;
}): void {
const pending = this.pendingRequests.get(mediaId);
if (!pending || !pending.has(timestamp)) {
return;
}
pending.delete(timestamp);
if (pending.size === 0) {
this.pendingRequests.delete(mediaId);
}
const frame: FilmstripFrame = {
mediaId,
timestamp,
tier,
dataUrl,
};
this.setFrame({ frame });
this.emit("frame", { mediaId, timestamp, dataUrl });
}
private setFrame({ frame }: { frame: FilmstripFrame }): void {
const key = createCacheKey({
mediaId: frame.mediaId,
timestamp: frame.timestamp,
tier: frame.tier,
});
if (this.cache.has(key)) {
this.moveToEnd({ key });
return;
}
if (this.cache.size >= MAX_CACHE_SIZE) {
this.evictOldest();
}
this.cache.set(key, { key, frame });
this.cacheOrder.push(key);
}
private moveToEnd({ key }: { key: CacheKey }): void {
const index = this.cacheOrder.indexOf(key);
if (index !== -1) {
this.cacheOrder.splice(index, 1);
this.cacheOrder.push(key);
}
}
private evictOldest(): void {
const oldestKey = this.cacheOrder.shift();
if (oldestKey) {
this.cache.delete(oldestKey);
}
}
getFrames({
mediaId,
tier,
startTime,
endTime,
}: {
mediaId: string;
tier: number;
startTime: number;
endTime: number;
}): FilmstripFrame[] {
const timestamps = getTimestampsForRange({
startTime,
endTime,
tier: tier as 0 | 1 | 2 | 3,
});
const frames: FilmstripFrame[] = [];
for (const timestamp of timestamps) {
const key = createCacheKey({ mediaId, timestamp, tier });
const entry = this.cache.get(key);
if (entry) {
this.moveToEnd({ key });
frames.push(entry.frame);
}
}
return frames;
}
requestFrames({
mediaId,
file,
timestamps,
tier,
}: {
mediaId: string;
file: File;
timestamps: number[];
tier: number;
}): void {
if (timestamps.length === 0) {
return;
}
const pending = this.pendingRequests.get(mediaId) ?? new Map();
const newTimestamps = timestamps.filter((ts) => !pending.has(ts));
if (newTimestamps.length === 0) {
return;
}
for (const timestamp of newTimestamps) {
pending.set(timestamp, tier);
}
this.pendingRequests.set(mediaId, pending);
const worker = this.getWorker();
worker.postMessage({
type: "extract",
request: {
mediaId,
file,
timestamps: newTimestamps,
},
});
}
clearMedia({ mediaId }: { mediaId: string }): void {
const keysToDelete: CacheKey[] = [];
for (const [key, entry] of this.cache) {
if (entry.frame.mediaId === mediaId) {
keysToDelete.push(key);
}
}
for (const key of keysToDelete) {
this.cache.delete(key);
const index = this.cacheOrder.indexOf(key);
if (index !== -1) {
this.cacheOrder.splice(index, 1);
}
}
this.cancelPending({ mediaId });
}
cancelPending({ mediaId }: { mediaId: string }): void {
const pending = this.pendingRequests.get(mediaId);
if (pending) {
this.pendingRequests.delete(mediaId);
const worker = this.getWorker();
worker.postMessage({
type: "cancel",
mediaId,
});
}
}
destroy(): void {
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
this.cache.clear();
this.cacheOrder = [];
this.pendingRequests.clear();
}
}
export const filmstripService = new FilmstripService();
+25
View File
@@ -0,0 +1,25 @@
export interface FilmstripFrame {
mediaId: string;
timestamp: number;
tier: number;
dataUrl: string;
}
export interface FilmstripRequest {
mediaId: string;
file: File;
timestamps: number[];
}
export type FilmstripStatus = "idle" | "generating" | "ready" | "error";
export interface FilmstripGenerationProgress {
mediaId: string;
timestamp: number;
dataUrl: string;
}
export interface FilmstripGenerationComplete {
mediaId: string;
timestamps: number[];
}
+141
View File
@@ -0,0 +1,141 @@
import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
import type { FilmstripRequest } from "./types";
const THUMBNAIL_WIDTH = 106;
const THUMBNAIL_HEIGHT = 60;
const JPEG_QUALITY = 0.6;
export type WorkerMessage =
| { type: "extract"; request: FilmstripRequest }
| { type: "cancel"; mediaId: string };
export type WorkerResponse =
| {
type: "frame";
mediaId: string;
timestamp: number;
dataUrl: string;
}
| { type: "error"; mediaId: string; error: string }
| { type: "cancelled"; mediaId: string };
const cancelledMediaIds = new Set<string>();
self.onmessage = async (event: MessageEvent<WorkerMessage>) => {
const message = event.data;
switch (message.type) {
case "extract":
await handleExtract({ request: message.request });
break;
case "cancel":
cancelledMediaIds.add(message.mediaId);
self.postMessage({
type: "cancelled",
mediaId: message.mediaId,
} satisfies WorkerResponse);
break;
}
};
async function handleExtract({
request,
}: {
request: FilmstripRequest;
}): Promise<void> {
const { mediaId, file, timestamps } = request;
if (cancelledMediaIds.has(mediaId)) {
cancelledMediaIds.delete(mediaId);
return;
}
try {
const input = new Input({
source: new BlobSource(file),
formats: ALL_FORMATS,
});
const videoTrack = await input.getPrimaryVideoTrack();
if (!videoTrack) {
self.postMessage({
type: "error",
mediaId,
error: "No video track found",
} satisfies WorkerResponse);
return;
}
const canDecode = await videoTrack.canDecode();
if (!canDecode) {
self.postMessage({
type: "error",
mediaId,
error: "Video codec not supported for decoding",
} satisfies WorkerResponse);
return;
}
const sink = new VideoSampleSink(videoTrack);
for (const timestamp of timestamps) {
if (cancelledMediaIds.has(mediaId)) {
cancelledMediaIds.delete(mediaId);
return;
}
const frame = await sink.getSample(timestamp);
if (!frame) {
continue;
}
try {
const dataUrl = await renderFrameToDataUrl({ frame });
self.postMessage({
type: "frame",
mediaId,
timestamp,
dataUrl,
} satisfies WorkerResponse);
} finally {
frame.close();
}
}
} catch (error) {
self.postMessage({
type: "error",
mediaId,
error:
error instanceof Error ? error.message : "Failed to extract frames",
} satisfies WorkerResponse);
}
}
async function renderFrameToDataUrl({
frame,
}: {
frame: Awaited<ReturnType<VideoSampleSink["getSample"]>>;
}): Promise<string> {
const canvas = new OffscreenCanvas(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
const context = canvas.getContext("2d");
if (!context) {
throw new Error("Could not get canvas context");
}
frame.draw(context, 0, 0, THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
const blob = await canvas.convertToBlob({
type: "image/jpeg",
quality: JPEG_QUALITY,
});
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => {
resolve(reader.result as string);
};
reader.onerror = reject;
reader.readAsDataURL(blob);
});
}