External/OpenCut
1
0
Fork 0
mirror of https://github.com/OpenCut-app/OpenCut.git synced 2026-07-13 21:52:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

shipping

Browse Source
This commit is contained in:
Maze Winther
2026-01-21 00:59:03 +01:00
parent 7f50de14fc
commit e1d82eca09
28 changed files with 1223 additions and 1009 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/package.json
+1
View File
@@ -21,6 +21,7 @@
"@ffmpeg/util": "^0.12.2",
"@hello-pangea/dnd": "^18.0.1",
"@hookform/resolvers": "^3.9.1",
"@huggingface/transformers": "^3.8.1",
"@opencut/env": "workspace:*",
"@opencut/ui": "workspace:*",
"@radix-ui/react-dialog": "^1.1.15",
apps/web/src/app/api/get-upload-url/route.ts
-119
View File
@@ -1,119 +0,0 @@
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { AwsClient } from "aws4fetch";
import { nanoid } from "nanoid";
import { webEnv } from "@opencut/env/web";
import { checkRateLimit } from "@/lib/rate-limit";
import { isTranscriptionConfigured } from "@/lib/transcription-utils";
const uploadRequestSchema = z.object({
fileExtension: z.enum(["wav", "mp3", "m4a", "flac"], {
errorMap: () => ({
message: "File extension must be wav, mp3, m4a, or flac",
}),
}),
});
const apiResponseSchema = z.object({
uploadUrl: z.string().url(),
fileName: z.string().min(1),
});
export async function POST(request: NextRequest) {
try {
const { limited } = await checkRateLimit({ request });
if (limited) {
return NextResponse.json({ error: "Too many requests" }, { status: 429 });
}
const transcriptionCheck = isTranscriptionConfigured();
if (!transcriptionCheck.configured) {
console.error(
"Missing environment variables:",
JSON.stringify(transcriptionCheck.missingVars)
);
return NextResponse.json(
{
error: "Transcription not configured",
message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`,
},
{ status: 503 }
);
}
const rawBody = await request.json().catch(() => null);
if (!rawBody) {
return NextResponse.json(
{ error: "Invalid JSON in request body" },
{ status: 400 }
);
}
const validationResult = uploadRequestSchema.safeParse(rawBody);
if (!validationResult.success) {
return NextResponse.json(
{
error: "Invalid request parameters",
details: validationResult.error.flatten().fieldErrors,
},
{ status: 400 }
);
}
const { fileExtension } = validationResult.data;
const client = new AwsClient({
accessKeyId: webEnv.R2_ACCESS_KEY_ID,
secretAccessKey: webEnv.R2_SECRET_ACCESS_KEY,
});
const timestamp = Date.now();
const fileName = `audio/${timestamp}-${nanoid()}.${fileExtension}`;
const url = new URL(
`https://${webEnv.R2_BUCKET_NAME}.${webEnv.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com/${fileName}`
);
url.searchParams.set("X-Amz-Expires", "3600"); // 1 hour expiry
const signed = await client.sign(new Request(url, { method: "PUT" }), {
aws: { signQuery: true },
});
if (!signed.url) {
throw new Error("Failed to generate presigned URL");
}
const responseData = {
uploadUrl: signed.url,
fileName,
};
const responseValidation = apiResponseSchema.safeParse(responseData);
if (!responseValidation.success) {
console.error(
"Invalid API response structure:",
responseValidation.error
);
return NextResponse.json(
{ error: "Internal response formatting error" },
{ status: 500 }
);
}
return NextResponse.json(responseValidation.data);
} catch (error) {
console.error("Error generating upload URL:", error);
return NextResponse.json(
{
error: "Failed to generate upload URL",
message:
error instanceof Error
? error.message
: "An unexpected error occurred",
},
{ status: 500 }
);
}
}
apps/web/src/app/api/transcribe/route.ts
-196
View File
@@ -1,196 +0,0 @@
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { webEnv } from "@opencut/env/web";
import { checkRateLimit } from "@/lib/rate-limit";
import { isTranscriptionConfigured } from "@/lib/transcription-utils";
const transcribeRequestSchema = z.object({
filename: z.string().min(1, "Filename is required"),
language: z.string().optional().default("auto"),
decryptionKey: z.string().min(1, "Decryption key is required").optional(),
iv: z.string().min(1, "IV is required").optional(),
});
const modalResponseSchema = z.object({
text: z.string(),
segments: z.array(
z.object({
id: z.number(),
seek: z.number(),
start: z.number(),
end: z.number(),
text: z.string(),
tokens: z.array(z.number()),
temperature: z.number(),
avg_logprob: z.number(),
compression_ratio: z.number(),
no_speech_prob: z.number(),
})
),
language: z.string(),
});
const apiResponseSchema = z.object({
text: z.string(),
segments: z.array(
z.object({
id: z.number(),
seek: z.number(),
start: z.number(),
end: z.number(),
text: z.string(),
tokens: z.array(z.number()),
temperature: z.number(),
avg_logprob: z.number(),
compression_ratio: z.number(),
no_speech_prob: z.number(),
})
),
language: z.string(),
});
function buildModalRequestBody({ filename, language, decryptionKey, iv }: {
filename: string;
language: string;
decryptionKey?: string;
iv?: string;
}) {
const requestBody: Record<string, string> = {
filename,
language,
};
if (decryptionKey && iv) {
requestBody.decryptionKey = decryptionKey;
requestBody.iv = iv;
}
return requestBody;
}
function parseModalError({ errorText }: { errorText: string }) {
let errorMessage = "Transcription service unavailable";
try {
const errorData = JSON.parse(errorText);
errorMessage = errorData.error || errorMessage;
} catch {}
return errorMessage;
}
export async function POST(request: NextRequest) {
try {
const { limited } = await checkRateLimit({ request });
if (limited) {
return NextResponse.json({ error: "Too many requests" }, { status: 429 });
}
const transcriptionCheck = isTranscriptionConfigured();
if (!transcriptionCheck.configured) {
console.error(
"Missing environment variables:",
JSON.stringify(transcriptionCheck.missingVars)
);
return NextResponse.json(
{
error: "Transcription not configured",
message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`,
},
{ status: 503 }
);
}
const rawBody = await request.json().catch(() => null);
if (!rawBody) {
return NextResponse.json(
{ error: "Invalid JSON in request body" },
{ status: 400 }
);
}
const validationResult = transcribeRequestSchema.safeParse(rawBody);
if (!validationResult.success) {
return NextResponse.json(
{
error: "Invalid request parameters",
details: validationResult.error.flatten().fieldErrors,
},
{ status: 400 }
);
}
const { filename, language, decryptionKey, iv } = validationResult.data;
const modalRequestBody = buildModalRequestBody({
filename,
language,
decryptionKey,
iv
});
const response = await fetch(webEnv.MODAL_TRANSCRIPTION_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(modalRequestBody),
});
if (!response.ok) {
const errorText = await response.text();
console.error("Modal API error:", response.status, errorText);
const errorMessage = parseModalError({ errorText });
return NextResponse.json(
{
error: errorMessage,
message: "Failed to process transcription request",
},
{ status: response.status >= 500 ? 502 : response.status }
);
}
const rawResult = await response.json();
const modalValidation = modalResponseSchema.safeParse(rawResult);
if (!modalValidation.success) {
console.error("Invalid Modal API response:", modalValidation.error);
return NextResponse.json(
{ error: "Invalid response from transcription service" },
{ status: 502 }
);
}
const result = modalValidation.data;
const responseData = {
text: result.text,
segments: result.segments,
language: result.language,
};
const responseValidation = apiResponseSchema.safeParse(responseData);
if (!responseValidation.success) {
console.error(
"Invalid API response structure:",
responseValidation.error
);
return NextResponse.json(
{ error: "Internal response formatting error" },
{ status: 500 }
);
}
return NextResponse.json(responseValidation.data);
} catch (error) {
console.error("Transcription API error:", error);
return NextResponse.json(
{
error: "Internal server error",
message: "An unexpected error occurred during transcription",
},
{ status: 500 }
);
}
}
apps/web/src/app/contributors/page.tsx
+2 -2
View File
@@ -40,7 +40,7 @@ async function getContributors(): Promise<Contributor[]> {
"User-Agent": "OpenCut-Web-App",
},
next: { revalidate: 600 }, // 10 minutes
} as RequestInit,
},
);
if (!response.ok) {
@@ -51,7 +51,7 @@ async function getContributors(): Promise<Contributor[]> {
const contributors = (await response.json()) as Contributor[];
const filteredContributors = contributors.filter(
(contributor: Contributor) => contributor.type === "User",
(contributor) => contributor.type === "User",
);
return filteredContributors;
apps/web/src/components/editor/assets-panel/tabbar.tsx
+2 -2
View File
@@ -2,7 +2,7 @@
import { cn } from "@/lib/utils";
import {
Tab,
TAB_KEYS,
tabs,
useAssetsPanelStore,
} from "../../../stores/assets-panel-store";
@@ -50,7 +50,7 @@ export function TabBar() {
ref={scrollRef}
className="scrollbar-hidden relative flex h-full w-full flex-col items-center justify-start gap-5 overflow-y-auto px-4 py-4"
>
{(Object.keys(tabs) as Tab[]).map((tabKey) => {
{TAB_KEYS.map((tabKey) => {
const tab = tabs[tabKey];
return (
<div
apps/web/src/components/editor/assets-panel/views/captions.tsx
+37 -194
View File
@@ -2,44 +2,34 @@ import { Button } from "@/components/ui/button";
import { PropertyGroup } from "../../properties-panel/property-item";
import { PanelBaseView as BaseView } from "@/components/editor/panel-base-view";
import { LanguageSelect } from "@/components/language-select";
import { useState, useRef, useEffect } from "react";
import { useState, useRef } from "react";
import { extractTimelineAudio } from "@/lib/media/mediabunny";
import { useEditor } from "@/hooks/use-editor";
import { DEFAULT_TEXT_ELEMENT } from "@/constants/text-constants";
import { LANGUAGES } from "@/constants/captions-constants";
import { Loader2, Shield, Trash2, Upload } from "lucide-react";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { TextElement } from "@/types/timeline";
interface TranscriptionSegment {
text: string;
start: number;
end: number;
}
const PRIVACY_DIALOG_KEY = "opencut-transcription-privacy-accepted";
import { Loader2 } from "lucide-react";
import type { TranscriptionProgress } from "@/types/transcription";
import { transcriptionService } from "@/services/transcription";
import { decodeAudioToFloat32 } from "@/lib/audio-utils";
import { buildCaptionChunks } from "@/lib/caption-utils";
export function Captions() {
const [selectedCountry, setSelectedCountry] = useState("auto");
const [selectedLanguage, setSelectedLanguage] = useState("auto");
const [isProcessing, setIsProcessing] = useState(false);
const [processingStep, setProcessingStep] = useState<string>("");
const [processingStep, setProcessingStep] = useState("");
const [error, setError] = useState<string | null>(null);
const [showPrivacyDialog, setShowPrivacyDialog] = useState(false);
const [hasAcceptedPrivacy, setHasAcceptedPrivacy] = useState(false);
const containerRef = useRef<HTMLDivElement>(null);
const editor = useEditor();
useEffect(() => {
const hasAccepted = localStorage.getItem(PRIVACY_DIALOG_KEY) === "true";
setHasAcceptedPrivacy(hasAccepted);
}, []);
const handleProgress = (progress: TranscriptionProgress) => {
if (progress.status === "loading-model") {
setProcessingStep(
`Loading model ${Math.round(progress.progress)}%`,
);
} else if (progress.status === "transcribing") {
setProcessingStep("Transcribing...");
}
};
const handleGenerateTranscript = async () => {
try {
@@ -47,107 +37,44 @@ export function Captions() {
setError(null);
setProcessingStep("Extracting audio...");
const audioBlob = await extractTimelineAudio();
setProcessingStep("Encrypting audio...");
const audioBuffer = await audioBlob.arrayBuffer();
const encryptionResult = await encryptWithRandomKey(audioBuffer);
const encryptedBlob = new Blob([encryptionResult.encryptedData]);
setProcessingStep("Uploading...");
const uploadResponse = await fetch("/api/get-upload-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ fileExtension: "wav" }),
const audioBlob = await extractTimelineAudio({
tracks: editor.timeline.getTracks(),
mediaAssets: editor.media.getAssets(),
totalDuration: editor.timeline.getTotalDuration(),
});
if (!uploadResponse.ok) {
const error = await uploadResponse.json();
throw new Error(error.message || "Failed to get upload URL");
}
setProcessingStep("Preparing audio...");
const { samples } = await decodeAudioToFloat32({ audioBlob });
const { uploadUrl, fileName } = await uploadResponse.json();
await fetch(uploadUrl, {
method: "PUT",
body: encryptedBlob,
const result = await transcriptionService.transcribe({
audioData: samples,
language: selectedLanguage === "auto" ? "auto" : selectedLanguage.toLowerCase(),
onProgress: handleProgress,
});
setProcessingStep("Transcribing...");
const transcriptionResponse = await fetch("/api/transcribe", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
filename: fileName,
language:
selectedCountry === "auto" ? "auto" : selectedCountry.toLowerCase(),
decryptionKey: arrayBufferToBase64(encryptionResult.key),
iv: arrayBufferToBase64(encryptionResult.iv),
}),
});
if (!transcriptionResponse.ok) {
const error = await transcriptionResponse.json();
throw new Error(error.message || "Transcription failed");
}
const { text, segments } = await transcriptionResponse.json();
const shortCaptions: Array<{
text: string;
startTime: number;
duration: number;
}> = [];
let globalEndTime = 0;
segments.forEach((segment: TranscriptionSegment) => {
const words = segment.text.trim().split(/\s+/);
const segmentDuration = segment.end - segment.start;
const wordsPerSecond = words.length / segmentDuration;
const chunks: string[] = [];
for (let i = 0; i < words.length; i += 3) {
chunks.push(words.slice(i, i + 3).join(" "));
}
let chunkStartTime = segment.start;
chunks.forEach((chunk) => {
const chunkWords = chunk.split(/\s+/).length;
const chunkDuration = Math.max(0.8, chunkWords / wordsPerSecond);
const adjustedStartTime = Math.max(chunkStartTime, globalEndTime);
shortCaptions.push({
text: chunk,
startTime: adjustedStartTime,
duration: chunkDuration,
});
globalEndTime = adjustedStartTime + chunkDuration;
chunkStartTime += chunkDuration;
});
});
setProcessingStep("Generating captions...");
const captionChunks = buildCaptionChunks({ segments: result.segments });
const captionTrackId = editor.timeline.addTrack({
type: "text",
index: 0,
});
shortCaptions.forEach((caption, index) => {
for (let i = 0; i < captionChunks.length; i++) {
const caption = captionChunks[i];
editor.timeline.insertElement({
placement: { mode: "explicit", trackId: captionTrackId },
element: {
...DEFAULT_TEXT_ELEMENT,
name: `Caption ${index + 1}`,
name: `Caption ${i + 1}`,
content: caption.text,
duration: caption.duration,
startTime: caption.startTime,
fontSize: 65,
fontWeight: "bold",
} as TextElement,
},
});
});
}
} catch (error) {
console.error("Transcription failed:", error);
setError(
@@ -166,8 +93,8 @@ export function Captions() {
>
<PropertyGroup title="Language">
<LanguageSelect
selectedCountry={selectedCountry}
onSelect={setSelectedCountry}
selectedCountry={selectedLanguage}
onSelect={setSelectedLanguage}
containerRef={containerRef}
languages={LANGUAGES}
/>
@@ -182,96 +109,12 @@ export function Captions() {
<Button
className="w-full"
onClick={() => {
if (hasAcceptedPrivacy) {
handleGenerateTranscript();
} else {
setShowPrivacyDialog(true);
}
}}
onClick={handleGenerateTranscript}
disabled={isProcessing}
>
{isProcessing && <Loader2 className="mr-1 animate-spin" />}
{isProcessing ? processingStep : "Generate transcript"}
</Button>
<Dialog open={showPrivacyDialog} onOpenChange={setShowPrivacyDialog}>
<DialogContent className="sm:max-w-md">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<Shield className="size-5" />
Audio processing notice
</DialogTitle>
<DialogDescription className="space-y-3">
<p>
To generate captions, we need to process your timeline audio
using speech-to-text technology.
</p>
<div className="space-y-2 pt-2">
<div className="flex items-start gap-2">
<Shield className="size-4 flex-shrink-0" />
<span className="text-sm">
Zero-knowledge encryption - we cannot decrypt your files
even if we wanted to
</span>
</div>
<div className="flex items-start gap-2">
<Shield className="size-4 flex-shrink-0" />
<span className="text-sm">
Encryption keys generated randomly in your browser, never
stored anywhere
</span>
</div>
<div className="flex items-start gap-2">
<Upload className="size-4 flex-shrink-0" />
<span className="text-sm">
Audio encrypted before upload - raw audio never leaves
your device
</span>
</div>
<div className="flex items-start gap-2">
<Trash2 className="size-4 flex-shrink-0" />
<span className="text-sm">
Everything permanently deleted within seconds after
transcription
</span>
</div>
</div>
<p className="text-muted-foreground text-xs">
<strong>True zero-knowledge privacy:</strong> Encryption keys
are generated randomly in your browser and never stored
anywhere. It's cryptographically impossible for us, our cloud
providers, or anyone else to decrypt your audio files.
</p>
</DialogDescription>
</DialogHeader>
<DialogFooter className="gap-2">
<Button
variant="outline"
onClick={() => setShowPrivacyDialog(false)}
disabled={isProcessing}
>
Cancel
</Button>
<Button
onClick={() => {
localStorage.setItem(PRIVACY_DIALOG_KEY, "true");
setHasAcceptedPrivacy(true);
setShowPrivacyDialog(false);
handleGenerateTranscript();
}}
disabled={isProcessing}
>
Continue & generate captions
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
</BaseView>
);
apps/web/src/components/editor/assets-panel/views/settings.tsx
+1 -3
View File
@@ -26,11 +26,9 @@ import Image from "next/image";
import { cn } from "@/lib/utils";
import { colors } from "@/data/colors/solid";
import { patternCraftGradients } from "@/data/colors/pattern-craft";
import { PipetteIcon, PlusIcon } from "lucide-react";
import { PipetteIcon } from "lucide-react";
import { useMemo, memo, useCallback } from "react";
import { syntaxUIGradients } from "@/data/colors/syntax-ui";
import { Button } from "@/components/ui/button";
import { Separator } from "@/components/ui/separator";
import { useEditor } from "@/hooks/use-editor";
import type { TProject } from "@/types/project";
apps/web/src/components/editor/assets-panel/views/stickers.tsx
+34 -24
View File
@@ -1,6 +1,7 @@
"use client";
import { useEffect, useState, useMemo } from "react";
import type { CSSProperties } from "react";
import { useStickersStore } from "@/stores/stickers-store";
import {
Loader2,
@@ -38,6 +39,10 @@ import type { StickerCategory } from "@/types/stickers";
import { STICKER_CATEGORIES } from "@/constants/stickers-constants";
import { useInfiniteScroll } from "@/hooks/use-infinite-scroll";
function isStickerCategory(value: string): value is StickerCategory {
return STICKER_CATEGORIES.includes(value as StickerCategory);
}
export function StickersView() {
const { selectedCategory, setSelectedCategory } = useStickersStore();
@@ -45,8 +50,8 @@ export function StickersView() {
<BaseView
value={selectedCategory}
onValueChange={(v) => {
if (STICKER_CATEGORIES.includes(v as StickerCategory)) {
setSelectedCategory({ category: v as StickerCategory });
if (isStickerCategory(v)) {
setSelectedCategory({ category: v });
}
}}
tabs={[
@@ -91,16 +96,21 @@ function StickerGrid({
addingSticker: string | null;
capSize?: boolean;
}) {
const gridStyle: CSSProperties & {
"--sticker-min": string;
"--sticker-max"?: string;
} = {
gridTemplateColumns: capSize
? "repeat(auto-fill, minmax(var(--sticker-min, 96px), var(--sticker-max, 160px)))"
: "repeat(auto-fit, minmax(var(--sticker-min, 96px), 1fr))",
"--sticker-min": "96px",
...(capSize ? { "--sticker-max": "160px" } : {}),
};
return (
<div
className="grid gap-2"
style={{
gridTemplateColumns: capSize
? "repeat(auto-fill, minmax(var(--sticker-min, 96px), var(--sticker-max, 160px)))"
: "repeat(auto-fit, minmax(var(--sticker-min, 96px), 1fr))",
["--sticker-min" as any]: "96px",
...(capSize ? ({ ["--sticker-max"]: "160px" } as any) : {}),
}}
style={gridStyle}
>
{icons.map((iconName) => (
<StickerItem
@@ -201,17 +211,17 @@ function StickersContentView({ category }: { category: StickerCategory }) {
const collection = collections[c.prefix];
return collection
? {
prefix: c.prefix,
name: c.name,
total: collection.total,
}
prefix: c.prefix,
name: c.name,
total: collection.total,
}
: null;
})
.filter(Boolean) as Array<{
prefix: string;
name: string;
total: number;
}>;
prefix: string;
name: string;
total: number;
}>;
}, [collections, category]);
const { scrollAreaRef, handleScroll } = useInfiniteScroll({
@@ -524,10 +534,10 @@ function StickerItem({
hostIndex === 0
? getIconSvgUrl(iconName, { width: 64, height: 64 })
: buildIconSvgUrl(
ICONIFY_HOSTS[Math.min(hostIndex, ICONIFY_HOSTS.length - 1)],
iconName,
{ width: 64, height: 64 },
)
ICONIFY_HOSTS[Math.min(hostIndex, ICONIFY_HOSTS.length - 1)],
iconName,
{ width: 64, height: 64 },
)
}
alt={displayName}
width={64}
@@ -536,9 +546,9 @@ function StickerItem({
style={
capSize
? {
maxWidth: "var(--sticker-max, 160px)",
maxHeight: "var(--sticker-max, 160px)",
}
maxWidth: "var(--sticker-max, 160px)",
maxHeight: "var(--sticker-max, 160px)",
}
: undefined
}
onError={() => {
apps/web/src/components/editor/export-button.tsx
+25 -7
View File
@@ -11,7 +11,13 @@ import { Checkbox } from "../ui/checkbox";
import { cn } from "@/lib/utils";
import { getExportMimeType, getExportFileExtension } from "@/lib/export";
import { Check, Copy, Download, RotateCcw, X } from "lucide-react";
import { ExportFormat, ExportQuality, ExportResult } from "@/types/export";
import {
EXPORT_FORMAT_VALUES,
EXPORT_QUALITY_VALUES,
ExportFormat,
ExportQuality,
ExportResult,
} from "@/types/export";
import { PropertyGroup } from "./properties-panel/property-item";
import { useEditor } from "@/hooks/use-editor";
import { DEFAULT_EXPORT_OPTIONS } from "@/constants/export-constants";
@@ -160,9 +166,11 @@ function ExportPopover({
>
<RadioGroup
value={format}
onValueChange={(value) =>
setFormat(value as ExportFormat)
}
onValueChange={(value) => {
if (isExportFormat(value)) {
setFormat(value);
}
}}
>
<div className="flex items-center space-x-2">
<RadioGroupItem value="mp4" id="mp4" />
@@ -186,9 +194,11 @@ function ExportPopover({
>
<RadioGroup
value={quality}
onValueChange={(value) =>
setQuality(value as ExportQuality)
}
onValueChange={(value) => {
if (isExportQuality(value)) {
setQuality(value);
}
}}
>
<div className="flex items-center space-x-2">
<RadioGroupItem value="low" id="low" />
@@ -267,6 +277,14 @@ function ExportPopover({
);
}
function isExportFormat(value: string): value is ExportFormat {
return EXPORT_FORMAT_VALUES.some((formatValue) => formatValue === value);
}
function isExportQuality(value: string): value is ExportQuality {
return EXPORT_QUALITY_VALUES.some((qualityValue) => qualityValue === value);
}
function ExportError({
error,
onRetry,
apps/web/src/components/editor/timeline/audio-waveform.tsx
+7 -28
View File
@@ -1,4 +1,4 @@
import React, { useEffect, useRef, useState } from "react";
import { useEffect, useRef, useState } from "react";
import WaveSurfer from "wavesurfer.js";
interface AudioWaveformProps {
@@ -39,12 +39,12 @@ function extractPeaks({
return peaks;
}
const AudioWaveform: React.FC<AudioWaveformProps> = ({
export function AudioWaveform({
audioUrl,
audioBuffer,
height = 32,
className = "",
}) => {
}: AudioWaveformProps) {
const waveformRef = useRef<HTMLDivElement>(null);
const wavesurfer = useRef<WaveSurfer | null>(null);
const [isLoading, setIsLoading] = useState(true);
@@ -58,14 +58,10 @@ const AudioWaveform: React.FC<AudioWaveformProps> = ({
if (!waveformRef.current || (!audioUrl && !audioBuffer)) return;
try {
// Clear any existing instance safely
if (ws) {
// Instead of immediately destroying, just set to null
// We'll destroy it outside this function
wavesurfer.current = null;
}
// Create a fresh instance
const newWaveSurfer = WaveSurfer.create({
container: waveformRef.current,
waveColor: "rgba(255, 255, 255, 0.6)",
@@ -78,20 +74,16 @@ const AudioWaveform: React.FC<AudioWaveformProps> = ({
interact: false,
});
// Assign to ref only if component is still mounted
if (mounted) {
wavesurfer.current = newWaveSurfer;
} else {
// Component unmounted during initialization, clean up
try {
newWaveSurfer.destroy();
} catch (e) {
// Ignore destroy errors
} catch {
}
return;
}
// Event listeners
newWaveSurfer.on("ready", () => {
if (mounted) {
setIsLoading(false);
@@ -122,48 +114,35 @@ const AudioWaveform: React.FC<AudioWaveformProps> = ({
}
};
// First safely destroy previous instance if it exists
if (ws) {
// Use this pattern to safely destroy the previous instance
const wsToDestroy = ws;
// Detach from ref immediately
wavesurfer.current = null;
// Wait a tick to destroy so any pending operations can complete
requestAnimationFrame(() => {
try {
wsToDestroy.destroy();
} catch (e) {
// Ignore errors during destroy
} catch {
}
// Only initialize new instance after destroying the old one
if (mounted) {
initWaveSurfer();
}
});
} else {
// No previous instance to clean up, initialize directly
initWaveSurfer();
}
return () => {
// Mark component as unmounted
mounted = false;
// Store reference to current wavesurfer instance
const wsToDestroy = wavesurfer.current;
// Immediately clear the ref to prevent accessing it after unmount
wavesurfer.current = null;
// If we have an instance to clean up, do it safely
if (wsToDestroy) {
// Delay destruction to avoid race conditions
requestAnimationFrame(() => {
try {
wsToDestroy.destroy();
} catch (e) {
// Ignore destroy errors - they're expected
} catch {
}
});
}
@@ -195,6 +174,6 @@ const AudioWaveform: React.FC<AudioWaveformProps> = ({
/>
</div>
);
};
}
export default AudioWaveform;
apps/web/src/constants/export-constants.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { ExportOptions } from "@/types/export";
export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {
export const DEFAULT_EXPORT_OPTIONS = {
format: "mp4",
quality: "high",
includeAudio: true,
};
} satisfies ExportOptions;
apps/web/src/constants/transcription-constants.ts
+36
View File
@@ -0,0 +1,36 @@
import type { TranscriptionModel, TranscriptionModelId } from "@/types/transcription";
export const TRANSCRIPTION_MODELS: TranscriptionModel[] = [
{
id: "whisper-tiny",
name: "Tiny",
huggingFaceId: "onnx-community/whisper-tiny",
description: "Fastest, lower accuracy",
},
{
id: "whisper-small",
name: "Small",
huggingFaceId: "onnx-community/whisper-small",
description: "Good balance of speed and accuracy",
},
{
id: "whisper-medium",
name: "Medium",
huggingFaceId: "onnx-community/whisper-medium",
description: "Higher accuracy, slower",
},
{
id: "whisper-large-v3-turbo",
name: "Large v3 Turbo",
huggingFaceId: "onnx-community/whisper-large-v3-turbo",
description: "Best accuracy, requires WebGPU for good performance",
},
];
export const DEFAULT_TRANSCRIPTION_MODEL: TranscriptionModelId = "whisper-small";
export const DEFAULT_CHUNK_LENGTH_SECONDS = 30;
export const DEFAULT_STRIDE_SECONDS = 5;
export const DEFAULT_WORDS_PER_CAPTION = 3;
export const MIN_CAPTION_DURATION_SECONDS = 0.8;
apps/web/src/core/managers/project-manager.ts
+2 -6
View File
@@ -384,12 +384,8 @@ export class ProjectManager {
const tracks = this.editor.timeline.getTracks();
const mediaAssets = this.editor.media.getAssets();
const allElements: TimelineElement[] = tracks.flatMap(
(track) => track.elements as TimelineElement[],
);
const sortedElements = allElements.sort(
(a, b) => a.startTime - b.startTime,
);
const allElements = tracks.flatMap((track): TimelineElement[] => track.elements);
const sortedElements = allElements.sort((a, b) => a.startTime - b.startTime);
const firstElement = sortedElements[0];
if (
apps/web/src/hooks/timeline/element/use-element-interaction.ts
+1 -1
View File
@@ -539,7 +539,7 @@ export function useElementInteraction({
const clickOffsetTime = getClickOffsetTime({
clientX: event.clientX,
elementRect: (
event.currentTarget as HTMLElement
event.currentTarget
).getBoundingClientRect(),
zoomLevel,
});
apps/web/src/hooks/timeline/use-timeline-drag-drop.ts
+1 -1
View File
@@ -50,7 +50,7 @@ export function useTimelineDragDrop({
if (dragData.type === "text") return "text";
if (dragData.type === "sticker") return "sticker";
if (dragData.type === "media") {
return dragData.mediaType as ElementType;
return dragData.mediaType;
}
return null;
},
apps/web/src/hooks/timeline/use-timeline-snapping.ts
+1 -1
View File
@@ -102,7 +102,7 @@ export function useTimelineSnapping({
return {
snappedTime: closestSnapPoint
? (closestSnapPoint as SnapPoint).time
? closestSnapPoint.time
: targetTime,
snapPoint: closestSnapPoint,
snapDistance: closestDistance,
apps/web/src/hooks/use-keyboard-shortcuts-help.ts
+7 -6
View File
@@ -52,18 +52,15 @@ export function useKeyboardShortcutsHelp() {
}
for (const [actionId, keys] of Object.entries(actionToKeys)) {
if (!Object.prototype.hasOwnProperty.call(ACTIONS, actionId)) {
continue;
}
if (!isAction(actionId)) continue;
const action = actionId as TAction;
const actionDef = ACTIONS[action];
const actionDef = ACTIONS[actionId];
result.push({
id: actionId,
keys,
description: actionDef.description,
category: actionDef.category,
action,
action: actionId,
});
}
@@ -79,3 +76,7 @@ export function useKeyboardShortcutsHelp() {
shortcuts,
};
}
function isAction(id: string): id is TAction {
return id in ACTIONS;
}
apps/web/src/lib/audio-utils.ts
+30
View File
@@ -23,6 +23,36 @@ export function createAudioContext(): AudioContext {
return new AudioContextConstructor();
}
export interface DecodedAudio {
samples: Float32Array;
sampleRate: number;
}
export async function decodeAudioToFloat32({
audioBlob,
}: {
audioBlob: Blob;
}): Promise<DecodedAudio> {
const audioContext = createAudioContext();
const arrayBuffer = await audioBlob.arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
// mix down to mono
const numChannels = audioBuffer.numberOfChannels;
const length = audioBuffer.length;
const samples = new Float32Array(length);
for (let i = 0; i < length; i++) {
let sum = 0;
for (let channel = 0; channel < numChannels; channel++) {
sum += audioBuffer.getChannelData(channel)[i];
}
samples[i] = sum / numChannels;
}
return { samples, sampleRate: audioBuffer.sampleRate };
}
export async function collectAudioElements({
tracks,
mediaAssets,
apps/web/src/lib/caption-utils.ts
+49
View File
@@ -0,0 +1,49 @@
import type { TranscriptionSegment, CaptionChunk } from "@/types/transcription";
import {
DEFAULT_WORDS_PER_CAPTION,
MIN_CAPTION_DURATION_SECONDS,
} from "@/constants/transcription-constants";
export function buildCaptionChunks({
segments,
wordsPerChunk = DEFAULT_WORDS_PER_CAPTION,
minDuration = MIN_CAPTION_DURATION_SECONDS,
}: {
segments: TranscriptionSegment[];
wordsPerChunk?: number;
minDuration?: number;
}): CaptionChunk[] {
const captions: CaptionChunk[] = [];
let globalEndTime = 0;
for (const segment of segments) {
const words = segment.text.trim().split(/\s+/);
if (words.length === 0 || (words.length === 1 && words[0] === "")) continue;
const segmentDuration = segment.end - segment.start;
const wordsPerSecond = words.length / segmentDuration;
const chunks: string[] = [];
for (let i = 0; i < words.length; i += wordsPerChunk) {
chunks.push(words.slice(i, i + wordsPerChunk).join(" "));
}
let chunkStartTime = segment.start;
for (const chunk of chunks) {
const chunkWords = chunk.split(/\s+/).length;
const chunkDuration = Math.max(minDuration, chunkWords / wordsPerSecond);
const adjustedStartTime = Math.max(chunkStartTime, globalEndTime);
captions.push({
text: chunk,
startTime: adjustedStartTime,
duration: chunkDuration,
});
globalEndTime = adjustedStartTime + chunkDuration;
chunkStartTime += chunkDuration;
}
}
return captions;
}
apps/web/src/lib/media/mediabunny.ts
+15 -12
View File
@@ -1,7 +1,8 @@
import { FFmpeg } from "@ffmpeg/ffmpeg";
import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
import { collectAudioMixSources } from "@/lib/audio-utils";
import { useEditor } from "@/hooks/use-editor";
import type { TimelineTrack } from "@/types/timeline";
import type { MediaAsset } from "@/types/assets";
let ffmpeg: FFmpeg | null = null;
@@ -50,12 +51,18 @@ export async function getVideoInfo({
// audio mixing for timeline - keeping ffmpeg for now due to complexity
// TODO: Replace with Mediabunny audio processing when implementing canvas preview
export const extractTimelineAudio = async (
onProgress?: (progress: number) => void,
): Promise<Blob> => {
// Create fresh FFmpeg instance for this operation
export const extractTimelineAudio = async ({
tracks,
mediaAssets,
totalDuration,
onProgress,
}: {
tracks: TimelineTrack[];
mediaAssets: MediaAsset[];
totalDuration: number;
onProgress?: (progress: number) => void;
}): Promise<Blob> => {
const ffmpeg = new FFmpeg();
const editor = useEditor();
try {
await ffmpeg.load();
@@ -64,10 +71,6 @@ export const extractTimelineAudio = async (
throw new Error("Unable to initialize audio processing. Please try again.");
}
const tracks = editor.timeline.getTracks();
const mediaAssets = editor.media.getAssets();
const totalDuration = editor.timeline.getTotalDuration();
if (totalDuration === 0) {
const emptyAudioData = new ArrayBuffer(44);
return new Blob([emptyAudioData], { type: "audio/wav" });
@@ -182,11 +185,11 @@ export const extractTimelineAudio = async (
for (const inputFile of inputFiles) {
try {
await ffmpeg.deleteFile(inputFile);
} catch (cleanupError) {}
} catch (cleanupError) { }
}
try {
await ffmpeg.deleteFile("timeline_audio.wav");
} catch (cleanupError) {}
} catch (cleanupError) { }
}
};
apps/web/src/lib/transcription-utils.ts
-13
View File
@@ -1,13 +0,0 @@
import { webEnv } from "@opencut/env/web";
export function isTranscriptionConfigured() {
const missingVars = [];
if (!webEnv.CLOUDFLARE_ACCOUNT_ID) missingVars.push("CLOUDFLARE_ACCOUNT_ID");
if (!webEnv.R2_ACCESS_KEY_ID) missingVars.push("R2_ACCESS_KEY_ID");
if (!webEnv.R2_SECRET_ACCESS_KEY) missingVars.push("R2_SECRET_ACCESS_KEY");
if (!webEnv.R2_BUCKET_NAME) missingVars.push("R2_BUCKET_NAME");
if (!webEnv.MODAL_TRANSCRIPTION_URL) missingVars.push("MODAL_TRANSCRIPTION_URL");
return { configured: missingVars.length === 0, missingVars };
}
apps/web/src/services/transcription/index.ts
+186
View File
@@ -0,0 +1,186 @@
import type {
TranscriptionResult,
TranscriptionProgress,
TranscriptionModelId,
} from "@/types/transcription";
import {
DEFAULT_TRANSCRIPTION_MODEL,
TRANSCRIPTION_MODELS,
} from "@/constants/transcription-constants";
import type { WorkerMessage, WorkerResponse } from "./worker";
type ProgressCallback = (progress: TranscriptionProgress) => void;
class TranscriptionService {
private worker: Worker | null = null;
private currentModelId: TranscriptionModelId | null = null;
private isInitialized = false;
private isInitializing = false;
async transcribe({
audioData,
language = "auto",
modelId = DEFAULT_TRANSCRIPTION_MODEL,
onProgress,
}: {
audioData: Float32Array;
language?: string;
modelId?: TranscriptionModelId;
onProgress?: ProgressCallback;
}): Promise<TranscriptionResult> {
await this.ensureWorker({ modelId, onProgress });
return new Promise((resolve, reject) => {
if (!this.worker) {
reject(new Error("Worker not initialized"));
return;
}
const handleMessage = (event: MessageEvent<WorkerResponse>) => {
const response = event.data;
switch (response.type) {
case "transcribe-progress":
onProgress?.({
status: "transcribing",
progress: response.progress,
message: "Transcribing audio...",
});
break;
case "transcribe-complete":
this.worker?.removeEventListener("message", handleMessage);
resolve({
text: response.text,
segments: response.segments,
language,
});
break;
case "transcribe-error":
this.worker?.removeEventListener("message", handleMessage);
reject(new Error(response.error));
break;
case "cancelled":
this.worker?.removeEventListener("message", handleMessage);
reject(new Error("Transcription cancelled"));
break;
}
};
this.worker.addEventListener("message", handleMessage);
this.worker.postMessage({
type: "transcribe",
audio: audioData,
language,
} satisfies WorkerMessage);
});
}
cancel() {
this.worker?.postMessage({ type: "cancel" } satisfies WorkerMessage);
}
private async ensureWorker({
modelId,
onProgress,
}: {
modelId: TranscriptionModelId;
onProgress?: ProgressCallback;
}): Promise<void> {
const needsNewModel = this.currentModelId !== modelId;
if (this.worker && this.isInitialized && !needsNewModel) {
return;
}
if (this.isInitializing && !needsNewModel) {
await this.waitForInit();
return;
}
this.terminate();
this.isInitializing = true;
this.isInitialized = false;
const model = TRANSCRIPTION_MODELS.find((m) => m.id === modelId);
if (!model) {
throw new Error(`Unknown model: ${modelId}`);
}
this.worker = new Worker(
new URL("./worker.ts", import.meta.url),
{ type: "module" }
);
return new Promise((resolve, reject) => {
if (!this.worker) {
reject(new Error("Failed to create worker"));
return;
}
const handleMessage = (event: MessageEvent<WorkerResponse>) => {
const response = event.data;
switch (response.type) {
case "init-progress":
onProgress?.({
status: "loading-model",
progress: response.progress,
message: `Loading ${model.name} model...`,
});
break;
case "init-complete":
this.worker?.removeEventListener("message", handleMessage);
this.isInitialized = true;
this.isInitializing = false;
this.currentModelId = modelId;
resolve();
break;
case "init-error":
this.worker?.removeEventListener("message", handleMessage);
this.isInitializing = false;
this.terminate();
reject(new Error(response.error));
break;
}
};
this.worker.addEventListener("message", handleMessage);
this.worker.postMessage({
type: "init",
modelId: model.huggingFaceId,
} satisfies WorkerMessage);
});
}
private waitForInit(): Promise<void> {
return new Promise((resolve) => {
const checkInit = () => {
if (this.isInitialized) {
resolve();
} else if (!this.isInitializing) {
resolve();
} else {
setTimeout(checkInit, 100);
}
};
checkInit();
});
}
terminate() {
this.worker?.terminate();
this.worker = null;
this.isInitialized = false;
this.isInitializing = false;
this.currentModelId = null;
}
}
export const transcriptionService = new TranscriptionService();
apps/web/src/services/transcription/worker.ts
+163
View File
@@ -0,0 +1,163 @@
import {
pipeline,
type AutomaticSpeechRecognitionPipeline,
type AutomaticSpeechRecognitionOutput,
} from "@huggingface/transformers";
import type { TranscriptionSegment } from "@/types/transcription";
import { DEFAULT_CHUNK_LENGTH_SECONDS, DEFAULT_STRIDE_SECONDS } from "@/constants/transcription-constants";
export type WorkerMessage =
| { type: "init"; modelId: string }
| { type: "transcribe"; audio: Float32Array; language: string }
| { type: "cancel" };
export type WorkerResponse =
| { type: "init-progress"; progress: number }
| { type: "init-complete" }
| { type: "init-error"; error: string }
| { type: "transcribe-progress"; progress: number }
| { type: "transcribe-complete"; text: string; segments: TranscriptionSegment[] }
| { type: "transcribe-error"; error: string }
| { type: "cancelled" };
let transcriber: AutomaticSpeechRecognitionPipeline | null = null;
let cancelled = false;
let lastReportedProgress = -1;
const fileBytes = new Map<string, { loaded: number; total: number }>();
self.onmessage = async (event: MessageEvent<WorkerMessage>) => {
const message = event.data;
switch (message.type) {
case "init":
await handleInit({ modelId: message.modelId });
break;
case "transcribe":
await handleTranscribe({ audio: message.audio, language: message.language });
break;
case "cancel":
cancelled = true;
self.postMessage({ type: "cancelled" } satisfies WorkerResponse);
break;
}
};
async function handleInit({ modelId }: { modelId: string }) {
lastReportedProgress = -1;
fileBytes.clear();
try {
transcriber = (await pipeline("automatic-speech-recognition", modelId, {
dtype: "q4",
device: "auto",
progress_callback: (progressInfo: {
status?: string;
file?: string;
loaded?: number;
total?: number;
}) => {
const file = progressInfo.file;
if (!file) return;
const loaded = progressInfo.loaded ?? 0;
const total = progressInfo.total ?? 0;
if (progressInfo.status === "progress" && total > 0) {
fileBytes.set(file, { loaded, total });
} else if (progressInfo.status === "done") {
const existing = fileBytes.get(file);
if (existing) {
fileBytes.set(file, { loaded: existing.total, total: existing.total });
}
}
// sum all bytes
let totalLoaded = 0;
let totalSize = 0;
for (const { loaded, total } of fileBytes.values()) {
totalLoaded += loaded;
totalSize += total;
}
if (totalSize === 0) return;
const overallProgress = (totalLoaded / totalSize) * 100;
const roundedProgress = Math.floor(overallProgress);
if (roundedProgress !== lastReportedProgress) {
lastReportedProgress = roundedProgress;
self.postMessage({
type: "init-progress",
progress: roundedProgress,
} satisfies WorkerResponse);
}
},
})) as unknown as AutomaticSpeechRecognitionPipeline;
self.postMessage({ type: "init-complete" } satisfies WorkerResponse);
} catch (error) {
self.postMessage({
type: "init-error",
error: error instanceof Error ? error.message : "Failed to load model",
} satisfies WorkerResponse);
}
}
async function handleTranscribe({
audio,
language,
}: {
audio: Float32Array;
language: string;
}) {
if (!transcriber) {
self.postMessage({
type: "transcribe-error",
error: "Model not initialized",
} satisfies WorkerResponse);
return;
}
cancelled = false;
try {
const rawResult = await transcriber(audio, {
chunk_length_s: DEFAULT_CHUNK_LENGTH_SECONDS,
stride_length_s: DEFAULT_STRIDE_SECONDS,
language: language === "auto" ? undefined : language,
return_timestamps: true,
});
if (cancelled) return;
const result: AutomaticSpeechRecognitionOutput = Array.isArray(rawResult)
? rawResult[0]
: rawResult;
const segments: TranscriptionSegment[] = [];
if (result.chunks) {
for (const chunk of result.chunks) {
if (chunk.timestamp && chunk.timestamp.length >= 2) {
segments.push({
text: chunk.text,
start: chunk.timestamp[0] ?? 0,
end: chunk.timestamp[1] ?? chunk.timestamp[0] ?? 0,
});
}
}
}
self.postMessage({
type: "transcribe-complete",
text: result.text,
segments,
} satisfies WorkerResponse);
} catch (error) {
if (cancelled) return;
self.postMessage({
type: "transcribe-error",
error: error instanceof Error ? error.message : "Transcription failed",
} satisfies WorkerResponse);
}
}
apps/web/src/stores/assets-panel-store.ts
+16 -13
View File
@@ -13,19 +13,22 @@ import {
} from "lucide-react";
import { create } from "zustand";
export type Tab =
| "media"
| "sounds"
| "text"
| "stickers"
| "effects"
| "transitions"
| "captions"
| "filters"
| "adjustment"
| "settings";
export const TAB_KEYS = [
"media",
"sounds",
"text",
"stickers",
"effects",
"transitions",
"captions",
"filters",
"adjustment",
"settings",
] as const;
export const tabs: { [key in Tab]: { icon: LucideIcon; label: string } } = {
export type Tab = (typeof TAB_KEYS)[number];
export const tabs = {
media: {
icon: VideoIcon,
label: "Media",
@@ -66,7 +69,7 @@ export const tabs: { [key in Tab]: { icon: LucideIcon; label: string } } = {
icon: SettingsIcon,
label: "Settings",
},
};
} satisfies Record<Tab, { icon: LucideIcon; label: string }>;
type MediaViewMode = "grid" | "list";
apps/web/src/types/export.ts
+11 -2
View File
@@ -1,5 +1,14 @@
export type ExportFormat = "mp4" | "webm";
export type ExportQuality = "low" | "medium" | "high" | "very_high";
export const EXPORT_QUALITY_VALUES = [
"low",
"medium",
"high",
"very_high",
] as const;
export const EXPORT_FORMAT_VALUES = ["mp4", "webm"] as const;
export type ExportFormat = (typeof EXPORT_FORMAT_VALUES)[number];
export type ExportQuality = (typeof EXPORT_QUALITY_VALUES)[number];
export interface ExportOptions {
format: ExportFormat;
apps/web/src/types/transcription.ts
+43
View File
@@ -0,0 +1,43 @@
export interface TranscriptionSegment {
text: string;
start: number;
end: number;
}
export interface TranscriptionResult {
text: string;
segments: TranscriptionSegment[];
language: string;
}
export type TranscriptionStatus =
| "idle"
| "loading-model"
| "transcribing"
| "complete"
| "error";
export interface TranscriptionProgress {
status: TranscriptionStatus;
progress: number;
message?: string;
}
export type TranscriptionModelId =
| "whisper-tiny"
| "whisper-small"
| "whisper-medium"
| "whisper-large-v3-turbo";
export interface TranscriptionModel {
id: TranscriptionModelId;
name: string;
huggingFaceId: string;
description: string;
}
export interface CaptionChunk {
text: string;
startTime: number;
duration: number;
}