feat: auto-captions

apps/web/src/lib/ffmpeg-utils.ts

@@ -1,5 +1,7 @@
 import { FFmpeg } from "@ffmpeg/ffmpeg";
 import { toBlobURL } from "@ffmpeg/util";
+import { useTimelineStore } from "@/stores/timeline-store";
+import { useMediaStore } from "@/stores/media-store";
 
 let ffmpeg: FFmpeg | null = null;
 
@@ -7,14 +9,7 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
   if (ffmpeg) return ffmpeg;
 
   ffmpeg = new FFmpeg();
-
-  // Use locally hosted files instead of CDN
-  const baseURL = "/ffmpeg";
-
-  await ffmpeg.load({
-    coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"),
-    wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"),
-  });
+  await ffmpeg.load(); // Use default config
 
   return ffmpeg;
 };
@@ -268,3 +263,206 @@ export const extractAudio = async (
 
   return blob;
 };
+
+export const extractTimelineAudio = async (
+  onProgress?: (progress: number) => void
+): Promise<Blob> => {
+  // Create fresh FFmpeg instance for this operation
+  const ffmpeg = new FFmpeg();
+
+  try {
+    await ffmpeg.load();
+  } catch (error) {
+    console.error("Failed to load fresh FFmpeg instance:", error);
+    throw new Error("Unable to initialize audio processing. Please try again.");
+  }
+
+  const timeline = useTimelineStore.getState();
+  const mediaStore = useMediaStore.getState();
+
+  const tracks = timeline.tracks;
+  const totalDuration = timeline.getTotalDuration();
+
+  if (totalDuration === 0) {
+    const emptyAudioData = new ArrayBuffer(44);
+    return new Blob([emptyAudioData], { type: "audio/wav" });
+  }
+
+  if (onProgress) {
+    ffmpeg.on("progress", ({ progress }) => {
+      onProgress(progress * 100);
+    });
+  }
+
+  const audioElements: Array<{
+    file: File;
+    startTime: number;
+    duration: number;
+    trimStart: number;
+    trimEnd: number;
+    trackMuted: boolean;
+  }> = [];
+
+  for (const track of tracks) {
+    if (track.muted) continue;
+
+    for (const element of track.elements) {
+      if (element.type === "media") {
+        const mediaItem = mediaStore.mediaItems.find(
+          (m) => m.id === element.mediaId
+        );
+        if (!mediaItem) continue;
+
+        if (mediaItem.type === "video" || mediaItem.type === "audio") {
+          audioElements.push({
+            file: mediaItem.file,
+            startTime: element.startTime,
+            duration: element.duration,
+            trimStart: element.trimStart,
+            trimEnd: element.trimEnd,
+            trackMuted: track.muted || false,
+          });
+        }
+      }
+    }
+  }
+
+  if (audioElements.length === 0) {
+    // Return silent audio if no audio elements
+    const silentDuration = Math.max(1, totalDuration); // At least 1 second
+    try {
+      const silentAudio = await generateSilentAudio(silentDuration);
+      return silentAudio;
+    } catch (error) {
+      console.error("Failed to generate silent audio:", error);
+      throw new Error("Unable to generate audio for empty timeline.");
+    }
+  }
+
+  // Create a complex filter to mix all audio sources
+  const inputFiles: string[] = [];
+  const filterInputs: string[] = [];
+
+  try {
+    for (let i = 0; i < audioElements.length; i++) {
+      const element = audioElements[i];
+      const inputName = `input_${i}.${element.file.name.split(".").pop()}`;
+      inputFiles.push(inputName);
+
+      try {
+        await ffmpeg.writeFile(
+          inputName,
+          new Uint8Array(await element.file.arrayBuffer())
+        );
+      } catch (error) {
+        console.error(`Failed to write file ${element.file.name}:`, error);
+        throw new Error(
+          `Unable to process file: ${element.file.name}. The file may be corrupted or in an unsupported format.`
+        );
+      }
+
+      const actualStart = element.trimStart;
+      const actualDuration =
+        element.duration - element.trimStart - element.trimEnd;
+
+      const filterName = `audio_${i}`;
+      filterInputs.push(
+        `[${i}:a]atrim=start=${actualStart}:duration=${actualDuration},asetpts=PTS-STARTPTS,adelay=${element.startTime * 1000}|${element.startTime * 1000}[${filterName}]`
+      );
+    }
+
+    const mixFilter =
+      audioElements.length === 1
+        ? `[audio_0]aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`
+        : `${filterInputs.map((_, i) => `[audio_${i}]`).join("")}amix=inputs=${audioElements.length}:duration=longest:dropout_transition=2,aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`;
+
+    const complexFilter = [...filterInputs, mixFilter].join(";");
+    const outputName = "timeline_audio.wav";
+
+    const ffmpegArgs = [
+      ...inputFiles.flatMap((name) => ["-i", name]),
+      "-filter_complex",
+      complexFilter,
+      "-map",
+      "[out]",
+      "-t",
+      totalDuration.toString(),
+      "-c:a",
+      "pcm_s16le",
+      "-ar",
+      "44100",
+      outputName,
+    ];
+
+    try {
+      await ffmpeg.exec(ffmpegArgs);
+    } catch (error) {
+      console.error("FFmpeg execution failed:", error);
+      throw new Error(
+        "Audio processing failed. Some audio files may be corrupted or incompatible."
+      );
+    }
+
+    const data = await ffmpeg.readFile(outputName);
+    const blob = new Blob([data], { type: "audio/wav" });
+
+    return blob;
+  } catch (error) {
+    for (const inputFile of inputFiles) {
+      try {
+        await ffmpeg.deleteFile(inputFile);
+      } catch (cleanupError) {
+        console.warn(`Failed to cleanup file ${inputFile}:`, cleanupError);
+      }
+    }
+    try {
+      await ffmpeg.deleteFile("timeline_audio.wav");
+    } catch (cleanupError) {
+      console.warn("Failed to cleanup output file:", cleanupError);
+    }
+
+    throw error;
+  } finally {
+    for (const inputFile of inputFiles) {
+      try {
+        await ffmpeg.deleteFile(inputFile);
+      } catch (cleanupError) {}
+    }
+    try {
+      await ffmpeg.deleteFile("timeline_audio.wav");
+    } catch (cleanupError) {}
+  }
+};
+
+const generateSilentAudio = async (durationSeconds: number): Promise<Blob> => {
+  const ffmpeg = await initFFmpeg();
+  const outputName = "silent.wav";
+
+  try {
+    await ffmpeg.exec([
+      "-f",
+      "lavfi",
+      "-i",
+      `anullsrc=channel_layout=stereo:sample_rate=44100`,
+      "-t",
+      durationSeconds.toString(),
+      "-c:a",
+      "pcm_s16le",
+      outputName,
+    ]);
+
+    const data = await ffmpeg.readFile(outputName);
+    const blob = new Blob([data], { type: "audio/wav" });
+
+    return blob;
+  } catch (error) {
+    console.error("Failed to generate silent audio:", error);
+    throw error;
+  } finally {
+    try {
+      await ffmpeg.deleteFile(outputName);
+    } catch (cleanupError) {
+      // Silent cleanup
+    }
+  }
+};

apps/web/src/lib/transcription-utils.ts

@@ -0,0 +1,13 @@
+import { env } from "@/env";
+
+export function isTranscriptionConfigured() {
+  const missingVars = [];
+
+  if (!env.CLOUDFLARE_ACCOUNT_ID) missingVars.push("CLOUDFLARE_ACCOUNT_ID");
+  if (!env.R2_ACCESS_KEY_ID) missingVars.push("R2_ACCESS_KEY_ID");
+  if (!env.R2_SECRET_ACCESS_KEY) missingVars.push("R2_SECRET_ACCESS_KEY");
+  if (!env.R2_BUCKET_NAME) missingVars.push("R2_BUCKET_NAME");
+  if (!env.MODAL_TRANSCRIPTION_URL) missingVars.push("MODAL_TRANSCRIPTION_URL");
+
+  return { configured: missingVars.length === 0, missingVars };
+}

apps/web/src/lib/zk-encryption.ts

@@ -0,0 +1,71 @@
+/**
+ * True zero-knowledge encryption utilities
+ * Keys are generated randomly in the browser and never derived from server secrets
+ */
+
+export interface ZeroKnowledgeEncryptionResult {
+  encryptedData: ArrayBuffer;
+  key: ArrayBuffer;
+  iv: ArrayBuffer;
+}
+
+/**
+ * Encrypt data with a randomly generated key (true zero-knowledge)
+ */
+export async function encryptWithRandomKey(
+  data: ArrayBuffer
+): Promise<ZeroKnowledgeEncryptionResult> {
+  // Generate a truly random 256-bit key
+  const key = crypto.getRandomValues(new Uint8Array(32));
+
+  // Generate random IV
+  const iv = crypto.getRandomValues(new Uint8Array(12));
+
+  // Import the key for encryption
+  const cryptoKey = await crypto.subtle.importKey(
+    "raw",
+    key,
+    { name: " " },
+    false,
+    ["encrypt"]
+  );
+
+  // Encrypt the data
+  const encryptedResult = await crypto.subtle.encrypt(
+    { name: "AES-GCM", iv },
+    cryptoKey,
+    data
+  );
+
+  // For AES-GCM, we need to append the authentication tag
+  // The encrypted result contains both ciphertext and tag
+  return {
+    encryptedData: encryptedResult,
+    key: key.buffer,
+    iv: iv.buffer,
+  };
+}
+
+/**
+ * Convert ArrayBuffer to base64 string for transmission
+ */
+export function arrayBufferToBase64(buffer: ArrayBuffer): string {
+  const bytes = new Uint8Array(buffer);
+  let binary = "";
+  for (let i = 0; i < bytes.byteLength; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Convert base64 string back to ArrayBuffer
+ */
+export function base64ToArrayBuffer(base64: string): ArrayBuffer {
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes.buffer;
+}