feat(tools): 2.0 phase 5 wave 5b - ai pool: ocr-pdf, transcription, background composites (5 tools) (#226)

apps/api/src/lib/subtitle-format.ts

@@ -0,0 +1,36 @@
+export interface TranscriptSegment {
+  startS: number;
+  endS: number;
+  text: string;
+}
+
+function pad(n: number, w: number): string {
+  return String(n).padStart(w, "0");
+}
+
+function stamp(totalS: number, msSep: string): string {
+  const ms = Math.max(0, Math.round(totalS * 1000));
+  const h = Math.floor(ms / 3600000);
+  const m = Math.floor((ms % 3600000) / 60000);
+  const s = Math.floor((ms % 60000) / 1000);
+  const frac = ms % 1000;
+  return `${pad(h, 2)}:${pad(m, 2)}:${pad(s, 2)}${msSep}${pad(frac, 3)}`;
+}
+
+/** SubRip: 1-based counters, comma millisecond separator. */
+export function toSrt(segments: TranscriptSegment[]): string {
+  return segments
+    .map(
+      (seg, i) =>
+        `${i + 1}\n${stamp(seg.startS, ",")} --> ${stamp(seg.endS, ",")}\n${seg.text.trim()}\n`,
+    )
+    .join("\n");
+}
+
+/** WebVTT: header + dot millisecond separator, no counters. */
+export function toVtt(segments: TranscriptSegment[]): string {
+  const body = segments
+    .map((seg) => `${stamp(seg.startS, ".")} --> ${stamp(seg.endS, ".")}\n${seg.text.trim()}\n`)
+    .join("\n");
+  return `WEBVTT\n\n${body}`;
+}

apps/api/src/routes/tools/auto-subtitles.ts

@@ -0,0 +1,169 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { transcribeAudio } from "@snapotter/ai";
+import { probeMedia, runFfmpeg } from "@snapotter/media-engine";
+import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
+import { enqueueToolJob } from "../../jobs/enqueue.js";
+import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
+import { isToolInstalled } from "../../lib/feature-status.js";
+import { type TranscriptSegment, toSrt, toVtt } from "../../lib/subtitle-format.js";
+import { receiveUpload } from "../../lib/upload-stream.js";
+
+const settingsSchema = z.object({
+  language: z
+    .enum(["auto", "en", "de", "fr", "es", "zh", "ja", "ko", "id", "th", "vi"])
+    .default("auto"),
+  format: z.enum(["srt", "vtt"]).default("srt"),
+});
+
+const OUTPUT_CONTENT_TYPES: Record<string, string> = {
+  srt: "application/x-subrip",
+  vtt: "text/vtt",
+};
+
+// -- AI job handler (runs inside the BullMQ worker) --
+registerAiJobHandler("auto-subtitles", async (input, data, ctx) => {
+  const settings = settingsSchema.parse(data.settings);
+
+  ctx.report(5, "Preparing video");
+
+  // Write the video buffer to scratch so ffmpeg and ffprobe can read it
+  const videoDir = join(ctx.scratchDir, "video");
+  await mkdir(videoDir, { recursive: true });
+  const videoPath = join(videoDir, data.filename);
+  await writeFile(videoPath, input);
+
+  // Probe for audio streams
+  const info = await probeMedia(videoPath);
+  const hasAudio = info.streams.some((s) => s.type === "audio");
+  if (!hasAudio) {
+    throw new Error("This video has no audio track to transcribe");
+  }
+
+  ctx.report(10, "Extracting audio");
+
+  // Extract 16kHz mono WAV (whisper's preferred input)
+  const wavPath = join(videoDir, "audio-16k.wav");
+  await runFfmpeg(
+    ["-i", videoPath, "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", wavPath],
+    { timeoutMs: 10 * 60_000 },
+  );
+
+  ctx.report(20, "Transcribing audio");
+
+  const result = await transcribeAudio(
+    wavPath,
+    { language: settings.language },
+    (percent, stage) => {
+      // Scale transcription progress into 20..95 range
+      const scaled = 20 + Math.round(percent * 0.75);
+      ctx.report(Math.min(scaled, 95), stage);
+    },
+  );
+
+  const segments: TranscriptSegment[] = result.segments.map((seg) => ({
+    startS: seg.startS,
+    endS: seg.endS,
+    text: seg.text,
+  }));
+
+  const base = data.filename.replace(/\.[^.]+$/, "");
+  const ext = settings.format;
+  const outName = `${base}.${ext}`;
+
+  const content = ext === "vtt" ? toVtt(segments) : toSrt(segments);
+
+  return {
+    buffer: Buffer.from(content, "utf-8"),
+    filename: outName,
+    contentType: OUTPUT_CONTENT_TYPES[ext],
+    resultPayload: {
+      language: result.language,
+      segments: segments.length,
+    },
+  };
+});
+
+export function registerAutoSubtitles(app: FastifyInstance) {
+  app.post("/api/v1/tools/auto-subtitles", async (request: FastifyRequest, reply: FastifyReply) => {
+    const toolId = "auto-subtitles";
+    if (!isToolInstalled(toolId)) {
+      const bundle = getBundleForTool(toolId);
+      return reply.status(501).send({
+        error: "Feature not installed",
+        code: "FEATURE_NOT_INSTALLED",
+        feature: TOOL_BUNDLE_MAP[toolId],
+        featureName: bundle?.name ?? toolId,
+        estimatedSize: bundle?.estimatedSize ?? "unknown",
+      });
+    }
+
+    const jobId = randomUUID();
+    let filename = "video";
+    let settingsRaw: string | null = null;
+    let clientJobId: string | null = null;
+    let inputKey: string | null = null;
+
+    try {
+      const parts = request.parts();
+      for await (const part of parts) {
+        if (part.type === "file") {
+          const upload = await receiveUpload(part, jobId);
+          inputKey = upload.key;
+          filename = upload.filename;
+        } else if (part.fieldname === "settings") {
+          settingsRaw = part.value as string;
+        } else if (part.fieldname === "clientJobId") {
+          const raw = part.value as string;
+          if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) {
+            clientJobId = raw;
+          }
+        }
+      }
+    } catch (err) {
+      return reply.status(400).send({
+        error: "Failed to parse multipart request",
+        details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
+      });
+    }
+
+    if (!inputKey) {
+      return reply.status(400).send({ error: "No video file provided" });
+    }
+
+    let settings: z.infer<typeof settingsSchema>;
+    try {
+      const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+      const result = settingsSchema.safeParse(parsed);
+      if (!result.success) {
+        return reply.status(400).send({
+          error: "Invalid settings",
+          details: formatZodErrors(result.error.issues),
+        });
+      }
+      settings = result.data;
+    } catch {
+      return reply.status(400).send({ error: "Settings must be valid JSON" });
+    }
+
+    const progressJobId = clientJobId || jobId;
+
+    await enqueueToolJob({
+      jobId,
+      toolId,
+      userId: null,
+      pool: "ai",
+      inputRefs: [inputKey],
+      filename,
+      settings,
+      clientJobId: clientJobId ?? undefined,
+      kind: "ai-tool",
+    });
+
+    return reply.status(202).send({ jobId: progressJobId, async: true });
+  });
+}

apps/api/src/routes/tools/background-replace.ts

@@ -0,0 +1,171 @@
+import { randomUUID } from "node:crypto";
+import { removeBackground } from "@snapotter/ai";
+import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
+import { enqueueToolJob } from "../../jobs/enqueue.js";
+import { autoOrient } from "../../lib/auto-orient.js";
+import { compositeOnColor } from "../../lib/bg-effects.js";
+import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
+import { isToolInstalled } from "../../lib/feature-status.js";
+import { validateImageBuffer } from "../../lib/file-validation.js";
+import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js";
+import { decodeHeic } from "../../lib/heic-converter.js";
+import { receiveUpload } from "../../lib/upload-stream.js";
+
+const settingsSchema = z.object({
+  color: z
+    .string()
+    .regex(/^#[0-9a-fA-F]{6}$/)
+    .default("#ffffff"),
+});
+
+// -- AI job handler (runs inside the BullMQ worker) --
+registerAiJobHandler("background-replace", async (input, data, ctx) => {
+  const settings = settingsSchema.parse(data.settings);
+
+  ctx.report(5, "Removing background");
+
+  const subjectPng = await removeBackground(input, ctx.scratchDir, {}, (percent, stage) => {
+    // Scale rembg progress into 5..80 range
+    const scaled = 5 + Math.round(percent * 0.75);
+    ctx.report(Math.min(scaled, 80), stage);
+  });
+
+  ctx.report(85, "Compositing on color");
+
+  const result = await compositeOnColor(subjectPng, settings.color);
+
+  const base = data.filename.replace(/\.[^.]+$/, "");
+  const outName = `${base}_bg.png`;
+
+  return {
+    buffer: result,
+    filename: outName,
+    contentType: "image/png",
+  };
+});
+
+export function registerBackgroundReplace(app: FastifyInstance) {
+  app.post(
+    "/api/v1/tools/background-replace",
+    async (request: FastifyRequest, reply: FastifyReply) => {
+      const toolId = "background-replace";
+      if (!isToolInstalled(toolId)) {
+        const bundle = getBundleForTool(toolId);
+        return reply.status(501).send({
+          error: "Feature not installed",
+          code: "FEATURE_NOT_INSTALLED",
+          feature: TOOL_BUNDLE_MAP[toolId],
+          featureName: bundle?.name ?? toolId,
+          estimatedSize: bundle?.estimatedSize ?? "unknown",
+        });
+      }
+
+      const jobId = randomUUID();
+      let fileBuffer: Buffer | null = null;
+      let filename = "image";
+      let settingsRaw: string | null = null;
+      let clientJobId: string | null = null;
+      let inputKey: string | null = null;
+
+      try {
+        const parts = request.parts();
+        for await (const part of parts) {
+          if (part.type === "file") {
+            const upload = await receiveUpload(part, jobId);
+            inputKey = upload.key;
+            filename = upload.filename;
+          } else if (part.fieldname === "settings") {
+            settingsRaw = part.value as string;
+          } else if (part.fieldname === "clientJobId") {
+            const raw = part.value as string;
+            if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) {
+              clientJobId = raw;
+            }
+          }
+        }
+      } catch (err) {
+        return reply.status(400).send({
+          error: "Failed to parse multipart request",
+          details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
+        });
+      }
+
+      if (!inputKey) {
+        return reply.status(400).send({ error: "No image file provided" });
+      }
+
+      const { getObjectBuffer, putObject } = await import("../../lib/object-storage.js");
+      fileBuffer = await getObjectBuffer(inputKey);
+
+      if (!fileBuffer || fileBuffer.length === 0) {
+        return reply.status(400).send({ error: "No image file provided" });
+      }
+
+      const validation = await validateImageBuffer(fileBuffer, filename);
+      if (!validation.valid) {
+        return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
+      }
+
+      let settings: z.infer<typeof settingsSchema>;
+      try {
+        const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+        const result = settingsSchema.safeParse(parsed);
+        if (!result.success) {
+          return reply
+            .status(400)
+            .send({ error: "Invalid settings", details: formatZodErrors(result.error.issues) });
+        }
+        settings = result.data;
+      } catch {
+        return reply.status(400).send({ error: "Settings must be valid JSON" });
+      }
+
+      try {
+        if (validation.format === "heif") {
+          fileBuffer = await decodeHeic(fileBuffer);
+          const ext = filename.match(/\.[^.]+$/)?.[0];
+          if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
+        }
+        if (needsCliDecode(validation.format)) {
+          fileBuffer = await decodeToSharpCompat(fileBuffer, validation.format);
+          const ext = filename.match(/\.[^.]+$/)?.[0];
+          if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
+        }
+        fileBuffer = await autoOrient(fileBuffer);
+      } catch (err) {
+        request.log.error({ err, toolId }, "Input decoding failed");
+        return reply.status(422).send({
+          error: "Processing failed",
+          details: stripInternalPaths(err instanceof Error ? err.message : "Unknown error"),
+        });
+      }
+
+      const decodedKey = `uploads/${jobId}/${filename}`;
+      if (decodedKey !== inputKey) {
+        await putObject(decodedKey, fileBuffer);
+        inputKey = decodedKey;
+      } else {
+        await putObject(inputKey, fileBuffer);
+      }
+
+      const progressJobId = clientJobId || jobId;
+
+      await enqueueToolJob({
+        jobId,
+        toolId,
+        userId: null,
+        pool: "ai",
+        inputRefs: [inputKey],
+        filename,
+        settings,
+        clientJobId: clientJobId ?? undefined,
+        kind: "ai-tool",
+      });
+
+      return reply.status(202).send({ jobId: progressJobId, async: true });
+    },
+  );
+}

apps/api/src/routes/tools/blur-background.ts

@@ -0,0 +1,168 @@
+import { randomUUID } from "node:crypto";
+import { removeBackground } from "@snapotter/ai";
+import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
+import { enqueueToolJob } from "../../jobs/enqueue.js";
+import { autoOrient } from "../../lib/auto-orient.js";
+import { blurBackground } from "../../lib/bg-effects.js";
+import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
+import { isToolInstalled } from "../../lib/feature-status.js";
+import { validateImageBuffer } from "../../lib/file-validation.js";
+import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js";
+import { decodeHeic } from "../../lib/heic-converter.js";
+import { receiveUpload } from "../../lib/upload-stream.js";
+
+const settingsSchema = z.object({
+  intensity: z.number().int().min(1).max(100).default(50),
+});
+
+// -- AI job handler (runs inside the BullMQ worker) --
+registerAiJobHandler("blur-background", async (input, data, ctx) => {
+  const settings = settingsSchema.parse(data.settings);
+
+  ctx.report(5, "Removing background");
+
+  const subjectPng = await removeBackground(input, ctx.scratchDir, {}, (percent, stage) => {
+    // Scale rembg progress into 5..80 range
+    const scaled = 5 + Math.round(percent * 0.75);
+    ctx.report(Math.min(scaled, 80), stage);
+  });
+
+  ctx.report(85, "Blurring background");
+
+  const result = await blurBackground(input, subjectPng, settings.intensity);
+
+  const base = data.filename.replace(/\.[^.]+$/, "");
+  const outName = `${base}_blurbg.png`;
+
+  return {
+    buffer: result,
+    filename: outName,
+    contentType: "image/png",
+  };
+});
+
+export function registerBlurBackground(app: FastifyInstance) {
+  app.post(
+    "/api/v1/tools/blur-background",
+    async (request: FastifyRequest, reply: FastifyReply) => {
+      const toolId = "blur-background";
+      if (!isToolInstalled(toolId)) {
+        const bundle = getBundleForTool(toolId);
+        return reply.status(501).send({
+          error: "Feature not installed",
+          code: "FEATURE_NOT_INSTALLED",
+          feature: TOOL_BUNDLE_MAP[toolId],
+          featureName: bundle?.name ?? toolId,
+          estimatedSize: bundle?.estimatedSize ?? "unknown",
+        });
+      }
+
+      const jobId = randomUUID();
+      let fileBuffer: Buffer | null = null;
+      let filename = "image";
+      let settingsRaw: string | null = null;
+      let clientJobId: string | null = null;
+      let inputKey: string | null = null;
+
+      try {
+        const parts = request.parts();
+        for await (const part of parts) {
+          if (part.type === "file") {
+            const upload = await receiveUpload(part, jobId);
+            inputKey = upload.key;
+            filename = upload.filename;
+          } else if (part.fieldname === "settings") {
+            settingsRaw = part.value as string;
+          } else if (part.fieldname === "clientJobId") {
+            const raw = part.value as string;
+            if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) {
+              clientJobId = raw;
+            }
+          }
+        }
+      } catch (err) {
+        return reply.status(400).send({
+          error: "Failed to parse multipart request",
+          details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
+        });
+      }
+
+      if (!inputKey) {
+        return reply.status(400).send({ error: "No image file provided" });
+      }
+
+      const { getObjectBuffer, putObject } = await import("../../lib/object-storage.js");
+      fileBuffer = await getObjectBuffer(inputKey);
+
+      if (!fileBuffer || fileBuffer.length === 0) {
+        return reply.status(400).send({ error: "No image file provided" });
+      }
+
+      const validation = await validateImageBuffer(fileBuffer, filename);
+      if (!validation.valid) {
+        return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
+      }
+
+      let settings: z.infer<typeof settingsSchema>;
+      try {
+        const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+        const result = settingsSchema.safeParse(parsed);
+        if (!result.success) {
+          return reply
+            .status(400)
+            .send({ error: "Invalid settings", details: formatZodErrors(result.error.issues) });
+        }
+        settings = result.data;
+      } catch {
+        return reply.status(400).send({ error: "Settings must be valid JSON" });
+      }
+
+      try {
+        if (validation.format === "heif") {
+          fileBuffer = await decodeHeic(fileBuffer);
+          const ext = filename.match(/\.[^.]+$/)?.[0];
+          if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
+        }
+        if (needsCliDecode(validation.format)) {
+          fileBuffer = await decodeToSharpCompat(fileBuffer, validation.format);
+          const ext = filename.match(/\.[^.]+$/)?.[0];
+          if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
+        }
+        fileBuffer = await autoOrient(fileBuffer);
+      } catch (err) {
+        request.log.error({ err, toolId }, "Input decoding failed");
+        return reply.status(422).send({
+          error: "Processing failed",
+          details: stripInternalPaths(err instanceof Error ? err.message : "Unknown error"),
+        });
+      }
+
+      const decodedKey = `uploads/${jobId}/${filename}`;
+      if (decodedKey !== inputKey) {
+        await putObject(decodedKey, fileBuffer);
+        inputKey = decodedKey;
+      } else {
+        await putObject(inputKey, fileBuffer);
+      }
+
+      const progressJobId = clientJobId || jobId;
+
+      await enqueueToolJob({
+        jobId,
+        toolId,
+        userId: null,
+        pool: "ai",
+        inputRefs: [inputKey],
+        filename,
+        settings,
+        clientJobId: clientJobId ?? undefined,
+        kind: "ai-tool",
+      });
+
+      return reply.status(202).send({ jobId: progressJobId, async: true });
+    },
+  );
+}

apps/api/src/routes/tools/index.ts

@@ -8,9 +8,12 @@ import { registerAspectPad } from "./aspect-pad.js";
 import { registerAudioChannels } from "./audio-channels.js";
 import { registerAudioMetadata } from "./audio-metadata.js";
 import { registerAudioSpeed } from "./audio-speed.js";
+import { registerAutoSubtitles } from "./auto-subtitles.js";
+import { registerBackgroundReplace } from "./background-replace.js";
 import { registerBarcodeGenerate } from "./barcode-generate.js";
 import { registerBarcodeRead } from "./barcode-read.js";
 import { registerBeautify } from "./beautify.js";
+import { registerBlurBackground } from "./blur-background.js";
 import { registerBlurFaces } from "./blur-faces.js";
 import { registerBlurPad } from "./blur-pad.js";
 import { registerBookletPdf } from "./booklet-pdf.js";
@@ -87,6 +90,7 @@ import { registerNoiseRemoval } from "./noise-removal.js";
 import { registerNormalizeAudio } from "./normalize-audio.js";
 import { registerNupPdf } from "./nup-pdf.js";
 import { registerOcr } from "./ocr.js";
+import { registerOcrPdf } from "./ocr-pdf.js";
 import { registerOptimizeForWeb } from "./optimize-for-web.js";
 import { registerOrganizePdf } from "./organize-pdf.js";
 import { registerPassportPhoto } from "./passport-photo.js";
@@ -131,6 +135,7 @@ import { registerStripMetadata } from "./strip-metadata.js";
 import { registerSvgToRaster } from "./svg-to-raster.js";
 import { registerTextOverlay } from "./text-overlay.js";
 import { registerToEpub } from "./to-epub.js";
+import { registerTranscribeAudio } from "./transcribe-audio.js";
 import { registerTransparencyFixer } from "./transparency-fixer.js";
 import { registerTrimAudio } from "./trim-audio.js";
 import { registerTrimVideo } from "./trim-video.js";
@@ -346,21 +351,26 @@ export async function registerToolRoutes(app: FastifyInstance): Promise<void> {
     { id: "yaml-json", register: registerYamlJson },
 
     // AI Tools
+    { id: "background-replace", register: registerBackgroundReplace },
+    { id: "blur-background", register: registerBlurBackground },
     { id: "remove-background", register: registerRemoveBackground },
     { id: "upscale", register: registerUpscale },
     { id: "ocr", register: registerOcr },
+    { id: "ocr-pdf", register: registerOcrPdf },
     { id: "blur-faces", register: registerBlurFaces },
     { id: "erase-object", register: registerEraseObject },
     { id: "smart-crop", register: registerSmartCrop },
     { id: "image-enhancement", register: registerImageEnhancement },
     { id: "content-aware-resize", register: registerContentAwareResize },
     { id: "ai-canvas-expand", register: registerAiCanvasExpand },
+    { id: "auto-subtitles", register: registerAutoSubtitles },
     { id: "colorize", register: registerColorize },
     { id: "enhance-faces", register: registerEnhanceFaces },
     { id: "noise-removal", register: registerNoiseRemoval },
     { id: "passport-photo", register: registerPassportPhoto },
     { id: "red-eye-removal", register: registerRedEyeRemoval },
     { id: "restore-photo", register: registerRestorePhoto },
+    { id: "transcribe-audio", register: registerTranscribeAudio },
     { id: "transparency-fixer", register: registerTransparencyFixer },
   ];
 

apps/api/src/routes/tools/ocr-pdf.ts

@@ -0,0 +1,136 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { extractPdfText } from "@snapotter/ai";
+import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
+import { enqueueToolJob } from "../../jobs/enqueue.js";
+import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
+import { isToolInstalled } from "../../lib/feature-status.js";
+import { receiveUpload } from "../../lib/upload-stream.js";
+
+const settingsSchema = z.object({
+  quality: z.enum(["fast", "balanced", "best"]).default("balanced"),
+  language: z.enum(["auto", "en", "de", "fr", "es", "zh", "ja", "ko"]).default("auto"),
+  pages: z.string().max(100).default("all"),
+});
+
+// -- AI job handler (runs inside the BullMQ worker) --
+registerAiJobHandler("ocr-pdf", async (input, data, ctx) => {
+  const settings = settingsSchema.parse(data.settings);
+
+  ctx.report(5, "Preparing PDF");
+
+  // Write the input buffer to a temp file (extractPdfText needs a file path)
+  const pdfDir = join(ctx.scratchDir, "pdf");
+  await mkdir(pdfDir, { recursive: true });
+  const pdfPath = join(pdfDir, data.filename);
+  await writeFile(pdfPath, input);
+
+  ctx.report(10, "Extracting text from PDF");
+
+  const result = await extractPdfText(
+    pdfPath,
+    {
+      quality: settings.quality,
+      language: settings.language,
+      pages: settings.pages,
+    },
+    (percent, stage) => ctx.report(percent, stage),
+  );
+
+  const base = data.filename.replace(/\.[^.]+$/, "");
+  const outName = `${base}_ocr.txt`;
+
+  return {
+    buffer: Buffer.from(result.text, "utf-8"),
+    filename: outName,
+    contentType: "text/plain",
+    resultPayload: {
+      pages: result.pages,
+      engine: result.engine,
+    },
+  };
+});
+
+export function registerOcrPdf(app: FastifyInstance) {
+  app.post("/api/v1/tools/ocr-pdf", async (request: FastifyRequest, reply: FastifyReply) => {
+    const toolId = "ocr-pdf";
+    if (!isToolInstalled(toolId)) {
+      const bundle = getBundleForTool(toolId);
+      return reply.status(501).send({
+        error: "Feature not installed",
+        code: "FEATURE_NOT_INSTALLED",
+        feature: TOOL_BUNDLE_MAP[toolId],
+        featureName: bundle?.name ?? toolId,
+        estimatedSize: bundle?.estimatedSize ?? "unknown",
+      });
+    }
+
+    const jobId = randomUUID();
+    let filename = "document.pdf";
+    let settingsRaw: string | null = null;
+    let clientJobId: string | null = null;
+    let inputKey: string | null = null;
+
+    try {
+      const parts = request.parts();
+      for await (const part of parts) {
+        if (part.type === "file") {
+          const upload = await receiveUpload(part, jobId);
+          inputKey = upload.key;
+          filename = upload.filename;
+        } else if (part.fieldname === "settings") {
+          settingsRaw = part.value as string;
+        } else if (part.fieldname === "clientJobId") {
+          const raw = part.value as string;
+          if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) {
+            clientJobId = raw;
+          }
+        }
+      }
+    } catch (err) {
+      return reply.status(400).send({
+        error: "Failed to parse multipart request",
+        details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
+      });
+    }
+
+    if (!inputKey) {
+      return reply.status(400).send({ error: "No PDF file provided" });
+    }
+
+    let settings: z.infer<typeof settingsSchema>;
+    try {
+      const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+      const result = settingsSchema.safeParse(parsed);
+      if (!result.success) {
+        return reply.status(400).send({
+          error: "Invalid settings",
+          details: formatZodErrors(result.error.issues),
+        });
+      }
+      settings = result.data;
+    } catch {
+      return reply.status(400).send({ error: "Settings must be valid JSON" });
+    }
+
+    const progressJobId = clientJobId || jobId;
+
+    await enqueueToolJob({
+      jobId,
+      toolId,
+      userId: null,
+      pool: "ai",
+      inputRefs: [inputKey],
+      filename,
+      settings,
+      clientJobId: clientJobId ?? undefined,
+      kind: "ai-tool",
+    });
+
+    return reply.status(202).send({ jobId: progressJobId, async: true });
+  });
+}

apps/api/src/routes/tools/transcribe-audio.ts

@@ -0,0 +1,160 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { transcribeAudio } from "@snapotter/ai";
+import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
+import { enqueueToolJob } from "../../jobs/enqueue.js";
+import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
+import { isToolInstalled } from "../../lib/feature-status.js";
+import { type TranscriptSegment, toSrt, toVtt } from "../../lib/subtitle-format.js";
+import { receiveUpload } from "../../lib/upload-stream.js";
+
+const settingsSchema = z.object({
+  language: z
+    .enum(["auto", "en", "de", "fr", "es", "zh", "ja", "ko", "id", "th", "vi"])
+    .default("auto"),
+  outputFormat: z.enum(["txt", "srt", "vtt"]).default("txt"),
+});
+
+const OUTPUT_CONTENT_TYPES: Record<string, string> = {
+  txt: "text/plain",
+  srt: "application/x-subrip",
+  vtt: "text/vtt",
+};
+
+// -- AI job handler (runs inside the BullMQ worker) --
+registerAiJobHandler("transcribe-audio", async (input, data, ctx) => {
+  const settings = settingsSchema.parse(data.settings);
+
+  ctx.report(5, "Preparing audio");
+
+  // Write the input buffer to a temp file (transcribeAudio needs a file path)
+  const audioDir = join(ctx.scratchDir, "audio");
+  await mkdir(audioDir, { recursive: true });
+  const audioPath = join(audioDir, data.filename);
+  await writeFile(audioPath, input);
+
+  ctx.report(10, "Transcribing audio");
+
+  const result = await transcribeAudio(
+    audioPath,
+    { language: settings.language },
+    (percent, stage) => ctx.report(percent, stage),
+  );
+
+  // Adapt bridge segments to the api-side TranscriptSegment shape (identical structure).
+  const segments: TranscriptSegment[] = result.segments.map((seg) => ({
+    startS: seg.startS,
+    endS: seg.endS,
+    text: seg.text,
+  }));
+
+  const base = data.filename.replace(/\.[^.]+$/, "");
+  const ext = settings.outputFormat;
+  const outName = `${base}.${ext}`;
+
+  let content: string;
+  if (ext === "srt") {
+    content = toSrt(segments);
+  } else if (ext === "vtt") {
+    content = toVtt(segments);
+  } else {
+    content = result.text;
+  }
+
+  return {
+    buffer: Buffer.from(content, "utf-8"),
+    filename: outName,
+    contentType: OUTPUT_CONTENT_TYPES[ext],
+    resultPayload: {
+      language: result.language,
+      segments: segments.length,
+    },
+  };
+});
+
+export function registerTranscribeAudio(app: FastifyInstance) {
+  app.post(
+    "/api/v1/tools/transcribe-audio",
+    async (request: FastifyRequest, reply: FastifyReply) => {
+      const toolId = "transcribe-audio";
+      if (!isToolInstalled(toolId)) {
+        const bundle = getBundleForTool(toolId);
+        return reply.status(501).send({
+          error: "Feature not installed",
+          code: "FEATURE_NOT_INSTALLED",
+          feature: TOOL_BUNDLE_MAP[toolId],
+          featureName: bundle?.name ?? toolId,
+          estimatedSize: bundle?.estimatedSize ?? "unknown",
+        });
+      }
+
+      const jobId = randomUUID();
+      let filename = "audio";
+      let settingsRaw: string | null = null;
+      let clientJobId: string | null = null;
+      let inputKey: string | null = null;
+
+      try {
+        const parts = request.parts();
+        for await (const part of parts) {
+          if (part.type === "file") {
+            const upload = await receiveUpload(part, jobId);
+            inputKey = upload.key;
+            filename = upload.filename;
+          } else if (part.fieldname === "settings") {
+            settingsRaw = part.value as string;
+          } else if (part.fieldname === "clientJobId") {
+            const raw = part.value as string;
+            if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(raw)) {
+              clientJobId = raw;
+            }
+          }
+        }
+      } catch (err) {
+        return reply.status(400).send({
+          error: "Failed to parse multipart request",
+          details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
+        });
+      }
+
+      if (!inputKey) {
+        return reply.status(400).send({ error: "No audio file provided" });
+      }
+
+      let settings: z.infer<typeof settingsSchema>;
+      try {
+        const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
+        const result = settingsSchema.safeParse(parsed);
+        if (!result.success) {
+          return reply.status(400).send({
+            error: "Invalid settings",
+            details: formatZodErrors(result.error.issues),
+          });
+        }
+        settings = result.data;
+      } catch {
+        return reply.status(400).send({ error: "Settings must be valid JSON" });
+      }
+
+      const progressJobId = clientJobId || jobId;
+
+      await enqueueToolJob({
+        jobId,
+        toolId,
+        userId: null,
+        pool: "ai",
+        inputRefs: [inputKey],
+        filename,
+        settings,
+        clientJobId: clientJobId ?? undefined,
+        kind: "ai-tool",
+      });
+
+      return reply.status(202).send({ jobId: progressJobId, async: true });
+    },
+  );
+}