feat(modality)!: SnapOtter 2.0 phase 3 modality framework: media/doc engines, pool routing, display modes (#218)

This commit is contained in:
SnapOtter
2026-06-13 10:18:39 +08:00
parent c451b939c7
commit d647d8ed19
99 changed files with 4380 additions and 1813 deletions
+4 -10
View File
@@ -20,7 +20,7 @@ import { captureException, initAnalytics, shutdownAnalytics } from "./lib/analyt
import { shouldRunStartupCleanup } from "./lib/cleanup.js";
import { buildCsp } from "./lib/csp.js";
import { ensureAiDirs, recoverInterruptedInstalls } from "./lib/feature-status.js";
import { shutdownWorkerPool } from "./lib/worker-pool.js";
import { requirePermission } from "./permissions.js";
import {
authMiddleware,
@@ -507,16 +507,10 @@ async function shutdown(signal: string) {
}
try {
await shutdownWorkerPool();
console.log("Worker pool shut down");
} catch (err) {
console.error("Error shutting down worker pool:", err);
}
try {
const { shutdownDispatcher } = await import("@snapotter/ai");
const { shutdownDispatcher, shutdownDocsDispatcher } = await import("@snapotter/ai");
shutdownDispatcher();
console.log("Python dispatcher shut down");
await shutdownDocsDispatcher();
console.log("Python dispatchers shut down");
} catch {
// AI package may not be available
}
+21 -2
View File
@@ -12,6 +12,7 @@ import { eq } from "drizzle-orm";
import sharp from "sharp";
import { db, schema } from "../db/index.js";
import { putObject } from "../lib/object-storage.js";
import { pdfFirstPagePreview, videoPosterPreview } from "../modality/preview.js";
// ── Content-type to extension map ──────────────────────────────
@@ -87,9 +88,9 @@ const BROWSER_PREVIEWABLE = new Set([
]);
/**
* Generate a browser-previewable WebP thumbnail for formats that browsers
* Generate a browser-previewable thumbnail for formats that browsers
* cannot render in <img> tags. Writes to object storage under
* `outputs/<jobId>/preview.webp`.
* `outputs/<jobId>/preview.<ext>` (webp for images/video, png for PDF).
*
* Returns the object key on success, undefined when the format is already
* previewable or when generation fails (non-fatal).
@@ -100,6 +101,24 @@ export async function generatePreview(
jobId: string,
fallbackInput?: Buffer,
): Promise<string | undefined> {
// Per-modality dispatch (before the image logic)
if (contentType.startsWith("video/")) {
const poster = await videoPosterPreview(buffer);
if (!poster) return undefined;
const key = `outputs/${jobId}/preview.webp`;
await putObject(key, poster);
return key;
}
if (contentType.startsWith("audio/")) return undefined; // no preview (spec 4.5)
if (contentType === "application/pdf") {
const page = await pdfFirstPagePreview(buffer);
if (!page) return undefined;
const key = `outputs/${jobId}/preview.png`;
await putObject(key, page);
return key;
}
// Image logic unchanged below
if (BROWSER_PREVIEWABLE.has(contentType)) return undefined;
const key = `outputs/${jobId}/preview.webp`;
+57 -7
View File
@@ -21,7 +21,7 @@
* are deferred until the final attempt so intermediate retries stay
* invisible to the client.
*/
import { mkdir, rm } from "node:fs/promises";
import { mkdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { type Job, UnrecoverableError, Worker } from "bullmq";
@@ -32,7 +32,11 @@ import { resolveConcurrency } from "../lib/env.js";
import { jobDuration, jobsTotal } from "../lib/metrics.js";
import { getObjectBuffer, putObject } from "../lib/object-storage.js";
import { publishEphemeral, updateSingleFileProgress } from "../routes/progress.js";
import { getToolConfig, type ToolProcessCtx } from "../routes/tool-factory.js";
import {
getToolConfig,
type ToolProcessCtx,
type ToolProcessInputV2,
} from "../routes/tool-factory.js";
import { hasAiJobHandler, runAiToolJob } from "./ai-handlers.js";
import { recordChildOutcome } from "./batch-progress.js";
import { registerCancelable, unregisterCancelable } from "./cancel.js";
@@ -80,7 +84,8 @@ export function buildLegacyResultPayload(
processedSize: jobResult.processedSize,
};
if (jobResult.previewRef) {
payload.previewUrl = `/api/v1/download/${jobId}/preview.webp`;
const previewFilename = jobResult.previewRef.split("/").pop();
payload.previewUrl = `/api/v1/download/${jobId}/${previewFilename}`;
}
if (jobResult.savedFileId) {
payload.savedFileId = jobResult.savedFileId;
@@ -123,8 +128,18 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
})
.where(eq(schema.jobs.id, jobId));
// Load input from object storage
const inputBuffer = await getObjectBuffer(data.inputRefs[0]);
// Load all input refs from object storage. The primary input keeps
// the client-facing filename; secondary inputs derive filenames from
// their ref basenames.
const inputs: ToolProcessInputV2[] = await Promise.all(
data.inputRefs.map(async (ref) => ({
ref,
buffer: await getObjectBuffer(ref),
filename: ref.split("/").slice(2).join("/") || data.filename,
})),
);
inputs[0].filename = data.filename; // primary keeps the client-facing name
const inputBuffer = inputs[0].buffer; // existing metrics/size/preview paths
// Progress reporter: emits both Redis pub/sub and BullMQ job progress
const progressJobId = data.clientJobId ?? jobId;
@@ -161,10 +176,45 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
} else {
const config = getToolConfig(data.toolId);
if (!config) throw new Error(`No tool config for ${data.toolId}`);
const result = await config.process(inputBuffer, data.settings, data.filename, ctx);
resultBuffer = result.buffer;
// Use the resolved v2 process function (adapter or native)
if (!config.processV2) throw new Error(`No processV2 for ${data.toolId}`);
const result = await config.processV2({
inputs,
settings: data.settings,
scratchDir,
signal,
report,
});
// Resolve buffer OR scratchPath for the primary output
if (result.buffer) {
resultBuffer = result.buffer;
} else if (result.scratchPath) {
resultBuffer = await readFile(result.scratchPath);
} else {
throw new Error(`Tool ${data.toolId} returned neither buffer nor scratchPath`);
}
resultFilename = result.filename;
resultContentType = result.contentType;
resultPayload = result.resultPayload;
// Resolve extra outputs with the same buffer/scratchPath duality
if (result.extraOutputs) {
extraOutputs = await Promise.all(
result.extraOutputs.map(async (extra) => {
let buf: Buffer;
if (extra.buffer) {
buf = extra.buffer;
} else if (extra.scratchPath) {
buf = await readFile(extra.scratchPath);
} else {
throw new Error(`Extra output "${extra.name}" has neither buffer nor scratchPath`);
}
return { name: extra.name, buffer: buf, contentType: extra.contentType };
}),
);
}
}
// Build output name with tool suffix and extension fixup
+4
View File
@@ -52,6 +52,10 @@ const envSchema = z
MAX_STORAGE_PER_USER_MB: z.coerce.number().default(5000),
MAX_WORKSPACE_SIZE_GB: z.coerce.number().default(10),
MAX_PDF_PAGES: z.coerce.number().default(0),
MAX_VIDEO_DURATION_S: z.coerce.number().default(0),
MAX_AUDIO_DURATION_S: z.coerce.number().default(0),
MAX_VIDEO_BITRATE_KBPS: z.coerce.number().default(0),
LIBREOFFICE_TIMEOUT_S: z.coerce.number().default(120),
SESSION_DURATION_HOURS: z.coerce.number().default(168),
LOGIN_ATTEMPT_LIMIT: z.coerce.number().default(30),
TRUST_PROXY: z
-63
View File
@@ -1,63 +0,0 @@
/**
* Piscina worker that executes image tool processing in a worker thread.
*
* On first call, it imports all tool registration modules using a mock
* Fastify instance (only the registry-populating side effects are needed,
* not the HTTP route registrations). Subsequent calls reuse the populated
* registry for O(1) lookup.
*/
import { autoOrient } from "./auto-orient.js";
export interface WorkerInput {
toolId: string;
inputBuffer: Buffer;
settings: unknown;
filename: string;
inputFormat?: string;
}
export interface WorkerOutput {
buffer: Buffer;
filename: string;
contentType: string;
}
let registryReady = false;
async function ensureRegistry(): Promise<void> {
if (registryReady) return;
// Create a minimal mock that satisfies the register functions.
// createToolRoute calls app.post() (no-op here) and toolRegistry.set() (the part we want).
// AI tools also call app.post() and registerToolProcessFn() (also populates the registry).
const mockApp = {
post: () => {},
get: () => {},
log: { info: () => {}, warn: () => {}, error: () => {} },
};
const { registerToolRoutes } = await import("../routes/tools/index.js");
await registerToolRoutes(mockApp as never);
registryReady = true;
}
export default async function processInWorker(input: WorkerInput): Promise<WorkerOutput> {
await ensureRegistry();
const { getToolConfig } = await import("../routes/tool-factory.js");
const config = getToolConfig(input.toolId);
if (!config) {
throw new Error(`Tool "${input.toolId}" not found in worker registry`);
}
const buf = Buffer.from(input.inputBuffer);
const oriented = input.inputFormat === "svg" ? buf : await autoOrient(buf);
const result = await config.process(oriented, input.settings, input.filename);
return {
buffer: result.buffer,
filename: result.filename,
contentType: result.contentType,
};
}
+14
View File
@@ -0,0 +1,14 @@
import { MODALITY_POOL, TOOL_BUNDLE_MAP, TOOLS } from "@snapotter/shared";
import { hasAiJobHandler } from "../jobs/ai-handlers.js";
import type { Pool } from "../jobs/types.js";
/** ai handler/bundle wins; else the tool's modality decides (spec 4.5). */
export function resolveToolPool(toolId: string): Pool {
if (hasAiJobHandler(toolId) || TOOL_BUNDLE_MAP[toolId]) return "ai";
const tool = TOOLS.find((t) => t.id === toolId);
return tool ? MODALITY_POOL[tool.modality] : "image";
}
export function shouldSkipSyncWindow(executionHint: "fast" | "long" | undefined): boolean {
return executionHint === "long";
}
-36
View File
@@ -1,36 +0,0 @@
/**
* Worker pool for offloading CPU-bound image processing from the main event loop.
*
* Uses Piscina (backed by worker_threads) so Sharp operations don't block
* HTTP request handling, SSE streams, or health checks.
*/
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import Piscina from "piscina";
import { loadEnv, resolveWorkerThreads } from "./env.js";
const __dirname = dirname(fileURLToPath(import.meta.url));
const maxThreads = resolveWorkerThreads(loadEnv());
let pool: Piscina | null = null;
export function getWorkerPool(): Piscina {
if (!pool) {
pool = new Piscina({
filename: resolve(__dirname, "image-worker.ts"),
// Inherit tsx loader flags from the main process so .ts files work in workers
execArgv: [...process.execArgv],
maxThreads,
idleTimeout: 30000,
});
}
return pool;
}
export async function shutdownWorkerPool(): Promise<void> {
if (pool) {
await pool.destroy();
pool = null;
}
}
+30
View File
@@ -0,0 +1,30 @@
/**
* Shared types for modality input handlers. Lives in its own file to
* break the import cycle between input-handler.ts (registry) and the
* per-modality implementations that reference these types.
*/
export class InputValidationError extends Error {
statusCode: number;
details?: string;
constructor(message: string, statusCode = 400, details?: string) {
super(message);
this.name = "InputValidationError";
this.statusCode = statusCode;
if (details !== undefined) this.details = details;
}
}
export interface PreparedInput {
buffer: Buffer;
filename: string;
}
/**
* Modality-specific upload validation/normalization (spec 4.5). Throws
* InputValidationError (400) on rejection. The factory owns storage and
* enqueueing; handlers own format logic only.
*/
export interface InputHandler {
prepare(raw: Buffer, filename: string, opts: { scratchDir: string }): Promise<PreparedInput>;
}
+62
View File
@@ -0,0 +1,62 @@
import { randomUUID } from "node:crypto";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { qpdfAvailable, qpdfCheck, qpdfPageCount } from "@snapotter/doc-engine";
import { env } from "../config.js";
import { type InputHandler, InputValidationError, type PreparedInput } from "./contract.js";
const ZIP_MAGIC = Buffer.from("PK");
/**
* Documents: header magic + qpdf structural check + page caps for PDFs
* (spec 4.5/4.7). Office/EPUB containers get a zip-magic sanity check in
* phase 3; deep validation happens when conversion engines consume them.
* The "file" modality (csv/json/...) shares this handler as a passthrough.
*/
export class DocumentInputHandler implements InputHandler {
async prepare(
raw: Buffer,
filename: string,
opts: { scratchDir: string },
): Promise<PreparedInput> {
if (raw.length === 0) throw new InputValidationError("Empty file");
const lower = filename.toLowerCase();
if (lower.endsWith(".pdf")) {
if (raw.subarray(0, 5).toString() !== "%PDF-") {
throw new InputValidationError("File does not start with a PDF header");
}
if (qpdfAvailable()) {
const dir = join(opts.scratchDir, `qpdf-${randomUUID()}`);
await mkdir(dir, { recursive: true });
const p = join(dir, "input.pdf");
try {
await writeFile(p, raw);
try {
await qpdfCheck(p);
} catch (err) {
throw new InputValidationError(
`Damaged PDF: ${err instanceof Error ? err.message.slice(0, 300) : "structural check failed"}`,
);
}
if (env.MAX_PDF_PAGES > 0) {
const pages = await qpdfPageCount(p);
if (pages > env.MAX_PDF_PAGES) {
throw new InputValidationError(
`PDF has ${pages} pages, exceeding the maximum of ${env.MAX_PDF_PAGES}`,
);
}
}
} finally {
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
}
} else if (
[".docx", ".xlsx", ".pptx", ".epub", ".odt", ".ods", ".odp"].some((e) => lower.endsWith(e))
) {
if (!raw.subarray(0, 2).equals(ZIP_MAGIC)) {
throw new InputValidationError("File is not a valid Office/EPUB container");
}
}
return { buffer: raw, filename };
}
}
+116
View File
@@ -0,0 +1,116 @@
import sharp from "sharp";
import { autoOrient } from "../lib/auto-orient.js";
import { stripInternalPaths } from "../lib/errors.js";
import { validateImageBuffer } from "../lib/file-validation.js";
import { decodeAnyFormat, decodeToSharpCompat, needsCliDecode } from "../lib/format-decoders.js";
import { decodeHeic } from "../lib/heic-converter.js";
import { decompressSvgz, sanitizeSvg } from "../lib/svg-sanitize.js";
import { type InputHandler, InputValidationError, type PreparedInput } from "./contract.js";
/**
* Image input handler: validateImageBuffer, HEIC decode, CLI decode,
* SVG sanitize, AVIF probe fallback, autoOrient. Extracted verbatim
* from the tool-factory validation/decode chain.
*/
export class ImageInputHandler implements InputHandler {
async prepare(
raw: Buffer,
originalFilename: string,
_opts: { scratchDir: string },
): Promise<PreparedInput> {
let fileBuffer = raw;
let name = originalFilename;
// Validate the uploaded image
const validation = await validateImageBuffer(fileBuffer, name);
if (!validation.valid) {
throw new InputValidationError(`Invalid image: ${validation.reason}`);
}
// Decode HEIC/HEIF input via system heif-dec (Sharp's bundled libheif
// lacks the HEVC decoder needed for iPhone photos).
// The decoded buffer is PNG, so update the filename extension to match.
const isHeif = validation.format === "heif";
if (isHeif) {
try {
fileBuffer = await decodeHeic(fileBuffer);
const ext = name.match(/\.[^.]+$/)?.[0];
if (ext) name = `${name.slice(0, -ext.length)}.png`;
} catch (err) {
throw new InputValidationError(
"Failed to decode HEIC file. Ensure libheif-examples is installed.",
422,
stripInternalPaths(err instanceof Error ? err.message : String(err)),
);
}
}
// Decode CLI-decoded formats (RAW, PSD, TGA, EXR, HDR) via external tools.
// The decoded buffer is PNG, so update the filename extension to match.
// Pass the original file extension so RAW decoder can use the correct
// temp file suffix (e.g. .cr3, .nef) for format identification.
if (needsCliDecode(validation.format)) {
try {
const fileExt = name.split(".").pop()?.toLowerCase();
fileBuffer = await decodeToSharpCompat(fileBuffer, validation.format, fileExt);
} catch {
try {
await sharp(fileBuffer).metadata();
} catch (err) {
throw new InputValidationError(
`Failed to decode ${validation.format.toUpperCase()} file`,
422,
stripInternalPaths(err instanceof Error ? err.message : String(err)),
);
}
}
const ext = name.match(/\.[^.]+$/)?.[0];
if (ext) name = `${name.slice(0, -ext.length)}.png`;
}
// Sanitize SVG input to prevent XXE, SSRF, and script injection
const isSvg = validation.format === "svg";
if (isSvg) {
try {
fileBuffer = decompressSvgz(fileBuffer);
fileBuffer = sanitizeSvg(fileBuffer);
} catch (err) {
throw new InputValidationError(err instanceof Error ? err.message : "Invalid SVG");
}
}
// AVIF can pass metadata validation but fail pixel decode when
// Sharp's bundled libheif lacks support for the bitstream version.
// A 1x1 resize forces a minimal pixel decode to catch this early.
if (validation.format === "avif") {
try {
await sharp(fileBuffer).resize(1).raw().toBuffer();
} catch {
try {
fileBuffer = await decodeAnyFormat(fileBuffer, "avif");
const ext = name.match(/\.[^.]+$/)?.[0];
if (ext) name = `${name.slice(0, -ext.length)}.png`;
} catch (fallbackErr) {
throw new InputValidationError(
"Failed to decode AVIF file",
422,
stripInternalPaths(
fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr),
),
);
}
}
}
// Auto-orient non-SVG images: physically rotate pixels to match
// the EXIF orientation tag so the worker sees upright pixels.
if (!isSvg) {
fileBuffer = await autoOrient(fileBuffer);
}
return {
buffer: fileBuffer,
filename: name,
};
}
}
+17
View File
@@ -0,0 +1,17 @@
import type { Modality } from "@snapotter/shared";
import type { InputHandler } from "./contract.js";
import { DocumentInputHandler } from "./document-input.js";
import { ImageInputHandler } from "./image-input.js";
import { MediaInputHandler } from "./media-input.js";
const HANDLERS: Record<Modality, InputHandler> = {
image: new ImageInputHandler(),
video: new MediaInputHandler("video"),
audio: new MediaInputHandler("audio"),
document: new DocumentInputHandler(),
file: new DocumentInputHandler(),
};
export function inputHandlerFor(modality: Modality): InputHandler {
return HANDLERS[modality];
}
+63
View File
@@ -0,0 +1,63 @@
import { randomUUID } from "node:crypto";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { probeMedia } from "@snapotter/media-engine";
import { env } from "../config.js";
import { type InputHandler, InputValidationError, type PreparedInput } from "./contract.js";
/**
* Video/audio validation via capped ffprobe (spec 4.7). ffprobe needs a real
* file (mp4 moov atoms may trail), so the buffer lands in the scratch dir.
*/
export class MediaInputHandler implements InputHandler {
constructor(private kind: "video" | "audio") {}
async prepare(
raw: Buffer,
filename: string,
opts: { scratchDir: string },
): Promise<PreparedInput> {
const probeDir = join(opts.scratchDir, `probe-${randomUUID()}`);
await mkdir(probeDir, { recursive: true });
const probePath = join(probeDir, "input");
try {
await writeFile(probePath, raw);
let info: Awaited<ReturnType<typeof probeMedia>>;
try {
info = await probeMedia(probePath);
} catch (err) {
throw new InputValidationError(
`Unrecognized ${this.kind} file: ${err instanceof Error ? err.message : String(err)}`,
);
}
const hasVideo = info.streams.some((s) => s.type === "video");
const hasAudio = info.streams.some((s) => s.type === "audio");
if (this.kind === "video" && !hasVideo) {
throw new InputValidationError("File contains no video stream");
}
if (this.kind === "audio" && !hasAudio) {
throw new InputValidationError("File contains no audio stream");
}
const durationCap =
this.kind === "video" ? env.MAX_VIDEO_DURATION_S : env.MAX_AUDIO_DURATION_S;
if (durationCap > 0 && info.durationS !== null && info.durationS > durationCap) {
throw new InputValidationError(
`Duration ${Math.round(info.durationS)}s exceeds the maximum of ${durationCap}s`,
);
}
if (
this.kind === "video" &&
env.MAX_VIDEO_BITRATE_KBPS > 0 &&
info.bitrateKbps !== null &&
info.bitrateKbps > env.MAX_VIDEO_BITRATE_KBPS
) {
throw new InputValidationError(
`Bitrate ${info.bitrateKbps}kbps exceeds the maximum of ${env.MAX_VIDEO_BITRATE_KBPS}kbps`,
);
}
return { buffer: raw, filename };
} finally {
await rm(probeDir, { recursive: true, force: true }).catch(() => {});
}
}
}
+89
View File
@@ -0,0 +1,89 @@
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveGs } from "@snapotter/doc-engine";
import { ffmpegAvailable, runFfmpeg } from "@snapotter/media-engine";
const PREVIEW_WIDTH = 480;
/** Video poster frame as WebP, or null when ffmpeg is unavailable/fails. */
export async function videoPosterPreview(buffer: Buffer): Promise<Buffer | null> {
if (!ffmpegAvailable()) return null;
const dir = join(tmpdir(), "snapotter-scratch", `preview-${randomUUID()}`);
await mkdir(dir, { recursive: true });
try {
const input = join(dir, "in");
const output = join(dir, "poster.webp");
await writeFile(input, buffer);
await runFfmpeg(
["-ss", "0", "-i", input, "-frames:v", "1", "-vf", `scale=${PREVIEW_WIDTH}:-2`, output],
{ timeoutMs: 30_000 },
);
return await readFile(output);
} catch {
return null; // previews must never fail the job
} finally {
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
}
/** First PDF page rendered to PNG via ghostscript, or null. */
export async function pdfFirstPagePreview(buffer: Buffer): Promise<Buffer | null> {
const gs = resolveGs();
if (!gs) return null;
const dir = join(tmpdir(), "snapotter-scratch", `preview-${randomUUID()}`);
await mkdir(dir, { recursive: true });
try {
const input = join(dir, "in.pdf");
const output = join(dir, "page1.png");
await writeFile(input, buffer);
await new Promise<void>((resolvePromise, reject) => {
const child = spawn(
gs,
[
"-dSAFER",
"-dBATCH",
"-dNOPAUSE",
"-dFirstPage=1",
"-dLastPage=1",
"-sDEVICE=png16m",
"-r96",
`-sOutputFile=${output}`,
input,
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
let err = "";
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
child.kill("SIGKILL");
reject(new Error("ghostscript preview timed out"));
}, 30_000);
child.stderr.on("data", (c: Buffer) => {
err = (err + c.toString("utf8")).slice(-2048);
});
child.on("error", (e) => {
if (settled) return;
settled = true;
clearTimeout(timer);
reject(e);
});
child.on("close", (code, signal) => {
if (settled) return;
settled = true;
clearTimeout(timer);
if (code === 0) resolvePromise();
else reject(new Error(`gs exited ${code ?? signal}: ${err}`));
});
});
return await readFile(output);
} catch {
return null;
} finally {
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
}
+2 -2
View File
@@ -17,7 +17,6 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import sharp from "sharp";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { hasAiJobHandler } from "../jobs/ai-handlers.js";
import { recordChildOutcome } from "../jobs/batch-progress.js";
import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
@@ -30,6 +29,7 @@ import { sanitizeFilename } from "../lib/filename.js";
import { decodeToSharpCompat, needsCliDecode } from "../lib/format-decoders.js";
import { decodeHeic } from "../lib/heic-converter.js";
import { getObjectStream, putObject } from "../lib/object-storage.js";
import { resolveToolPool } from "../lib/pool.js";
import { getAuthUser } from "../plugins/auth.js";
import { updateJobProgress } from "./progress.js";
import { getToolConfig } from "./tool-factory.js";
@@ -135,7 +135,7 @@ export async function registerBatchRoutes(app: FastifyInstance): Promise<void> {
// ── Create job ID and initial progress ────────────────────────
const parentId = clientJobId || randomUUID();
const userId = getAuthUser(request)?.id ?? null;
const pool: Pool = hasAiJobHandler(toolId) || TOOL_BUNDLE_MAP[toolId] ? "ai" : "image";
const pool: Pool = resolveToolPool(toolId);
// Insert the parent row BEFORE updateJobProgress, because the
// progress persist layer does a check-then-insert that races
+3 -8
View File
@@ -16,7 +16,6 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { hasAiJobHandler } from "../jobs/ai-handlers.js";
import { recordChildOutcome } from "../jobs/batch-progress.js";
import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
@@ -30,6 +29,7 @@ import { sanitizeFilename } from "../lib/filename.js";
import { decodeToSharpCompat, needsCliDecode } from "../lib/format-decoders.js";
import { decodeHeic } from "../lib/heic-converter.js";
import { getObjectStream, putObject } from "../lib/object-storage.js";
import { resolveToolPool } from "../lib/pool.js";
import { isSvgBuffer, sanitizeSvg } from "../lib/svg-sanitize.js";
import { hasEffectivePermission } from "../permissions.js";
import { getAuthUser, requireAuth } from "../plugins/auth.js";
@@ -64,11 +64,6 @@ const savePipelineSchema = z.object({
// ── Helpers ────────────────────────────────────────────────────
function resolvePool(toolId: string): Pool {
if (hasAiJobHandler(toolId) || TOOL_BUNDLE_MAP[toolId]) return "ai";
return "image";
}
interface ParsedStep {
toolId: string;
resolvedToolId: string;
@@ -335,7 +330,7 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
toolId: step.toolId,
resolvedToolId,
parsedSettings: settingsResult.data,
pool: resolvePool(resolvedToolId),
pool: resolveToolPool(resolvedToolId),
});
}
@@ -718,7 +713,7 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
toolId: step.toolId,
resolvedToolId,
parsedSettings: settingsResult.data,
pool: resolvePool(resolvedToolId),
pool: resolveToolPool(resolvedToolId),
});
}
+196 -188
View File
@@ -1,20 +1,20 @@
import { randomUUID } from "node:crypto";
import { mkdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { ANALYTICS_EVENTS, getBundleForTool, TOOL_BUNDLE_MAP, TOOLS } from "@snapotter/shared";
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import sharp from "sharp";
import type { z } from "zod";
import { env } from "../config.js";
import { enqueueToolJob, waitForJob } from "../jobs/enqueue.js";
import { trackEvent } from "../lib/analytics.js";
import { autoOrient } from "../lib/auto-orient.js";
import { formatZodErrors, stripInternalPaths } from "../lib/errors.js";
import { isToolInstalled } from "../lib/feature-status.js";
import { validateImageBuffer } from "../lib/file-validation.js";
import { decodeAnyFormat, decodeToSharpCompat, needsCliDecode } from "../lib/format-decoders.js";
import { decodeHeic } from "../lib/heic-converter.js";
import { getObjectBuffer, putObject } from "../lib/object-storage.js";
import { decompressSvgz, sanitizeSvg } from "../lib/svg-sanitize.js";
import { resolveToolPool, shouldSkipSyncWindow } from "../lib/pool.js";
import { receiveUpload } from "../lib/upload-stream.js";
import { InputValidationError } from "../modality/contract.js";
import { inputHandlerFor } from "../modality/input-handler.js";
import { getAuthUser } from "../plugins/auth.js";
import { updateSingleFileProgress } from "./progress.js";
@@ -25,6 +25,41 @@ export interface ToolProcessCtx {
report: (percent: number, stage?: string) => void;
}
// ── V2 process contract (ref-based, multi-input) ──────────────
export interface ToolProcessInputV2 {
buffer: Buffer;
filename: string;
ref: string;
}
export interface ToolProcessCtxV2 {
inputs: ToolProcessInputV2[];
settings: unknown;
scratchDir: string;
signal: AbortSignal;
report: (percent: number, stage?: string) => void;
}
export interface ToolProcessResultV2 {
/** Exactly one of buffer | scratchPath must be set. */
buffer?: Buffer;
scratchPath?: string;
filename: string;
contentType: string;
resultPayload?: Record<string, unknown>;
extraOutputs?: Array<{
name: string;
buffer?: Buffer;
scratchPath?: string;
contentType: string;
}>;
}
export type ToolProcessV2 = (ctx: ToolProcessCtxV2) => Promise<ToolProcessResultV2>;
// ── Tool route config ─────────────────────────────────────────
export interface ToolRouteConfig<T> {
/** Unique tool identifier, used as the URL path segment. */
toolId: string;
@@ -37,6 +72,8 @@ export interface ToolRouteConfig<T> {
filename: string,
ctx?: ToolProcessCtx,
) => Promise<{ buffer: Buffer; filename: string; contentType: string }>;
/** Optional v2 process function. When set, the worker calls this instead of the legacy process. */
processV2?: ToolProcessV2;
}
/** Type-erased config stored in the registry (settings type is widened to avoid variance issues). */
@@ -49,6 +86,26 @@ export interface AnyToolRouteConfig {
filename: string,
ctx?: ToolProcessCtx,
) => Promise<{ buffer: Buffer; filename: string; contentType: string }>;
processV2?: ToolProcessV2;
}
// ── Legacy adapter ────────────────────────────────────────────
/**
* Wraps a legacy process function as a ToolProcessV2. The first input
* is forwarded as the primary buffer/filename; extra inputs are ignored
* (legacy tools accept only one input).
*/
function adaptLegacyProcess(config: AnyToolRouteConfig): ToolProcessV2 {
return async (ctx) => {
const primary = ctx.inputs[0];
const result = await config.process(primary.buffer, ctx.settings, primary.filename, {
signal: ctx.signal,
scratchDir: ctx.scratchDir,
report: ctx.report,
});
return { buffer: result.buffer, filename: result.filename, contentType: result.contentType };
};
}
/**
@@ -75,9 +132,13 @@ export function getRegisteredToolIds(): string[] {
* Register a tool's process function in the pipeline/batch registry
* without creating an HTTP route. Use this for tools that have their
* own custom HTTP route but should still be usable in pipelines.
*
* Resolves processV2: uses the config's processV2 when provided,
* otherwise wraps the legacy process function via adaptLegacyProcess.
*/
export function registerToolProcessFn(config: AnyToolRouteConfig): void {
toolRegistry.set(config.toolId, config);
const resolved = { ...config, processV2: config.processV2 ?? adaptLegacyProcess(config) };
toolRegistry.set(config.toolId, resolved);
}
/**
@@ -96,8 +157,14 @@ export function registerToolProcessFn(config: AnyToolRouteConfig): void {
* - Response formatting (legacy envelope)
*/
export function createToolRoute<T>(app: FastifyInstance, config: ToolRouteConfig<T>): void {
// Register in the tool registry for batch processing (cast to type-erased form)
toolRegistry.set(config.toolId, config as AnyToolRouteConfig);
// Register a resolved copy in the tool registry for batch processing.
// Spread avoids mutating the caller's config object.
const erased = config as AnyToolRouteConfig;
const resolved: AnyToolRouteConfig = {
...erased,
processV2: erased.processV2 ?? adaptLegacyProcess(erased),
};
toolRegistry.set(config.toolId, resolved);
app.post(
`/api/v1/tools/${config.toolId}`,
@@ -181,203 +248,144 @@ export function createToolRoute<T>(app: FastifyInstance, config: ToolRouteConfig
reportProgress(5, "Validating...");
// Validate the uploaded image
const validation = await validateImageBuffer(fileBuffer, filename);
if (!validation.valid) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({ error: `Invalid image: ${validation.reason}` });
}
// Resolve the tool's modality (default "image" for registry-only test tools)
const toolMeta = TOOLS.find((t) => t.id === config.toolId);
const modality = toolMeta?.modality ?? "image";
// Decode HEIC/HEIF input via system heif-dec (Sharp's bundled libheif
// lacks the HEVC decoder needed for iPhone photos).
// The decoded buffer is PNG, so update the filename extension to match.
const isHeif = validation.format === "heif";
if (isHeif) {
reportProgress(10, "Decoding HEIC...");
try {
fileBuffer = await decodeHeic(fileBuffer);
const ext = filename.match(/\.[^.]+$/)?.[0];
if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
} catch (err) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(422).send({
error: "Failed to decode HEIC file. Ensure libheif-examples is installed.",
details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
});
}
}
// Decode CLI-decoded formats (RAW, PSD, TGA, EXR, HDR) via external tools.
// The decoded buffer is PNG, so update the filename extension to match.
// Pass the original file extension so RAW decoder can use the correct
// temp file suffix (e.g. .cr3, .nef) for format identification.
if (needsCliDecode(validation.format)) {
reportProgress(10, "Decoding...");
try {
const fileExt = filename.split(".").pop()?.toLowerCase();
fileBuffer = await decodeToSharpCompat(fileBuffer, validation.format, fileExt);
} catch {
try {
await sharp(fileBuffer).metadata();
} catch (err) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(422).send({
error: `Failed to decode ${validation.format.toUpperCase()} file`,
details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
});
}
}
const ext = filename.match(/\.[^.]+$/)?.[0];
if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
}
// Sanitize SVG input to prevent XXE, SSRF, and script injection
const isSvg = validation.format === "svg";
if (isSvg) {
try {
fileBuffer = decompressSvgz(fileBuffer);
fileBuffer = sanitizeSvg(fileBuffer);
} catch (err) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({
error: err instanceof Error ? err.message : "Invalid SVG",
});
}
}
// AVIF can pass metadata validation but fail pixel decode when
// Sharp's bundled libheif lacks support for the bitstream version.
// A 1x1 resize forces a minimal pixel decode to catch this early.
if (validation.format === "avif") {
try {
await sharp(fileBuffer).resize(1).raw().toBuffer();
} catch {
try {
reportProgress(10, "Decoding...");
fileBuffer = await decodeAnyFormat(fileBuffer, "avif");
const ext = filename.match(/\.[^.]+$/)?.[0];
if (ext) filename = `${filename.slice(0, -ext.length)}.png`;
} catch (fallbackErr) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(422).send({
error: "Failed to decode AVIF file",
details: stripInternalPaths(
fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr),
),
});
}
}
}
// Auto-orient non-SVG images: physically rotate pixels to match
// the EXIF orientation tag so the worker sees upright pixels.
if (!isSvg) {
fileBuffer = await autoOrient(fileBuffer);
}
reportProgress(15, "Preparing...");
// Parse and validate settings
if (settingsRaw && settingsRaw.length > 65536) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({ error: "Settings payload too large (max 64KB)" });
}
let settings: T;
// Per-request scratch dir for handlers that need temp files
const scratchDir = join(tmpdir(), "snapotter-scratch", jobId);
await mkdir(scratchDir, { recursive: true });
try {
const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
const result = config.settingsSchema.safeParse(parsed);
if (!result.success) {
// Modality-specific input validation and normalization
try {
const prepared = await inputHandlerFor(modality).prepare(fileBuffer, filename, {
scratchDir,
});
fileBuffer = prepared.buffer;
filename = prepared.filename;
} catch (err) {
if (err instanceof InputValidationError) {
const body: Record<string, string> = { error: err.message };
if (err.details) body.details = err.details;
return reply.status(err.statusCode).send(body);
}
throw err;
}
reportProgress(15, "Preparing...");
// Parse and validate settings
if (settingsRaw && settingsRaw.length > 65536) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({
error: "Invalid settings",
details: formatZodErrors(result.error.issues),
return reply.status(400).send({ error: "Settings payload too large (max 64KB)" });
}
let settings: T;
try {
const parsed = settingsRaw ? JSON.parse(settingsRaw) : {};
const result = config.settingsSchema.safeParse(parsed);
if (!result.success) {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({
error: "Invalid settings",
details: formatZodErrors(result.error.issues),
});
}
settings = result.data;
} catch {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
// Guard: check if the tool's AI feature bundle is installed
const bundleId = TOOL_BUNDLE_MAP[config.toolId];
if (bundleId && !isToolInstalled(config.toolId)) {
const bundle = getBundleForTool(config.toolId);
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(501).send({
error: "Feature not installed",
code: "FEATURE_NOT_INSTALLED",
feature: bundleId,
featureName: bundle?.name ?? bundleId,
estimatedSize: bundle?.estimatedSize ?? "unknown",
});
}
settings = result.data;
} catch {
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(400).send({ error: "Settings must be valid JSON" });
}
// Guard: check if the tool's AI feature bundle is installed
const bundleId = TOOL_BUNDLE_MAP[config.toolId];
if (bundleId && !isToolInstalled(config.toolId)) {
const bundle = getBundleForTool(config.toolId);
// Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
return reply.status(501).send({
error: "Feature not installed",
code: "FEATURE_NOT_INSTALLED",
feature: bundleId,
featureName: bundle?.name ?? bundleId,
estimatedSize: bundle?.estimatedSize ?? "unknown",
// If decode/orient transformed the buffer or changed the filename,
// write the final version so the worker processes the correct data.
// Skip re-upload when the buffer is reference-identical to the
// originally streamed bytes and the filename hasn't changed.
const decodedName = filename;
const decodedKey = `uploads/${jobId}/${decodedName}`;
if (decodedKey !== inputKey) {
await putObject(decodedKey, fileBuffer);
inputKey = decodedKey;
} else if (fileBuffer !== originalBuffer) {
await putObject(inputKey, fileBuffer);
}
const startTime = Date.now();
const pool = resolveToolPool(config.toolId);
// Enqueue for the BullMQ worker
await enqueueToolJob({
jobId,
toolId: config.toolId,
userId: getAuthUser(request)?.id ?? null,
pool,
inputRefs: [inputKey],
filename,
settings,
fileId: fileId ?? undefined,
clientJobId: clientJobId ?? undefined,
kind: "tool",
});
}
// If decode/orient transformed the buffer or changed the filename,
// write the final version so the worker processes the correct data.
// Skip re-upload when the buffer is reference-identical to the
// originally streamed bytes and the filename hasn't changed.
const decodedName = filename;
const decodedKey = `uploads/${jobId}/${decodedName}`;
if (decodedKey !== inputKey) {
await putObject(decodedKey, fileBuffer);
inputKey = decodedKey;
} else if (fileBuffer !== originalBuffer) {
await putObject(inputKey, fileBuffer);
}
// Long tools never block the HTTP request (spec 4.5): straight to SSE.
if (shouldSkipSyncWindow(toolMeta?.executionHint)) {
return reply.status(202).send({ jobId: clientJobId || jobId, async: true });
}
const startTime = Date.now();
try {
const result = await waitForJob(pool, jobId);
if (result) {
trackEvent(request, ANALYTICS_EVENTS.TOOL_USED, {
tool_id: config.toolId,
status: "completed",
duration_ms: Date.now() - startTime,
category: TOOLS.find((t) => t.id === config.toolId)?.category ?? "unknown",
is_ai_tool: getBundleForTool(config.toolId) !== null,
});
// Enqueue for the BullMQ worker
await enqueueToolJob({
jobId,
toolId: config.toolId,
userId: getAuthUser(request)?.id ?? null,
pool: "image",
inputRefs: [inputKey],
filename,
settings,
fileId: fileId ?? undefined,
clientJobId: clientJobId ?? undefined,
kind: "tool",
});
try {
const result = await waitForJob("image", jobId);
if (result) {
return reply.send({
jobId,
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(result.filename)}`,
previewUrl: result.previewRef
? `/api/v1/download/${jobId}/${result.previewRef.split("/").pop()}`
: undefined,
originalSize: result.originalSize,
processedSize: result.processedSize,
savedFileId: result.savedFileId,
});
}
return reply.status(202).send({ jobId: clientJobId || jobId, async: true });
} catch (err) {
trackEvent(request, ANALYTICS_EVENTS.TOOL_USED, {
tool_id: config.toolId,
status: "completed",
status: "failed",
duration_ms: Date.now() - startTime,
category: TOOLS.find((t) => t.id === config.toolId)?.category ?? "unknown",
is_ai_tool: getBundleForTool(config.toolId) !== null,
error_code: err instanceof Error ? err.constructor.name : "UnknownError",
error_message:
err instanceof Error ? err.message.slice(0, 200) : "Image processing failed",
});
return reply.send({
jobId,
downloadUrl: `/api/v1/download/${jobId}/${encodeURIComponent(result.filename)}`,
previewUrl: result.previewRef ? `/api/v1/download/${jobId}/preview.webp` : undefined,
originalSize: result.originalSize,
processedSize: result.processedSize,
savedFileId: result.savedFileId,
return reply.status(422).send({
error: "Processing failed",
details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
});
}
return reply.status(202).send({ jobId: clientJobId || jobId, async: true });
} catch (err) {
trackEvent(request, ANALYTICS_EVENTS.TOOL_USED, {
tool_id: config.toolId,
status: "failed",
duration_ms: Date.now() - startTime,
category: TOOLS.find((t) => t.id === config.toolId)?.category ?? "unknown",
is_ai_tool: getBundleForTool(config.toolId) !== null,
error_code: err instanceof Error ? err.constructor.name : "UnknownError",
error_message:
err instanceof Error ? err.message.slice(0, 200) : "Image processing failed",
});
return reply.status(422).send({
error: "Processing failed",
details: stripInternalPaths(err instanceof Error ? err.message : String(err)),
});
} finally {
await rm(scratchDir, { recursive: true, force: true }).catch(() => {});
}
},
);