From be254f9ca697602c0f7aff3911321d5e06dc6008 Mon Sep 17 00:00:00 2001 From: ashim-hq Date: Mon, 20 Apr 2026 21:46:07 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20dynamic=20timeouts=20=E2=80=94=20scale?= =?UTF-8?q?=20with=20image=20size,=20respect=20PROCESSING=5FTIMEOUT=5FS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create timeout.ts utility for dynamic timeout computation. Replace hardcoded timeouts across the stack: - tool-factory worker: 30s → dynamic based on megapixels - Python bridge default: 300s → 600s (or env override) - background-removal: fixed → dynamic based on image size - OCR: fixed 600s → dynamic based on image size - seam-carving: 120s → dynamic based on image size - ExifTool: 30s → 60s - HEIC converter: 30s → 120s - SQLite busy_timeout: 5s → 10s --- apps/api/src/db/index.ts | 2 +- apps/api/src/lib/exiftool.ts | 4 ++-- apps/api/src/lib/heic-converter.ts | 4 ++-- apps/api/src/lib/timeout.ts | 26 ++++++++++++++++++++++++++ apps/api/src/routes/tool-factory.ts | 6 +++++- packages/ai/src/background-removal.ts | 7 +++++-- packages/ai/src/bridge.ts | 12 ++++++++++-- packages/ai/src/ocr.ts | 6 +++++- packages/ai/src/seam-carving.ts | 4 +++- 9 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 apps/api/src/lib/timeout.ts diff --git a/apps/api/src/db/index.ts b/apps/api/src/db/index.ts index 33631ea4..977b37fd 100644 --- a/apps/api/src/db/index.ts +++ b/apps/api/src/db/index.ts @@ -13,7 +13,7 @@ const sqlite: DatabaseType = new Database(env.DB_PATH); // Critical SQLite pragmas for reliability. // busy_timeout must be set first so journal_mode = WAL can retry // if another connection holds the lock (e.g. parallel test files). -sqlite.pragma("busy_timeout = 5000"); +sqlite.pragma("busy_timeout = 10000"); sqlite.pragma("journal_mode = WAL"); sqlite.pragma("synchronous = NORMAL"); sqlite.pragma("foreign_keys = ON"); diff --git a/apps/api/src/lib/exiftool.ts b/apps/api/src/lib/exiftool.ts index 788c2b75..737a1b2e 100644 --- a/apps/api/src/lib/exiftool.ts +++ b/apps/api/src/lib/exiftool.ts @@ -51,7 +51,7 @@ export async function inspectMetadata(buffer: Buffer, filename: string): Promise try { await writeFile(tempPath, buffer); const { stdout } = await execFileAsync(bin, ["-json", "-G", "-struct", "-n", tempPath], { - timeout: 30_000, + timeout: 60_000, maxBuffer: 10 * 1024 * 1024, }); @@ -126,7 +126,7 @@ export async function writeMetadata( try { await writeFile(tempPath, buffer); await execFileAsync(bin, ["-overwrite_original", ...tags, tempPath], { - timeout: 30_000, + timeout: 60_000, maxBuffer: 10 * 1024 * 1024, }); return await readFile(tempPath); diff --git a/apps/api/src/lib/heic-converter.ts b/apps/api/src/lib/heic-converter.ts index 7d4a4b75..86df09eb 100644 --- a/apps/api/src/lib/heic-converter.ts +++ b/apps/api/src/lib/heic-converter.ts @@ -45,7 +45,7 @@ export async function decodeHeic(buffer: Buffer): Promise { try { await writeFile(inputPath, buffer); - await execFileAsync(cmd, [inputPath, outputPath], { timeout: 30_000 }); + await execFileAsync(cmd, [inputPath, outputPath], { timeout: 120_000 }); // Single-image HEIF: exact filename. Multi-image: -1 suffix on first image. try { @@ -94,7 +94,7 @@ export async function encodeHeic(buffer: Buffer, quality = 80): Promise try { await writeFile(inputPath, buffer); await execFileAsync("heif-enc", ["-q", String(quality), "-o", outputPath, inputPath], { - timeout: 30_000, + timeout: 120_000, }); return await readFile(outputPath); } finally { diff --git a/apps/api/src/lib/timeout.ts b/apps/api/src/lib/timeout.ts new file mode 100644 index 00000000..5806710b --- /dev/null +++ b/apps/api/src/lib/timeout.ts @@ -0,0 +1,26 @@ +import { env } from "../config.js"; + +type ToolCategory = "sharp" | "ai_cpu" | "ai_gpu" | "external" | "python"; + +const TIMEOUT_RATES: Record = { + sharp: 2, + ai_cpu: 30, + ai_gpu: 5, + external: 10, + python: 15, +}; + +export function computeTimeout(megapixels: number, category: ToolCategory, fileCount = 1): number { + if (env.PROCESSING_TIMEOUT_S > 0) { + return env.PROCESSING_TIMEOUT_S * 1000; + } + const perFile = Math.max(60_000, megapixels * TIMEOUT_RATES[category] * 1000); + return perFile * fileCount; +} + +export function computeExternalToolTimeout(megapixels: number): number { + if (env.PROCESSING_TIMEOUT_S > 0) { + return env.PROCESSING_TIMEOUT_S * 1000; + } + return Math.max(60_000, megapixels * TIMEOUT_RATES.external * 1000); +} diff --git a/apps/api/src/routes/tool-factory.ts b/apps/api/src/routes/tool-factory.ts index 3fcf718a..5821f11b 100644 --- a/apps/api/src/routes/tool-factory.ts +++ b/apps/api/src/routes/tool-factory.ts @@ -15,6 +15,7 @@ import { sanitizeFilename } from "../lib/filename.js"; import { decodeHeic } from "../lib/heic-converter.js"; import type { WorkerInput, WorkerOutput } from "../lib/image-worker.js"; import { sanitizeSvg } from "../lib/svg-sanitize.js"; +import { computeTimeout } from "../lib/timeout.js"; import { getWorkerPool } from "../lib/worker-pool.js"; import { createWorkspace } from "../lib/workspace.js"; @@ -227,8 +228,11 @@ export function createToolRoute(app: FastifyInstance, config: ToolRouteConfig filename, inputFormat: validation.format, }; + const meta = await sharp(fileBuffer).metadata(); + const megapixels = ((meta.width ?? 0) * (meta.height ?? 0)) / 1_000_000; + const timeoutMs = computeTimeout(megapixels, "sharp"); const workerResult: WorkerOutput = await pool.run(workerInput, { - signal: AbortSignal.timeout(30_000), + signal: AbortSignal.timeout(timeoutMs), }); result = { buffer: Buffer.from(workerResult.buffer), diff --git a/packages/ai/src/background-removal.ts b/packages/ai/src/background-removal.ts index 817f95a7..bf7a24a5 100644 --- a/packages/ai/src/background-removal.ts +++ b/packages/ai/src/background-removal.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { readFile, unlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import sharp from "sharp"; import { type ProgressCallback, parseStdoutJson, runPythonWithProgress } from "./bridge.js"; export interface RemoveBackgroundOptions { @@ -21,8 +22,10 @@ export async function removeBackground( await writeFile(inputPath, inputBuffer); try { - // BiRefNet models need longer timeout (up to 10 min for first load) - const timeout = options.model?.startsWith("birefnet") ? 600000 : 300000; + const meta = await sharp(inputBuffer).metadata(); + const megapixels = ((meta.width ?? 0) * (meta.height ?? 0)) / 1_000_000; + const baseTimeout = options.model?.startsWith("birefnet") ? 600000 : 300000; + const timeout = Math.max(baseTimeout, megapixels * 30 * 1000); const { stdout } = await runPythonWithProgress( "remove_bg.py", [inputPath, outputPath, JSON.stringify(options)], diff --git a/packages/ai/src/bridge.ts b/packages/ai/src/bridge.ts index 762af194..2f1d3002 100644 --- a/packages/ai/src/bridge.ts +++ b/packages/ai/src/bridge.ts @@ -218,7 +218,11 @@ function dispatcherRun( if (!proc || !proc.stdin || !dispatcherReady) return null; const id = randomUUID(); - const timeout = options.timeout ?? 300000; + const timeout = + options.timeout ?? + (process.env.PROCESSING_TIMEOUT_S && parseInt(process.env.PROCESSING_TIMEOUT_S, 10) > 0 + ? parseInt(process.env.PROCESSING_TIMEOUT_S, 10) * 1000 + : 600000); return new Promise((resolvePromise, rejectPromise) => { const timer = setTimeout(() => { @@ -278,7 +282,11 @@ function runPythonPerRequest( } = {}, ): Promise<{ stdout: string; stderr: string }> { const scriptPath = resolve(PYTHON_DIR, scriptName); - const timeout = options.timeout ?? 300000; + const timeout = + options.timeout ?? + (process.env.PROCESSING_TIMEOUT_S && parseInt(process.env.PROCESSING_TIMEOUT_S, 10) > 0 + ? parseInt(process.env.PROCESSING_TIMEOUT_S, 10) * 1000 + : 600000); return new Promise((resolvePromise, rejectPromise) => { const trySpawn = (pythonBin: string, isFallback: boolean) => { diff --git a/packages/ai/src/ocr.ts b/packages/ai/src/ocr.ts index 511154e6..c2695877 100644 --- a/packages/ai/src/ocr.ts +++ b/packages/ai/src/ocr.ts @@ -31,9 +31,13 @@ export async function extractText( const pngBuffer = await sharp(inputBuffer).png().toBuffer(); await writeFile(inputPath, pngBuffer); + const meta = await sharp(inputBuffer).metadata(); + const megapixels = ((meta.width ?? 0) * (meta.height ?? 0)) / 1_000_000; + const timeout = Math.max(600_000, megapixels * 30 * 1000); + const { stdout } = await runPythonWithProgress("ocr.py", [inputPath, JSON.stringify(options)], { onProgress, - timeout: 600_000, // 10 min timeout for VLM on CPU + timeout, }); const result = parseStdoutJson(stdout); diff --git a/packages/ai/src/seam-carving.ts b/packages/ai/src/seam-carving.ts index 1b1b070c..5f58ff30 100644 --- a/packages/ai/src/seam-carving.ts +++ b/packages/ai/src/seam-carving.ts @@ -122,7 +122,9 @@ export async function seamCarve( if (options.blurRadius !== undefined) args.push("-blur", String(options.blurRadius)); if (options.sobelThreshold !== undefined) args.push("-sobel", String(options.sobelThreshold)); - await execFileAsync(cairePath, args, { timeout: 120_000 }); + const megapixels = (origWidth * origHeight) / 1_000_000; + const timeoutMs = Math.max(120_000, megapixels * 10 * 1000); + await execFileAsync(cairePath, args, { timeout: timeoutMs }); const buffer = await readFile(outputPath); const outMeta = await sharp(buffer).metadata();