feat: add worker threads, persistent Python sidecar, graceful shutdown, and architectural improvements

- Graceful shutdown: SIGTERM/SIGINT handlers drain HTTP, stop workers, close DB
- Thumbnail caching: disk-cached thumbnails with immutable Cache-Control headers
- Worker thread pool: Piscina offloads Sharp processing off the main event loop
- Persistent Python dispatcher: pre-imports ML libraries, eliminates cold-start latency
- Tool page registry: declarative tool-to-component mapping replaces 750-line switch
- File store cleanup: remove dead derived fields, stable files array reference
- Job persistence: progress written to SQLite jobs table, stale jobs recovered on startup

apps/api/src/routes/progress.ts

@@ -4,8 +4,13 @@
  * GET /api/v1/jobs/:jobId/progress
  *
  * Sends Server-Sent Events with progress data until the job finishes.
+ *
+ * Progress is held in-memory for real-time SSE delivery and also
+ * persisted to the `jobs` table so that state survives container restarts.
  */
+import { eq } from "drizzle-orm";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { db, schema } from "../db/index.js";
 
 export interface JobProgress {
   jobId: string;
@@ -34,11 +39,128 @@ const jobProgressStore = new Map<string, JobProgress>();
 /** SSE listeners waiting for updates, keyed by jobId. */
 const listeners = new Map<string, Set<(data: JobProgress | SingleFileProgress) => void>>();
 
+// ── DB persistence helpers ──────────────────────────────────────────
+
+function persistJobProgress(progress: JobProgress): void {
+  try {
+    const completionRatio =
+      progress.totalFiles > 0 ? progress.completedFiles / progress.totalFiles : 0;
+    const existing = db
+      .select({ id: schema.jobs.id })
+      .from(schema.jobs)
+      .where(eq(schema.jobs.id, progress.jobId))
+      .get();
+
+    if (existing) {
+      db.update(schema.jobs)
+        .set({
+          status: progress.status,
+          progress: completionRatio,
+          error: progress.errors.length > 0 ? JSON.stringify(progress.errors) : null,
+          completedAt:
+            progress.status === "completed" || progress.status === "failed" ? new Date() : null,
+        })
+        .where(eq(schema.jobs.id, progress.jobId))
+        .run();
+    } else {
+      db.insert(schema.jobs)
+        .values({
+          id: progress.jobId,
+          type: "batch",
+          status: progress.status,
+          progress: completionRatio,
+          inputFiles: JSON.stringify({ totalFiles: progress.totalFiles }),
+          error: progress.errors.length > 0 ? JSON.stringify(progress.errors) : null,
+        })
+        .run();
+    }
+  } catch {
+    // DB persistence is best-effort; don't break real-time SSE
+  }
+}
+
+function persistSingleFileProgress(progress: Omit<SingleFileProgress, "type">): void {
+  try {
+    const status =
+      progress.phase === "complete"
+        ? "completed"
+        : progress.phase === "failed"
+          ? "failed"
+          : "processing";
+    const existing = db
+      .select({ id: schema.jobs.id })
+      .from(schema.jobs)
+      .where(eq(schema.jobs.id, progress.jobId))
+      .get();
+
+    if (existing) {
+      db.update(schema.jobs)
+        .set({
+          status,
+          progress: progress.percent / 100,
+          error: progress.error ?? null,
+          completedAt: status === "completed" || status === "failed" ? new Date() : null,
+        })
+        .where(eq(schema.jobs.id, progress.jobId))
+        .run();
+    } else {
+      db.insert(schema.jobs)
+        .values({
+          id: progress.jobId,
+          type: "single",
+          status,
+          progress: progress.percent / 100,
+          inputFiles: "[]",
+          error: progress.error ?? null,
+        })
+        .run();
+    }
+  } catch {
+    // Best-effort
+  }
+}
+
+/**
+ * Mark any jobs left in "processing" or "queued" state as failed.
+ * Called once at startup to recover from unclean shutdown.
+ */
+export function recoverStaleJobs(): void {
+  try {
+    const result = db
+      .update(schema.jobs)
+      .set({
+        status: "failed",
+        error: "Server restarted while job was in progress",
+        completedAt: new Date(),
+      })
+      .where(eq(schema.jobs.status, "processing"))
+      .run();
+    const result2 = db
+      .update(schema.jobs)
+      .set({
+        status: "failed",
+        error: "Server restarted while job was queued",
+        completedAt: new Date(),
+      })
+      .where(eq(schema.jobs.status, "queued"))
+      .run();
+    const total = result.changes + result2.changes;
+    if (total > 0) {
+      console.log(`Recovered ${total} stale jobs from previous run`);
+    }
+  } catch {
+    // DB not ready
+  }
+}
+
+// ── Public API (unchanged signatures) ───────────────────────────────
+
 /**
  * Create or update progress for a job.
  */
 export function updateJobProgress(progress: JobProgress): void {
   jobProgressStore.set(progress.jobId, progress);
+  persistJobProgress(progress);
   // Notify all SSE listeners
   const subs = listeners.get(progress.jobId);
   if (subs) {
@@ -57,6 +179,7 @@ export function updateJobProgress(progress: JobProgress): void {
 
 export function updateSingleFileProgress(progress: Omit<SingleFileProgress, "type">): void {
   const event: SingleFileProgress = { ...progress, type: "single" };
+  persistSingleFileProgress(progress);
   const subs = listeners.get(progress.jobId);
   if (subs) {
     for (const cb of subs) {

apps/api/src/routes/tool-factory.ts

@@ -9,6 +9,8 @@ import { db, schema } from "../db/index.js";
 import { autoOrient } from "../lib/auto-orient.js";
 import { validateImageBuffer } from "../lib/file-validation.js";
 import { sanitizeFilename } from "../lib/filename.js";
+import type { WorkerInput, WorkerOutput } from "../lib/image-worker.js";
+import { getWorkerPool } from "../lib/worker-pool.js";
 import { createWorkspace } from "../lib/workspace.js";
 
 export interface ToolRouteConfig<T> {
@@ -41,6 +43,16 @@ export interface AnyToolRouteConfig {
  */
 const toolRegistry = new Map<string, AnyToolRouteConfig>();
 
+/** Tools that use the Python bridge and should NOT be offloaded to workers. */
+const SKIP_WORKER_TOOLS = new Set([
+  "remove-background",
+  "upscale",
+  "ocr",
+  "blur-faces",
+  "erase-object",
+  "smart-crop",
+]);
+
 /**
  * Retrieve a registered tool config by its ID.
  */
@@ -151,12 +163,43 @@ export function createToolRoute<T>(app: FastifyInstance, config: ToolRouteConfig
         return reply.status(400).send({ error: "Settings must be valid JSON" });
       }
 
-      // Auto-orient based on EXIF metadata before processing.
-      const processBuffer = await autoOrient(fileBuffer);
-
-      // Process the image
+      // Process the image (worker thread or main thread)
       try {
-        const result = await config.process(processBuffer, settings, filename);
+        let result: { buffer: Buffer; filename: string; contentType: string };
+
+        // Offload to worker thread for non-AI tools.
+        // Falls back to main-thread processing on any worker error.
+        // Disabled in test environments where worker_threads can't load .ts files.
+        const useWorker = !SKIP_WORKER_TOOLS.has(config.toolId) && process.env.NODE_ENV !== "test";
+        if (useWorker) {
+          try {
+            const pool = getWorkerPool();
+            const workerInput: WorkerInput = {
+              toolId: config.toolId,
+              inputBuffer: fileBuffer,
+              settings,
+              filename,
+            };
+            const workerResult: WorkerOutput = await pool.run(workerInput);
+            result = {
+              buffer: Buffer.from(workerResult.buffer),
+              filename: workerResult.filename,
+              contentType: workerResult.contentType,
+            };
+          } catch (workerErr) {
+            // Worker failed - fall back to main-thread processing
+            request.log.warn(
+              { workerErr, toolId: config.toolId },
+              "Worker processing failed, falling back to main thread",
+            );
+            const processBuffer = await autoOrient(fileBuffer);
+            result = await config.process(processBuffer, settings, filename);
+          }
+        } else {
+          // AI tools: always main thread (they use Python bridge)
+          const processBuffer = await autoOrient(fileBuffer);
+          result = await config.process(processBuffer, settings, filename);
+        }
 
         // Create workspace and save output
         const jobId = randomUUID();

apps/api/src/routes/user-files.ts

@@ -17,7 +17,14 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import sharp from "sharp";
 import { db, schema, sqlite } from "../db/index.js";
 import { auditLog } from "../lib/audit.js";
-import { deleteStoredFile, getStoredFilePath, saveFile } from "../lib/file-storage.js";
+import {
+  deleteStoredFile,
+  deleteThumbnail,
+  getCachedThumbnail,
+  getStoredFilePath,
+  saveFile,
+  saveThumbnail,
+} from "../lib/file-storage.js";
 import { validateImageBuffer } from "../lib/file-validation.js";
 import { sanitizeFilename } from "../lib/filename.js";
 import { getAuthUser } from "../plugins/auth.js";
@@ -342,6 +349,15 @@ export async function userFileRoutes(app: FastifyInstance): Promise<void> {
         return reply.status(404).send({ error: "File not found" });
       }
 
+      // Serve from disk cache if available
+      const cached = await getCachedThumbnail(file.storedName);
+      if (cached) {
+        return reply
+          .header("Content-Type", "image/jpeg")
+          .header("Cache-Control", "public, max-age=86400, immutable")
+          .send(cached);
+      }
+
       const filePath = getStoredFilePath(file.storedName);
 
       try {
@@ -350,9 +366,12 @@ export async function userFileRoutes(app: FastifyInstance): Promise<void> {
           .jpeg({ quality: 80 })
           .toBuffer();
 
+        // Cache to disk (non-blocking, don't fail the request)
+        saveThumbnail(file.storedName, thumbnail).catch(() => {});
+
         return reply
           .header("Content-Type", "image/jpeg")
-          .header("Cache-Control", "public, max-age=86400")
+          .header("Cache-Control", "public, max-age=86400, immutable")
           .send(thumbnail);
       } catch {
         return reply.status(422).send({ error: "Could not generate thumbnail" });
@@ -412,6 +431,7 @@ export async function userFileRoutes(app: FastifyInstance): Promise<void> {
 
       for (const row of chainRows) {
         await deleteStoredFile(row.stored_name);
+        await deleteThumbnail(row.stored_name);
         db.delete(schema.userFiles).where(eq(schema.userFiles.id, row.id)).run();
         deletedCount++;
       }