feat: add worker threads, persistent Python sidecar, graceful shutdown, and architectural improvements

- Graceful shutdown: SIGTERM/SIGINT handlers drain HTTP, stop workers, close DB
- Thumbnail caching: disk-cached thumbnails with immutable Cache-Control headers
- Worker thread pool: Piscina offloads Sharp processing off the main event loop
- Persistent Python dispatcher: pre-imports ML libraries, eliminates cold-start latency
- Tool page registry: declarative tool-to-component mapping replaces 750-line switch
- File store cleanup: remove dead derived fields, stable files array reference
- Job persistence: progress written to SQLite jobs table, stale jobs recovered on startup

apps/api/src/routes/progress.ts

@@ -4,8 +4,13 @@
  * GET /api/v1/jobs/:jobId/progress
  *
  * Sends Server-Sent Events with progress data until the job finishes.
+ *
+ * Progress is held in-memory for real-time SSE delivery and also
+ * persisted to the `jobs` table so that state survives container restarts.
  */
+import { eq } from "drizzle-orm";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { db, schema } from "../db/index.js";
 
 export interface JobProgress {
   jobId: string;
@@ -34,11 +39,128 @@ const jobProgressStore = new Map<string, JobProgress>();
 /** SSE listeners waiting for updates, keyed by jobId. */
 const listeners = new Map<string, Set<(data: JobProgress | SingleFileProgress) => void>>();
 
+// ── DB persistence helpers ──────────────────────────────────────────
+
+function persistJobProgress(progress: JobProgress): void {
+  try {
+    const completionRatio =
+      progress.totalFiles > 0 ? progress.completedFiles / progress.totalFiles : 0;
+    const existing = db
+      .select({ id: schema.jobs.id })
+      .from(schema.jobs)
+      .where(eq(schema.jobs.id, progress.jobId))
+      .get();
+
+    if (existing) {
+      db.update(schema.jobs)
+        .set({
+          status: progress.status,
+          progress: completionRatio,
+          error: progress.errors.length > 0 ? JSON.stringify(progress.errors) : null,
+          completedAt:
+            progress.status === "completed" || progress.status === "failed" ? new Date() : null,
+        })
+        .where(eq(schema.jobs.id, progress.jobId))
+        .run();
+    } else {
+      db.insert(schema.jobs)
+        .values({
+          id: progress.jobId,
+          type: "batch",
+          status: progress.status,
+          progress: completionRatio,
+          inputFiles: JSON.stringify({ totalFiles: progress.totalFiles }),
+          error: progress.errors.length > 0 ? JSON.stringify(progress.errors) : null,
+        })
+        .run();
+    }
+  } catch {
+    // DB persistence is best-effort; don't break real-time SSE
+  }
+}
+
+function persistSingleFileProgress(progress: Omit<SingleFileProgress, "type">): void {
+  try {
+    const status =
+      progress.phase === "complete"
+        ? "completed"
+        : progress.phase === "failed"
+          ? "failed"
+          : "processing";
+    const existing = db
+      .select({ id: schema.jobs.id })
+      .from(schema.jobs)
+      .where(eq(schema.jobs.id, progress.jobId))
+      .get();
+
+    if (existing) {
+      db.update(schema.jobs)
+        .set({
+          status,
+          progress: progress.percent / 100,
+          error: progress.error ?? null,
+          completedAt: status === "completed" || status === "failed" ? new Date() : null,
+        })
+        .where(eq(schema.jobs.id, progress.jobId))
+        .run();
+    } else {
+      db.insert(schema.jobs)
+        .values({
+          id: progress.jobId,
+          type: "single",
+          status,
+          progress: progress.percent / 100,
+          inputFiles: "[]",
+          error: progress.error ?? null,
+        })
+        .run();
+    }
+  } catch {
+    // Best-effort
+  }
+}
+
+/**
+ * Mark any jobs left in "processing" or "queued" state as failed.
+ * Called once at startup to recover from unclean shutdown.
+ */
+export function recoverStaleJobs(): void {
+  try {
+    const result = db
+      .update(schema.jobs)
+      .set({
+        status: "failed",
+        error: "Server restarted while job was in progress",
+        completedAt: new Date(),
+      })
+      .where(eq(schema.jobs.status, "processing"))
+      .run();
+    const result2 = db
+      .update(schema.jobs)
+      .set({
+        status: "failed",
+        error: "Server restarted while job was queued",
+        completedAt: new Date(),
+      })
+      .where(eq(schema.jobs.status, "queued"))
+      .run();
+    const total = result.changes + result2.changes;
+    if (total > 0) {
+      console.log(`Recovered ${total} stale jobs from previous run`);
+    }
+  } catch {
+    // DB not ready
+  }
+}
+
+// ── Public API (unchanged signatures) ───────────────────────────────
+
 /**
  * Create or update progress for a job.
  */
 export function updateJobProgress(progress: JobProgress): void {
   jobProgressStore.set(progress.jobId, progress);
+  persistJobProgress(progress);
   // Notify all SSE listeners
   const subs = listeners.get(progress.jobId);
   if (subs) {
@@ -57,6 +179,7 @@ export function updateJobProgress(progress: JobProgress): void {
 
 export function updateSingleFileProgress(progress: Omit<SingleFileProgress, "type">): void {
   const event: SingleFileProgress = { ...progress, type: "single" };
+  persistSingleFileProgress(progress);
   const subs = listeners.get(progress.jobId);
   if (subs) {
     for (const cb of subs) {