fix: reliable, self-healing AI feature-bundle installs (#472)

Make on-demand AI feature-bundle installs reliable and self-healing, closing
the failure modes behind most "some tool doesn't work" reports.

Multi-bundle installs: tools needing more than one bundle (Passport Photo,
Enhance Faces) install every required bundle from one action and stay
not-installed until all are present. Verified across all 19 AI tools.

Downloads: self-heal the accelerated Hugging Face (Xet) client so an upgraded
venv no longer silently falls back to slow urllib; restart instead of
corrupting a resumed partial when a proxy ignores Range and returns 200;
verify the completed size; fail fast on disk-full and HTTP 4xx; retry
transient errors five times; add hf_transfer fallback and document Xet egress.

Install integrity: crash-atomic venv writes so a killed or out-of-space
install can no longer tear the shared venv and break other tools; a boot
breadcrumb reseeds a torn venv to a clean state automatically; a post-install
smoke import test refuses to record a bundle whose libraries cannot load; an
install watchdog stops a wedged installer that would otherwise hold the venv
writer lock forever.

Adds unit and end-to-end tests for every failure mode above.

apps/api/src/lib/env.ts

@@ -108,6 +108,12 @@ const envSchema = z
     SYNC_WAIT_MS: z.coerce.number().default(8000),
     JOB_TIMEOUT_FAST_S: z.coerce.number().default(120),
     JOB_TIMEOUT_LONG_S: z.coerce.number().default(7200),
+    // AI bundle install watchdog. A wedged installer (dead download socket,
+    // hung pip) otherwise holds the venv writer lock forever and blocks every
+    // other install. STALL = max time with no progress frame before a kill;
+    // MAX = absolute wall-clock ceiling. 0 disables that check.
+    INSTALL_STALL_MS: z.coerce.number().default(1_200_000),
+    INSTALL_MAX_MS: z.coerce.number().default(7_200_000),
     JOBS_RETENTION_DAYS: z.coerce.number().default(30),
     AUDIT_RETENTION_DAYS: z.coerce.number().default(0),
     LOG_DIR: z.string().default("./data/logs"),

apps/api/src/lib/feature-status.ts

@@ -27,12 +27,16 @@ import { getQueuedBundleIds } from "./feature-install-queue.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PROJECT_ROOT = resolve(__dirname, "../../../..");
 
-const DATA_DIR = process.env.DATA_DIR || "/data";
+const DATA_DIR = process.env.DATA_DIR || "./data";
 const AI_DIR = join(DATA_DIR, "ai");
 const MODELS_DIR = join(AI_DIR, "models");
 const INSTALLED_PATH = join(AI_DIR, "installed.json");
 const INSTALLED_TMP_PATH = `${INSTALLED_PATH}.tmp`;
 const LOCK_PATH = join(AI_DIR, "install.lock");
+// Breadcrumb the installer drops right before it writes into the shared venv
+// site-packages and clears the instant that write completes. A survivor on boot
+// means the process died mid-write, so the venv may be torn (see move_tree).
+const VENV_WRITING_MARKER = join(AI_DIR, "venv.writing");
 const MANIFEST_PATH =
   process.env.FEATURE_MANIFEST_PATH || join(PROJECT_ROOT, "docker/feature-manifest.json");
 
@@ -61,10 +65,8 @@ export function ensureAiDirs(): void {
     mkdirSync(MODELS_DIR, { recursive: true });
     mkdirSync(join(AI_DIR, "pip-cache"), { recursive: true });
   } catch (err: unknown) {
-    // Never refuse to boot over the AI data dir. On native checkouts the
-    // default DATA_DIR (/data) is often uncreatable (ENOENT/EROFS on a
-    // sealed macOS root, EACCES on restrictive volumes); AI tools simply
-    // report as not installed until DATA_DIR points somewhere writable.
+    // Never refuse to boot over the AI data dir. AI tools simply report as not
+    // installed until DATA_DIR points somewhere writable.
     const code = (err as NodeJS.ErrnoException).code;
     console.error(
       `WARNING: Cannot create AI directories under "${AI_DIR}" (${code}). AI features will be unavailable. Set DATA_DIR to a writable path (or check volume permissions / PUID / PGID in Docker).`,
@@ -440,6 +442,37 @@ export function recoverInterruptedInstalls(): void {
     }
   }
 
+  // 4b. Heal a torn shared venv. If the venv-writing breadcrumb survived, an
+  // install died while rewriting the shared site-packages, which can leave a
+  // package half-replaced and break EVERY AI tool (not just the one installing).
+  // Model weight files under MODELS_DIR are unaffected, so the safe, automatic
+  // recovery is to reseed the venv from the image base and reset the install
+  // ledger; the user reinstalls bundles from a known-good state. This is the
+  // same repair as "Reset AI Environment", done automatically on next boot so a
+  // crash-broken install self-heals instead of leaving mysteriously dead tools.
+  if (existsSync(VENV_WRITING_MARKER)) {
+    if (isDockerEnvironment() && existsSync("/opt/venv")) {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write; reseeding the AI venv to a clean state and clearing installed bundles (reinstall to restore).",
+      );
+      try {
+        execFileSync("/usr/local/bin/reseed-ai-venv.sh", { stdio: "ignore", timeout: 120_000 });
+        writeInstalled({ bundles: {} });
+      } catch (err) {
+        console.error("[feature-status] Failed to reseed AI venv after interrupted install:", err);
+      }
+    } else {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write (non-Docker); the AI venv may be inconsistent. Reinstall the affected bundle.",
+      );
+    }
+    try {
+      unlinkSync(VENV_WRITING_MARKER);
+    } catch {
+      // best-effort
+    }
+  }
+
   // 5. Verify installed bundles still have their model files
   const manifest = readManifest();
   if (manifest) {

apps/api/src/lib/install-watchdog.ts

@@ -0,0 +1,42 @@
+/**
+ * Pure decision for the AI-bundle install watchdog.
+ *
+ * A wedged installer (dead download socket, hung pip) holds the venv writer lock
+ * forever and blocks every other install with no exit but a server restart. The
+ * watchdog kills such a child so its close handler frees the locks and the user
+ * gets a retryable failure. This function holds the (timer-free, side-effect
+ * free) rule so it can be unit-tested exhaustively; features.ts owns the timers.
+ *
+ * @param now            current time (ms)
+ * @param lastProgressAt time of the last progress frame (ms)
+ * @param startedAt      time the install started (ms)
+ * @param stallMs        max time with no progress before a kill (0 disables)
+ * @param maxMs          absolute wall-clock ceiling (0 disables)
+ */
+export function evaluateInstallWatchdog(
+  now: number,
+  lastProgressAt: number,
+  startedAt: number,
+  stallMs: number,
+  maxMs: number,
+): { kill: boolean; reason: string | null } {
+  const overMax = maxMs > 0 && now - startedAt > maxMs;
+  if (overMax) {
+    return {
+      kill: true,
+      reason: `Installation exceeded the ${Math.round(
+        maxMs / 60_000,
+      )} minute time limit and was stopped. Please retry.`,
+    };
+  }
+  const stalled = stallMs > 0 && now - lastProgressAt > stallMs;
+  if (stalled) {
+    return {
+      kill: true,
+      reason: `Installation made no progress for ${Math.round(
+        stallMs / 60_000,
+      )} minutes and was stopped. Check your connection and retry.`,
+    };
+  }
+  return { kill: false, reason: null };
+}

apps/api/src/routes/features.ts

@@ -2,11 +2,12 @@
  * Feature bundle management routes.
  *
  * GET  /api/v1/features                           - List feature bundles and their statuses
- * POST /api/v1/admin/features/:bundleId/install    - Install a feature bundle (async)
- * POST /api/v1/admin/features/:bundleId/uninstall  - Uninstall a feature bundle
- * POST /api/v1/admin/features/reset                - Wipe the AI venv/models, reset all bundles
- * GET  /api/v1/admin/features/disk-usage           - Get AI model disk usage
- * POST /api/v1/admin/features/import               - Import an offline bundle archive
+ * POST /api/v1/admin/features/:bundleId/install       - Install a feature bundle (async)
+ * POST /api/v1/admin/tools/:toolId/features/install   - Install every bundle a tool requires
+ * POST /api/v1/admin/features/:bundleId/uninstall     - Uninstall a feature bundle
+ * POST /api/v1/admin/features/reset                   - Wipe the AI venv/models, reset all bundles
+ * GET  /api/v1/admin/features/disk-usage              - Get AI model disk usage
+ * POST /api/v1/admin/features/import                  - Import an offline bundle archive
  */
 
 import { spawn } from "node:child_process";
@@ -22,8 +23,9 @@ import {
 } from "node:fs";
 import { join } from "node:path";
 import { acquireVenvLock, shutdownDispatcher } from "@snapotter/ai";
-import { ANALYTICS_EVENTS, FEATURE_BUNDLES } from "@snapotter/shared";
+import { ANALYTICS_EVENTS, FEATURE_BUNDLES, getRequiredBundlesForTool } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { env } from "../config.js";
 import { trackEvent } from "../lib/analytics.js";
 import {
   clearActive,
@@ -52,6 +54,7 @@ import {
   setInstallProgress,
   verifyBundleModels,
 } from "../lib/feature-status.js";
+import { evaluateInstallWatchdog } from "../lib/install-watchdog.js";
 import { requirePermission } from "../permissions.js";
 import { requireAuth } from "../plugins/auth.js";
 import { updateSingleFileProgress } from "./progress.js";
@@ -99,10 +102,32 @@ function startInstall(bundleId: string, jobId: string): void {
     // would release the file lock and active slot that pump() just handed to
     // the next queued bundle, letting two installers run into the same venv
     // at once (the corruption the lock exists to prevent).
+    // Install watchdog: a wedged installer (dead download socket, hung pip)
+    // otherwise holds the venv writer lock forever and blocks every other
+    // install with no way out but a server restart. Track the last progress
+    // frame; if the child goes silent past the stall budget, or blows the
+    // absolute ceiling, kill it so its close handler frees the locks and the
+    // user sees a retryable failure.
+    let lastProgressAt = Date.now();
+    let watchdog: ReturnType<typeof setInterval> | null = null;
+    let killGrace: ReturnType<typeof setTimeout> | null = null;
+    let watchdogError: string | null = null;
+    const clearWatchdog = () => {
+      if (watchdog) {
+        clearInterval(watchdog);
+        watchdog = null;
+      }
+      if (killGrace) {
+        clearTimeout(killGrace);
+        killGrace = null;
+      }
+    };
+
     let finalized = false;
     const finalizeOnce = (): boolean => {
       if (finalized) return false;
       finalized = true;
+      clearWatchdog();
       releaseVenvOnce();
       releaseInstallLock();
       clearActive();
@@ -118,6 +143,42 @@ function startInstall(bundleId: string, jobId: string): void {
       },
     });
 
+    const stallMs = env.INSTALL_STALL_MS;
+    const maxMs = env.INSTALL_MAX_MS;
+    if (stallMs > 0 || maxMs > 0) {
+      watchdog = setInterval(() => {
+        const verdict = evaluateInstallWatchdog(
+          Date.now(),
+          lastProgressAt,
+          installStartTime,
+          stallMs,
+          maxMs,
+        );
+        if (!verdict.kill) return;
+        watchdogError = verdict.reason;
+        setInstallProgress(bundleId, null, watchdogError);
+        if (watchdog) {
+          clearInterval(watchdog);
+          watchdog = null;
+        }
+        try {
+          child.kill("SIGTERM");
+        } catch {
+          // child already gone
+        }
+        // Escalate to SIGKILL if SIGTERM does not land (e.g. a C extension
+        // ignoring the signal); the close handler clears this grace timer.
+        killGrace = setTimeout(() => {
+          try {
+            child.kill("SIGKILL");
+          } catch {
+            // already gone
+          }
+        }, 10_000);
+      }, 30_000);
+      watchdog.unref?.();
+    }
+
     let stderrBuffer = "";
     let stdoutBuffer = "";
     const lastStderrLines: string[] = [];
@@ -142,6 +203,7 @@ function startInstall(bundleId: string, jobId: string): void {
         try {
           const parsed = JSON.parse(trimmed) as { progress?: number; stage?: string };
           if (typeof parsed.progress === "number") {
+            lastProgressAt = Date.now();
             setInstallProgress(
               bundleId,
               { percent: parsed.progress, stage: parsed.stage ?? "" },
@@ -174,10 +236,12 @@ function startInstall(bundleId: string, jobId: string): void {
           duration_ms: Date.now() - installStartTime,
         });
       } else {
+        // A watchdog kill wins: the child's exit code/stderr would otherwise
+        // read as a generic signal death and bury why it was stopped.
+        let errorMsg: string | undefined = watchdogError ?? undefined;
         // Extract the structured error from Python's fail() function first.
         // fail() writes {"error": "..."} to stderr - prefer this over raw lines.
-        let errorMsg: string | undefined;
-        for (let i = lastStderrLines.length - 1; i >= 0; i--) {
+        for (let i = lastStderrLines.length - 1; !errorMsg && i >= 0; i--) {
           const line = lastStderrLines[i];
           if (line.startsWith("{")) {
             try {
@@ -250,6 +314,23 @@ interface BundleIdParams {
   bundleId: string;
 }
 
+interface ToolIdParams {
+  toolId: string;
+}
+
+interface EnqueuedBundleInstall {
+  bundleId: string;
+  jobId: string;
+  queued: boolean;
+}
+
+interface ToolBundleInstallResult {
+  bundleId: string;
+  jobId?: string;
+  queued: boolean;
+  skipped?: boolean;
+}
+
 interface ManifestModel {
   id: string;
   path?: string;
@@ -275,6 +356,32 @@ function readManifest(): Manifest | null {
   }
 }
 
+function queueBundleInstallIfNeeded(bundleId: string): EnqueuedBundleInstall | null {
+  if (isFeatureInstalled(bundleId)) {
+    const modelError = verifyBundleModels(bundleId);
+    if (!modelError) return null;
+    markUninstalled(bundleId);
+  }
+
+  // Queue the install on the server so the POST is durable: enqueue()
+  // dedups an already-active/queued bundle and returns the effective jobId,
+  // then pump() starts it immediately if nothing else is installing. A bundle
+  // that lands behind another install stays queued server-side and starts
+  // automatically when the running install finishes.
+  const jobId = crypto.randomUUID();
+  const effectiveJobId = enqueue({ bundleId, jobId });
+  pump();
+
+  // queued === true means it did NOT start right now (another install is
+  // active, or an offline import holds the lock). The client polls queued
+  // bundles and opens SSE only for the active install.
+  return {
+    bundleId,
+    jobId: effectiveJobId,
+    queued: getActiveBundleId() !== bundleId,
+  };
+}
+
 /** Recursively calculate total size of a directory in bytes. */
 function getDirSize(dirPath: string): number {
   if (!existsSync(dirPath)) return 0;
@@ -348,30 +455,40 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
         return reply.status(404).send({ error: `Unknown bundle: ${bundleId}` });
       }
 
-      if (isFeatureInstalled(bundleId)) {
-        const modelError = verifyBundleModels(bundleId);
-        if (!modelError) {
-          return reply.status(409).send({ error: `Bundle "${bundleId}" is already installed` });
-        }
-        markUninstalled(bundleId);
+      const result = queueBundleInstallIfNeeded(bundleId);
+      if (!result) {
+        return reply.status(409).send({ error: `Bundle "${bundleId}" is already installed` });
       }
 
-      // Queue the install on the server so the POST is durable: enqueue()
-      // dedups an already-active/queued bundle and returns the effective jobId,
-      // then pump() starts it immediately if nothing else is installing. A
-      // bundle that lands behind another install stays queued server-side and
-      // starts automatically when the running install finishes.
-      const jobId = crypto.randomUUID();
-      const effectiveJobId = enqueue({ bundleId, jobId });
-      pump();
+      return reply.status(202).send({ jobId: result.jobId, queued: result.queued });
+    },
+  );
 
-      // queued === true means it did NOT start right now (another install is
-      // active, or an offline import holds the lock). The client still opens
-      // the SSE stream for the returned jobId; the progress route tolerates a
-      // not-yet-started jobId and streams once the installer begins.
-      const queued = getActiveBundleId() !== bundleId;
+  // POST /api/v1/admin/tools/:toolId/features/install - Install all bundles a tool needs
+  app.post(
+    "/api/v1/admin/tools/:toolId/features/install",
+    { config: { rateLimit: { max: 300, timeWindow: "1 minute" } } },
+    async (request: FastifyRequest<{ Params: ToolIdParams }>, reply: FastifyReply) => {
+      const admin = await requirePermission("features:manage")(request, reply);
+      if (!admin) return;
 
-      return reply.status(202).send({ jobId: effectiveJobId, queued });
+      const { toolId } = request.params;
+      const requiredBundles = getRequiredBundlesForTool(toolId);
+      if (requiredBundles.length === 0) {
+        return reply.status(404).send({ error: `No feature bundles required for tool: ${toolId}` });
+      }
+
+      const bundles: ToolBundleInstallResult[] = [];
+      for (const bundleId of requiredBundles) {
+        if (!FEATURE_BUNDLES[bundleId]) {
+          return reply.status(404).send({ error: `Unknown bundle: ${bundleId}` });
+        }
+
+        const result = queueBundleInstallIfNeeded(bundleId);
+        bundles.push(result ?? { bundleId, queued: false, skipped: true });
+      }
+
+      return reply.status(202).send({ bundles });
     },
   );
 

apps/api/src/routes/tool-factory.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { extname, join } from "node:path";
 import {
   apiToolPath,
-  getBundleForTool,
+  FEATURE_BUNDLES,
   type Section,
   TOOL_BUNDLE_MAP,
   TOOLS,
@@ -16,7 +16,7 @@ import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { enqueueToolJob, waitForJob } from "../jobs/enqueue.js";
 import { formatZodErrors, friendlyError, stripInternalPaths } from "../lib/errors.js";
-import { isToolInstalled } from "../lib/feature-status.js";
+import { getFirstMissingBundleForTool, isToolInstalled } from "../lib/feature-status.js";
 import { getObjectBuffer, putObject } from "../lib/object-storage.js";
 import { resolveToolPool, shouldSkipSyncWindow } from "../lib/pool.js";
 import { getSettingNumber } from "../lib/settings-helpers.js";
@@ -470,13 +470,14 @@ export function createToolRoute<T>(app: FastifyInstance, config: ToolRouteConfig
         // Guard: check if the tool's AI feature bundle is installed
         const bundleId = TOOL_BUNDLE_MAP[config.toolId];
         if (bundleId && !isToolInstalled(config.toolId)) {
-          const bundle = getBundleForTool(config.toolId);
+          const missingBundleId = getFirstMissingBundleForTool(config.toolId) ?? bundleId;
+          const bundle = FEATURE_BUNDLES[missingBundleId];
           // Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
           return reply.status(501).send({
             error: "Feature not installed",
             code: "FEATURE_NOT_INSTALLED",
-            feature: bundleId,
-            featureName: bundle?.name ?? bundleId,
+            feature: missingBundleId,
+            featureName: bundle?.name ?? missingBundleId,
             estimatedSize: bundle?.estimatedSize ?? "unknown",
           });
         }

apps/api/src/routes/tools/enhance-faces.ts

@@ -3,14 +3,14 @@ import { mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { enhanceFaces } from "@snapotter/ai";
-import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import { FEATURE_BUNDLES, TOOL_BUNDLE_MAP } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { z } from "zod";
 import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
 import { enqueueToolJob } from "../../jobs/enqueue.js";
 import { autoOrient } from "../../lib/auto-orient.js";
 import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
-import { isToolInstalled } from "../../lib/feature-status.js";
+import { getFirstMissingBundleForTool, isToolInstalled } from "../../lib/feature-status.js";
 import { validateImageBuffer } from "../../lib/file-validation.js";
 import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js";
 import { decodeHeic } from "../../lib/heic-converter.js";
@@ -64,12 +64,13 @@ export function registerEnhanceFaces(app: FastifyInstance) {
     async (request: FastifyRequest, reply: FastifyReply) => {
       const toolId = "enhance-faces";
       if (!isToolInstalled(toolId)) {
-        const bundle = getBundleForTool(toolId);
+        const missingBundleId = getFirstMissingBundleForTool(toolId) ?? TOOL_BUNDLE_MAP[toolId];
+        const bundle = FEATURE_BUNDLES[missingBundleId];
         return reply.status(501).send({
           error: "Feature not installed",
           code: "FEATURE_NOT_INSTALLED",
-          feature: TOOL_BUNDLE_MAP[toolId],
-          featureName: bundle?.name ?? toolId,
+          feature: missingBundleId,
+          featureName: bundle?.name ?? missingBundleId,
           estimatedSize: bundle?.estimatedSize ?? "unknown",
         });
       }