fix: reliable, self-healing AI feature-bundle installs (#472)

Make on-demand AI feature-bundle installs reliable and self-healing, closing
the failure modes behind most "some tool doesn't work" reports.

Multi-bundle installs: tools needing more than one bundle (Passport Photo,
Enhance Faces) install every required bundle from one action and stay
not-installed until all are present. Verified across all 19 AI tools.

Downloads: self-heal the accelerated Hugging Face (Xet) client so an upgraded
venv no longer silently falls back to slow urllib; restart instead of
corrupting a resumed partial when a proxy ignores Range and returns 200;
verify the completed size; fail fast on disk-full and HTTP 4xx; retry
transient errors five times; add hf_transfer fallback and document Xet egress.

Install integrity: crash-atomic venv writes so a killed or out-of-space
install can no longer tear the shared venv and break other tools; a boot
breadcrumb reseeds a torn venv to a clean state automatically; a post-install
smoke import test refuses to record a bundle whose libraries cannot load; an
install watchdog stops a wedged installer that would otherwise hold the venv
writer lock forever.

Adds unit and end-to-end tests for every failure mode above.

apps/api/src/lib/env.ts

@@ -108,6 +108,12 @@ const envSchema = z
     SYNC_WAIT_MS: z.coerce.number().default(8000),
     JOB_TIMEOUT_FAST_S: z.coerce.number().default(120),
     JOB_TIMEOUT_LONG_S: z.coerce.number().default(7200),
+    // AI bundle install watchdog. A wedged installer (dead download socket,
+    // hung pip) otherwise holds the venv writer lock forever and blocks every
+    // other install. STALL = max time with no progress frame before a kill;
+    // MAX = absolute wall-clock ceiling. 0 disables that check.
+    INSTALL_STALL_MS: z.coerce.number().default(1_200_000),
+    INSTALL_MAX_MS: z.coerce.number().default(7_200_000),
     JOBS_RETENTION_DAYS: z.coerce.number().default(30),
     AUDIT_RETENTION_DAYS: z.coerce.number().default(0),
     LOG_DIR: z.string().default("./data/logs"),

apps/api/src/lib/feature-status.ts

@@ -27,12 +27,16 @@ import { getQueuedBundleIds } from "./feature-install-queue.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PROJECT_ROOT = resolve(__dirname, "../../../..");
 
-const DATA_DIR = process.env.DATA_DIR || "/data";
+const DATA_DIR = process.env.DATA_DIR || "./data";
 const AI_DIR = join(DATA_DIR, "ai");
 const MODELS_DIR = join(AI_DIR, "models");
 const INSTALLED_PATH = join(AI_DIR, "installed.json");
 const INSTALLED_TMP_PATH = `${INSTALLED_PATH}.tmp`;
 const LOCK_PATH = join(AI_DIR, "install.lock");
+// Breadcrumb the installer drops right before it writes into the shared venv
+// site-packages and clears the instant that write completes. A survivor on boot
+// means the process died mid-write, so the venv may be torn (see move_tree).
+const VENV_WRITING_MARKER = join(AI_DIR, "venv.writing");
 const MANIFEST_PATH =
   process.env.FEATURE_MANIFEST_PATH || join(PROJECT_ROOT, "docker/feature-manifest.json");
 
@@ -61,10 +65,8 @@ export function ensureAiDirs(): void {
     mkdirSync(MODELS_DIR, { recursive: true });
     mkdirSync(join(AI_DIR, "pip-cache"), { recursive: true });
   } catch (err: unknown) {
-    // Never refuse to boot over the AI data dir. On native checkouts the
-    // default DATA_DIR (/data) is often uncreatable (ENOENT/EROFS on a
-    // sealed macOS root, EACCES on restrictive volumes); AI tools simply
-    // report as not installed until DATA_DIR points somewhere writable.
+    // Never refuse to boot over the AI data dir. AI tools simply report as not
+    // installed until DATA_DIR points somewhere writable.
     const code = (err as NodeJS.ErrnoException).code;
     console.error(
       `WARNING: Cannot create AI directories under "${AI_DIR}" (${code}). AI features will be unavailable. Set DATA_DIR to a writable path (or check volume permissions / PUID / PGID in Docker).`,
@@ -440,6 +442,37 @@ export function recoverInterruptedInstalls(): void {
     }
   }
 
+  // 4b. Heal a torn shared venv. If the venv-writing breadcrumb survived, an
+  // install died while rewriting the shared site-packages, which can leave a
+  // package half-replaced and break EVERY AI tool (not just the one installing).
+  // Model weight files under MODELS_DIR are unaffected, so the safe, automatic
+  // recovery is to reseed the venv from the image base and reset the install
+  // ledger; the user reinstalls bundles from a known-good state. This is the
+  // same repair as "Reset AI Environment", done automatically on next boot so a
+  // crash-broken install self-heals instead of leaving mysteriously dead tools.
+  if (existsSync(VENV_WRITING_MARKER)) {
+    if (isDockerEnvironment() && existsSync("/opt/venv")) {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write; reseeding the AI venv to a clean state and clearing installed bundles (reinstall to restore).",
+      );
+      try {
+        execFileSync("/usr/local/bin/reseed-ai-venv.sh", { stdio: "ignore", timeout: 120_000 });
+        writeInstalled({ bundles: {} });
+      } catch (err) {
+        console.error("[feature-status] Failed to reseed AI venv after interrupted install:", err);
+      }
+    } else {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write (non-Docker); the AI venv may be inconsistent. Reinstall the affected bundle.",
+      );
+    }
+    try {
+      unlinkSync(VENV_WRITING_MARKER);
+    } catch {
+      // best-effort
+    }
+  }
+
   // 5. Verify installed bundles still have their model files
   const manifest = readManifest();
   if (manifest) {

apps/api/src/lib/install-watchdog.ts

@@ -0,0 +1,42 @@
+/**
+ * Pure decision for the AI-bundle install watchdog.
+ *
+ * A wedged installer (dead download socket, hung pip) holds the venv writer lock
+ * forever and blocks every other install with no exit but a server restart. The
+ * watchdog kills such a child so its close handler frees the locks and the user
+ * gets a retryable failure. This function holds the (timer-free, side-effect
+ * free) rule so it can be unit-tested exhaustively; features.ts owns the timers.
+ *
+ * @param now            current time (ms)
+ * @param lastProgressAt time of the last progress frame (ms)
+ * @param startedAt      time the install started (ms)
+ * @param stallMs        max time with no progress before a kill (0 disables)
+ * @param maxMs          absolute wall-clock ceiling (0 disables)
+ */
+export function evaluateInstallWatchdog(
+  now: number,
+  lastProgressAt: number,
+  startedAt: number,
+  stallMs: number,
+  maxMs: number,
+): { kill: boolean; reason: string | null } {
+  const overMax = maxMs > 0 && now - startedAt > maxMs;
+  if (overMax) {
+    return {
+      kill: true,
+      reason: `Installation exceeded the ${Math.round(
+        maxMs / 60_000,
+      )} minute time limit and was stopped. Please retry.`,
+    };
+  }
+  const stalled = stallMs > 0 && now - lastProgressAt > stallMs;
+  if (stalled) {
+    return {
+      kill: true,
+      reason: `Installation made no progress for ${Math.round(
+        stallMs / 60_000,
+      )} minutes and was stopped. Check your connection and retry.`,
+    };
+  }
+  return { kill: false, reason: null };
+}

apps/api/src/routes/features.ts

@@ -2,11 +2,12 @@
  * Feature bundle management routes.
  *
  * GET  /api/v1/features                           - List feature bundles and their statuses
- * POST /api/v1/admin/features/:bundleId/install    - Install a feature bundle (async)
- * POST /api/v1/admin/features/:bundleId/uninstall  - Uninstall a feature bundle
- * POST /api/v1/admin/features/reset                - Wipe the AI venv/models, reset all bundles
- * GET  /api/v1/admin/features/disk-usage           - Get AI model disk usage
- * POST /api/v1/admin/features/import               - Import an offline bundle archive
+ * POST /api/v1/admin/features/:bundleId/install       - Install a feature bundle (async)
+ * POST /api/v1/admin/tools/:toolId/features/install   - Install every bundle a tool requires
+ * POST /api/v1/admin/features/:bundleId/uninstall     - Uninstall a feature bundle
+ * POST /api/v1/admin/features/reset                   - Wipe the AI venv/models, reset all bundles
+ * GET  /api/v1/admin/features/disk-usage              - Get AI model disk usage
+ * POST /api/v1/admin/features/import                  - Import an offline bundle archive
  */
 
 import { spawn } from "node:child_process";
@@ -22,8 +23,9 @@ import {
 } from "node:fs";
 import { join } from "node:path";
 import { acquireVenvLock, shutdownDispatcher } from "@snapotter/ai";
-import { ANALYTICS_EVENTS, FEATURE_BUNDLES } from "@snapotter/shared";
+import { ANALYTICS_EVENTS, FEATURE_BUNDLES, getRequiredBundlesForTool } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { env } from "../config.js";
 import { trackEvent } from "../lib/analytics.js";
 import {
   clearActive,
@@ -52,6 +54,7 @@ import {
   setInstallProgress,
   verifyBundleModels,
 } from "../lib/feature-status.js";
+import { evaluateInstallWatchdog } from "../lib/install-watchdog.js";
 import { requirePermission } from "../permissions.js";
 import { requireAuth } from "../plugins/auth.js";
 import { updateSingleFileProgress } from "./progress.js";
@@ -99,10 +102,32 @@ function startInstall(bundleId: string, jobId: string): void {
     // would release the file lock and active slot that pump() just handed to
     // the next queued bundle, letting two installers run into the same venv
     // at once (the corruption the lock exists to prevent).
+    // Install watchdog: a wedged installer (dead download socket, hung pip)
+    // otherwise holds the venv writer lock forever and blocks every other
+    // install with no way out but a server restart. Track the last progress
+    // frame; if the child goes silent past the stall budget, or blows the
+    // absolute ceiling, kill it so its close handler frees the locks and the
+    // user sees a retryable failure.
+    let lastProgressAt = Date.now();
+    let watchdog: ReturnType<typeof setInterval> | null = null;
+    let killGrace: ReturnType<typeof setTimeout> | null = null;
+    let watchdogError: string | null = null;
+    const clearWatchdog = () => {
+      if (watchdog) {
+        clearInterval(watchdog);
+        watchdog = null;
+      }
+      if (killGrace) {
+        clearTimeout(killGrace);
+        killGrace = null;
+      }
+    };
+
     let finalized = false;
     const finalizeOnce = (): boolean => {
       if (finalized) return false;
       finalized = true;
+      clearWatchdog();
       releaseVenvOnce();
       releaseInstallLock();
       clearActive();
@@ -118,6 +143,42 @@ function startInstall(bundleId: string, jobId: string): void {
       },
     });
 
+    const stallMs = env.INSTALL_STALL_MS;
+    const maxMs = env.INSTALL_MAX_MS;
+    if (stallMs > 0 || maxMs > 0) {
+      watchdog = setInterval(() => {
+        const verdict = evaluateInstallWatchdog(
+          Date.now(),
+          lastProgressAt,
+          installStartTime,
+          stallMs,
+          maxMs,
+        );
+        if (!verdict.kill) return;
+        watchdogError = verdict.reason;
+        setInstallProgress(bundleId, null, watchdogError);
+        if (watchdog) {
+          clearInterval(watchdog);
+          watchdog = null;
+        }
+        try {
+          child.kill("SIGTERM");
+        } catch {
+          // child already gone
+        }
+        // Escalate to SIGKILL if SIGTERM does not land (e.g. a C extension
+        // ignoring the signal); the close handler clears this grace timer.
+        killGrace = setTimeout(() => {
+          try {
+            child.kill("SIGKILL");
+          } catch {
+            // already gone
+          }
+        }, 10_000);
+      }, 30_000);
+      watchdog.unref?.();
+    }
+
     let stderrBuffer = "";
     let stdoutBuffer = "";
     const lastStderrLines: string[] = [];
@@ -142,6 +203,7 @@ function startInstall(bundleId: string, jobId: string): void {
         try {
           const parsed = JSON.parse(trimmed) as { progress?: number; stage?: string };
           if (typeof parsed.progress === "number") {
+            lastProgressAt = Date.now();
             setInstallProgress(
               bundleId,
               { percent: parsed.progress, stage: parsed.stage ?? "" },
@@ -174,10 +236,12 @@ function startInstall(bundleId: string, jobId: string): void {
           duration_ms: Date.now() - installStartTime,
         });
       } else {
+        // A watchdog kill wins: the child's exit code/stderr would otherwise
+        // read as a generic signal death and bury why it was stopped.
+        let errorMsg: string | undefined = watchdogError ?? undefined;
         // Extract the structured error from Python's fail() function first.
         // fail() writes {"error": "..."} to stderr - prefer this over raw lines.
-        let errorMsg: string | undefined;
-        for (let i = lastStderrLines.length - 1; i >= 0; i--) {
+        for (let i = lastStderrLines.length - 1; !errorMsg && i >= 0; i--) {
           const line = lastStderrLines[i];
           if (line.startsWith("{")) {
             try {
@@ -250,6 +314,23 @@ interface BundleIdParams {
   bundleId: string;
 }
 
+interface ToolIdParams {
+  toolId: string;
+}
+
+interface EnqueuedBundleInstall {
+  bundleId: string;
+  jobId: string;
+  queued: boolean;
+}
+
+interface ToolBundleInstallResult {
+  bundleId: string;
+  jobId?: string;
+  queued: boolean;
+  skipped?: boolean;
+}
+
 interface ManifestModel {
   id: string;
   path?: string;
@@ -275,6 +356,32 @@ function readManifest(): Manifest | null {
   }
 }
 
+function queueBundleInstallIfNeeded(bundleId: string): EnqueuedBundleInstall | null {
+  if (isFeatureInstalled(bundleId)) {
+    const modelError = verifyBundleModels(bundleId);
+    if (!modelError) return null;
+    markUninstalled(bundleId);
+  }
+
+  // Queue the install on the server so the POST is durable: enqueue()
+  // dedups an already-active/queued bundle and returns the effective jobId,
+  // then pump() starts it immediately if nothing else is installing. A bundle
+  // that lands behind another install stays queued server-side and starts
+  // automatically when the running install finishes.
+  const jobId = crypto.randomUUID();
+  const effectiveJobId = enqueue({ bundleId, jobId });
+  pump();
+
+  // queued === true means it did NOT start right now (another install is
+  // active, or an offline import holds the lock). The client polls queued
+  // bundles and opens SSE only for the active install.
+  return {
+    bundleId,
+    jobId: effectiveJobId,
+    queued: getActiveBundleId() !== bundleId,
+  };
+}
+
 /** Recursively calculate total size of a directory in bytes. */
 function getDirSize(dirPath: string): number {
   if (!existsSync(dirPath)) return 0;
@@ -348,30 +455,40 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
         return reply.status(404).send({ error: `Unknown bundle: ${bundleId}` });
       }
 
-      if (isFeatureInstalled(bundleId)) {
-        const modelError = verifyBundleModels(bundleId);
-        if (!modelError) {
-          return reply.status(409).send({ error: `Bundle "${bundleId}" is already installed` });
-        }
-        markUninstalled(bundleId);
+      const result = queueBundleInstallIfNeeded(bundleId);
+      if (!result) {
+        return reply.status(409).send({ error: `Bundle "${bundleId}" is already installed` });
       }
 
-      // Queue the install on the server so the POST is durable: enqueue()
-      // dedups an already-active/queued bundle and returns the effective jobId,
-      // then pump() starts it immediately if nothing else is installing. A
-      // bundle that lands behind another install stays queued server-side and
-      // starts automatically when the running install finishes.
-      const jobId = crypto.randomUUID();
-      const effectiveJobId = enqueue({ bundleId, jobId });
-      pump();
+      return reply.status(202).send({ jobId: result.jobId, queued: result.queued });
+    },
+  );
 
-      // queued === true means it did NOT start right now (another install is
-      // active, or an offline import holds the lock). The client still opens
-      // the SSE stream for the returned jobId; the progress route tolerates a
-      // not-yet-started jobId and streams once the installer begins.
-      const queued = getActiveBundleId() !== bundleId;
+  // POST /api/v1/admin/tools/:toolId/features/install - Install all bundles a tool needs
+  app.post(
+    "/api/v1/admin/tools/:toolId/features/install",
+    { config: { rateLimit: { max: 300, timeWindow: "1 minute" } } },
+    async (request: FastifyRequest<{ Params: ToolIdParams }>, reply: FastifyReply) => {
+      const admin = await requirePermission("features:manage")(request, reply);
+      if (!admin) return;
 
-      return reply.status(202).send({ jobId: effectiveJobId, queued });
+      const { toolId } = request.params;
+      const requiredBundles = getRequiredBundlesForTool(toolId);
+      if (requiredBundles.length === 0) {
+        return reply.status(404).send({ error: `No feature bundles required for tool: ${toolId}` });
+      }
+
+      const bundles: ToolBundleInstallResult[] = [];
+      for (const bundleId of requiredBundles) {
+        if (!FEATURE_BUNDLES[bundleId]) {
+          return reply.status(404).send({ error: `Unknown bundle: ${bundleId}` });
+        }
+
+        const result = queueBundleInstallIfNeeded(bundleId);
+        bundles.push(result ?? { bundleId, queued: false, skipped: true });
+      }
+
+      return reply.status(202).send({ bundles });
     },
   );
 

apps/api/src/routes/tool-factory.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { extname, join } from "node:path";
 import {
   apiToolPath,
-  getBundleForTool,
+  FEATURE_BUNDLES,
   type Section,
   TOOL_BUNDLE_MAP,
   TOOLS,
@@ -16,7 +16,7 @@ import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { enqueueToolJob, waitForJob } from "../jobs/enqueue.js";
 import { formatZodErrors, friendlyError, stripInternalPaths } from "../lib/errors.js";
-import { isToolInstalled } from "../lib/feature-status.js";
+import { getFirstMissingBundleForTool, isToolInstalled } from "../lib/feature-status.js";
 import { getObjectBuffer, putObject } from "../lib/object-storage.js";
 import { resolveToolPool, shouldSkipSyncWindow } from "../lib/pool.js";
 import { getSettingNumber } from "../lib/settings-helpers.js";
@@ -470,13 +470,14 @@ export function createToolRoute<T>(app: FastifyInstance, config: ToolRouteConfig
         // Guard: check if the tool's AI feature bundle is installed
         const bundleId = TOOL_BUNDLE_MAP[config.toolId];
         if (bundleId && !isToolInstalled(config.toolId)) {
-          const bundle = getBundleForTool(config.toolId);
+          const missingBundleId = getFirstMissingBundleForTool(config.toolId) ?? bundleId;
+          const bundle = FEATURE_BUNDLES[missingBundleId];
           // Orphaned uploads/<jobId>/ dir will be cleaned by T10 TTL sweeper
           return reply.status(501).send({
             error: "Feature not installed",
             code: "FEATURE_NOT_INSTALLED",
-            feature: bundleId,
-            featureName: bundle?.name ?? bundleId,
+            feature: missingBundleId,
+            featureName: bundle?.name ?? missingBundleId,
             estimatedSize: bundle?.estimatedSize ?? "unknown",
           });
         }

apps/api/src/routes/tools/enhance-faces.ts

@@ -3,14 +3,14 @@ import { mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { enhanceFaces } from "@snapotter/ai";
-import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
+import { FEATURE_BUNDLES, TOOL_BUNDLE_MAP } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { z } from "zod";
 import { registerAiJobHandler } from "../../jobs/ai-handlers.js";
 import { enqueueToolJob } from "../../jobs/enqueue.js";
 import { autoOrient } from "../../lib/auto-orient.js";
 import { formatZodErrors, stripInternalPaths } from "../../lib/errors.js";
-import { isToolInstalled } from "../../lib/feature-status.js";
+import { getFirstMissingBundleForTool, isToolInstalled } from "../../lib/feature-status.js";
 import { validateImageBuffer } from "../../lib/file-validation.js";
 import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js";
 import { decodeHeic } from "../../lib/heic-converter.js";
@@ -64,12 +64,13 @@ export function registerEnhanceFaces(app: FastifyInstance) {
     async (request: FastifyRequest, reply: FastifyReply) => {
       const toolId = "enhance-faces";
       if (!isToolInstalled(toolId)) {
-        const bundle = getBundleForTool(toolId);
+        const missingBundleId = getFirstMissingBundleForTool(toolId) ?? TOOL_BUNDLE_MAP[toolId];
+        const bundle = FEATURE_BUNDLES[missingBundleId];
         return reply.status(501).send({
           error: "Feature not installed",
           code: "FEATURE_NOT_INSTALLED",
-          feature: TOOL_BUNDLE_MAP[toolId],
-          featureName: bundle?.name ?? toolId,
+          feature: missingBundleId,
+          featureName: bundle?.name ?? missingBundleId,
           estimatedSize: bundle?.estimatedSize ?? "unknown",
         });
       }

apps/docs/api/ai.md

@@ -45,17 +45,30 @@ A separate "docs" dispatcher profile replaces the AI allowlist with document-pro
 
 ## Feature Bundles
 
-Each AI tool requires a model bundle to be installed before use. Bundles are installed on demand via the admin UI or `install_feature.py`.
+AI models are packaged by shared dependency stack, not one archive per tool. A feature bundle can enable several tools when they use the same model family, Python wheels, or native libraries. This keeps the release Docker image smaller and avoids storing duplicate copies of the same background matting, face detection, OCR, restoration, and speech models.
 
-| Bundle | Size | Tools |
-|--------|------|-------|
-| `background-removal` | 4-5 GB | remove-background, passport-photo, transparency-fixer, background-replace, blur-background |
-| `face-detection` | 200-300 MB | blur-faces, red-eye-removal, smart-crop |
-| `object-eraser-colorize` | 1-2 GB | erase-object, colorize, ai-canvas-expand |
-| `upscale-enhance` | 5-6 GB | upscale, enhance-faces, noise-removal |
-| `photo-restoration` | 4-5 GB | restore-photo |
-| `ocr` | 5-6 GB | ocr, ocr-pdf |
-| `transcription` | ~600 MB | transcribe-audio, auto-subtitles |
+The Docker image ships the application plus the common runtime. Large model archives are downloaded on demand into the persistent `/data/ai` volume, then reused by every tool that needs them. If a bundle is already installed because another tool needed it, enabling a new dependent tool does not download that bundle again.
+
+Each AI tool requires one or more feature bundles before it can run. The admin UI installs by tool through `POST /api/v1/admin/tools/:toolId/features/install`, which resolves the full bundle list, skips bundles that are already installed, and queues only the missing downloads. For example, enabling Passport Photo on a fresh instance queues `background-removal` and `face-detection`; enabling it after Background Removal is already installed queues only `face-detection`.
+
+| Bundle | Size | Shared dependency group | Tools that use it |
+|--------|------|-------------------------|-------------------|
+| `background-removal` | 4-5 GB | rembg / BiRefNet background matting | remove-background, passport-photo, transparency-fixer, background-replace, blur-background |
+| `face-detection` | 200-300 MB | MediaPipe face detection and landmarks | blur-faces, red-eye-removal, smart-crop |
+| `object-eraser-colorize` | 1-2 GB | LaMa inpainting/outpainting and DDColor | erase-object, colorize, ai-canvas-expand |
+| `upscale-enhance` | 5-6 GB | RealESRGAN, GFPGAN / CodeFormer, denoising | upscale, enhance-faces, noise-removal |
+| `photo-restoration` | 4-5 GB | scratch repair and restoration pipeline | restore-photo |
+| `ocr` | 5-6 GB | PaddleOCR / Tesseract OCR stack | ocr, ocr-pdf |
+| `transcription` | ~600 MB | faster-whisper speech-to-text models | transcribe-audio, auto-subtitles |
+
+Tools with cross-bundle dependencies:
+
+| Tool | Required bundles | Why |
+|------|------------------|-----|
+| `passport-photo` | `background-removal`, `face-detection` | Removes the background, then uses face landmarks to frame the crop to passport and ID photo rules. |
+| `enhance-faces` | `upscale-enhance`, `face-detection` | Detects faces before running GFPGAN or CodeFormer enhancement on the selected face regions. |
+
+A tool is available only when all of its required bundles are installed. Partial installs are valid and are handled incrementally: installed bundles are reused, missing bundles are shown as downloads, and queued installs run one at a time so the shared Python environment is not modified concurrently.
 
 ---
 

apps/docs/api/rest.md

@@ -589,12 +589,13 @@ Query parameters:
 
 ## Features / AI Bundles
 
-Manage AI feature bundles (install/uninstall AI model packages in the Docker environment).
+Manage AI feature bundles (install/uninstall AI model packages in the Docker environment). Prefer the tool-level install endpoint when enabling a tool from custom automation: some AI tools need more than one shared bundle, and this endpoint skips already-installed bundles while queuing only the missing ones.
 
 | Method | Path | Access | Description |
 |--------|------|--------|-------------|
 | `GET` | `/api/v1/features` | Auth | List all feature bundles and their install status |
 | `POST` | `/api/v1/admin/features/:bundleId/install` | Admin (`features:manage`) | Install a feature bundle (async, returns `jobId` for progress tracking) |
+| `POST` | `/api/v1/admin/tools/:toolId/features/install` | Admin (`features:manage`) | Install every bundle a tool requires; returns per-bundle queued/skipped status |
 | `POST` | `/api/v1/admin/features/:bundleId/uninstall` | Admin (`features:manage`) | Uninstall a feature bundle and clean up model files |
 | `GET` | `/api/v1/admin/features/disk-usage` | Admin (`features:manage`) | Get total disk usage of AI models |
 | `POST` | `/api/v1/admin/features/import` | Admin (`features:manage`) | Import an offline AI bundle archive |

apps/docs/guide/deployment.md

@@ -264,6 +264,10 @@ Most AI tools are perfectly usable on CPU; a couple really want a GPU. Measured
 | AI upscale (RealESRGAN) | ~33 s small; minutes on large images | Marginal — GPU strongly recommended |
 | Photo restoration (full pipeline) | several minutes | No — needs a GPU or a fast many-core CPU |
 
+SnapOtter intentionally does not bake these model downloads into the Docker image. AI bundles are pulled only when an admin enables the related tool, stored in the persistent `/data/ai` volume, and shared by every tool that depends on the same model stack. This keeps the final container image small while still letting a full AI installation reach the larger storage numbers below.
+
+Some tools depend on more than one shared bundle. For example, Passport Photo needs both `background-removal` and `face-detection`; if `background-removal` is already installed, enabling Passport Photo only downloads the missing `face-detection` bundle. The same reuse applies across all AI tools.
+
 AI model download sizes:
 
 | Bundle | Disk Size |

apps/docs/guide/security.md

@@ -122,16 +122,18 @@ During normal operation, the container makes **zero outbound network connections
 Browser  -->  Reverse Proxy (TLS)  -->  SnapOtter container  -->  (nothing)
 ```
 
-The only exception is **AI model downloads**: when a user installs an AI feature bundle through the UI, the container downloads model files from GitHub Releases and PyPI. These downloads happen once per bundle and are stored in the `/data` volume.
+The only exception is **AI model downloads**: when a user installs an AI feature bundle through the UI, the container downloads the pre-built bundle archive from Hugging Face, plus a few individual model files from GitHub Releases, Google Storage, and PyPI. These downloads happen once per bundle and are stored in the `/data` volume.
 
 **Firewall recommendations:**
 
 | Scenario | Outbound rule |
 |---|---|
 | Air-gapped (no AI) | Block all outbound traffic from the container |
-| AI bundles needed | Allow HTTPS to `github.com`, `objects.githubusercontent.com`, `pypi.org`, `files.pythonhosted.org` during install, then block |
+| AI bundles needed | Allow HTTPS to `huggingface.co`, `*.xethub.hf.co`, `cdn-lfs.huggingface.co`, `github.com`, `objects.githubusercontent.com`, `storage.googleapis.com`, `pypi.org`, `files.pythonhosted.org` during install, then block |
 | After AI install | Block all outbound traffic - models are cached locally |
 
+Bundle archives are served from Hugging Face's Xet storage, which transfers over the `*.xethub.hf.co` endpoints in parallel and is what makes multi-GB bundle downloads fast. If your firewall allows `huggingface.co` but blocks `*.xethub.hf.co`, installs still succeed but fall back to a slower single-stream download, so allowlist the Xet hosts to stay on the fast path. Fully offline installs can skip all of this and use [Offline Bundle Import](/guide/deployment) instead.
+
 For reverse proxy configuration (Nginx, Traefik, Caddy, Cloudflare Tunnels), see the [Deployment guide](/guide/deployment#reverse-proxy).
 
 ## Docker Secrets

apps/web/src/components/common/tool-card.tsx

@@ -1,5 +1,10 @@
 import type { Tool } from "@snapotter/shared";
-import { PYTHON_SIDECAR_TOOLS, SECTIONS, TOOL_BUNDLE_MAP, toolSection } from "@snapotter/shared";
+import {
+  getRequiredBundlesForTool,
+  PYTHON_SIDECAR_TOOLS,
+  SECTIONS,
+  toolSection,
+} from "@snapotter/shared";
 import { Clock, Download, FileImage, Loader2, Pin } from "lucide-react";
 import { useMemo } from "react";
 import { Link } from "react-router-dom";
@@ -63,12 +68,15 @@ export function ToolCard({ tool, variant = "compact", showModalityBadge, showPin
   const queued = useFeaturesStore((s) => s.queued);
   const aiStatus = useMemo(() => {
     if (!isAiTool) return "installed";
-    const bundleId = TOOL_BUNDLE_MAP[tool.id];
-    if (!bundleId) return "installed";
-    if (queued.includes(bundleId)) return "queued";
-    if (installing[bundleId]) return "installing";
-    const bundle = bundles.find((b) => b.id === bundleId);
-    return bundle?.status === "installed" ? "installed" : "not_installed";
+    const requiredBundleIds = getRequiredBundlesForTool(tool.id);
+    if (requiredBundleIds.length === 0) return "installed";
+    if (requiredBundleIds.some((bundleId) => queued.includes(bundleId))) return "queued";
+    if (requiredBundleIds.some((bundleId) => installing[bundleId])) return "installing";
+    return requiredBundleIds.every(
+      (bundleId) => bundles.find((bundle) => bundle.id === bundleId)?.status === "installed",
+    )
+      ? "installed"
+      : "not_installed";
   }, [isAiTool, tool.id, bundles, installing, queued]);
 
   const section = toolSection(tool);

apps/web/src/components/features/feature-install-prompt.tsx

@@ -1,4 +1,8 @@
-import type { FeatureBundleState } from "@snapotter/shared";
+import {
+  FEATURE_BUNDLES,
+  type FeatureBundleState,
+  getRequiredBundlesForTool,
+} from "@snapotter/shared";
 import { AlertCircle, Clock, Download, Loader2, RotateCcw } from "lucide-react";
 import { useEffect, useState } from "react";
 import { useTranslation } from "@/contexts/i18n-context";
@@ -48,18 +52,45 @@ function formatTimeRemaining(ms: number): string {
 interface FeatureInstallPromptProps {
   bundle: FeatureBundleState;
   isAdmin: boolean;
+  toolId?: string;
   toolName?: string;
   toolDescription?: string;
 }
 
+function fallbackBundleState(bundleId: string): FeatureBundleState | null {
+  const info = FEATURE_BUNDLES[bundleId];
+  if (!info) return null;
+  return {
+    id: info.id,
+    name: info.name,
+    description: info.description,
+    status: "not_installed",
+    installedVersion: null,
+    estimatedSize: info.estimatedSize,
+    enablesTools: info.enablesTools,
+    progress: null,
+    error: null,
+  };
+}
+
 export function FeatureInstallPrompt({
   bundle,
   isAdmin,
+  toolId,
   toolName,
   toolDescription,
 }: FeatureInstallPromptProps) {
   const { t } = useTranslation();
-  const { installBundle, clearError, installing, errors, startTimes, queued } = useFeaturesStore();
+  const {
+    bundles,
+    installBundle,
+    installTool,
+    clearError,
+    installing,
+    errors,
+    startTimes,
+    queued,
+  } = useFeaturesStore();
   const progress = installing[bundle.id] ?? null;
   const error = errors[bundle.id] ?? null;
   const isInstalling = !!progress;
@@ -68,6 +99,37 @@ export function FeatureInstallPrompt({
   const displayName = toolName || bundle.name;
   const displayDescription = toolDescription || bundle.description;
   const isRepair = bundle.status === "error";
+  const requiredBundleIds = toolId ? getRequiredBundlesForTool(toolId) : [bundle.id];
+  const requiredBundles = requiredBundleIds
+    .map(
+      (bundleId) =>
+        bundles.find((candidate) => candidate.id === bundleId) ??
+        (bundle.id === bundleId ? bundle : fallbackBundleState(bundleId)),
+    )
+    .filter((candidate): candidate is FeatureBundleState => candidate !== null);
+  const bundlesNeedingDownload = requiredBundles.filter(
+    (candidate) => candidate.status !== "installed",
+  );
+  const downloadSizeLabel =
+    bundlesNeedingDownload
+      .map((candidate) =>
+        candidate.downloadBytes ? formatFileSize(candidate.downloadBytes) : candidate.estimatedSize,
+      )
+      .join(" + ") ||
+    (bundle.downloadBytes ? formatFileSize(bundle.downloadBytes) : bundle.estimatedSize);
+  // Show the per-bundle breakdown for any multi-bundle tool, including the
+  // repair state: when one bundle of a multi-bundle tool (e.g. passport-photo)
+  // errors, the user still needs to see that the sibling bundle is installing,
+  // queued, or already done. Hiding it during repair is exactly when it hurts.
+  const showBundleBreakdown = toolId !== undefined && requiredBundles.length > 1;
+
+  function bundleStatusLabel(candidate: FeatureBundleState): string {
+    if (installing[candidate.id]) return t.settings.aiFeatures.installing;
+    if (queued.includes(candidate.id)) return t.settings.aiFeatures.queued;
+    if (candidate.status === "installed") return t.settings.aiFeatures.installed;
+    if (candidate.status === "error") return t.settings.aiFeatures.repair;
+    return t.settings.aiFeatures.notInstalled;
+  }
 
   const [messageIndex, setMessageIndex] = useState(() =>
     Math.floor(Math.random() * PROGRESS_MESSAGES.length),
@@ -94,7 +156,11 @@ export function FeatureInstallPrompt({
 
   function handleInstall() {
     clearError(bundle.id);
-    installBundle(bundle.id);
+    if (toolId) {
+      installTool(toolId);
+    } else {
+      installBundle(bundle.id);
+    }
   }
 
   // Defensive guard: if the bundle is already installed (status may have
@@ -126,14 +192,49 @@ export function FeatureInstallPrompt({
         {!isRepair && (
           <p className="text-sm text-muted-foreground">
             {format(t.features.requiresDownload, {
-              size: bundle.downloadBytes
-                ? formatFileSize(bundle.downloadBytes)
-                : bundle.estimatedSize,
+              size: downloadSizeLabel,
             })}
           </p>
         )}
       </div>
 
+      {showBundleBreakdown && (
+        <div className="w-full max-w-md rounded-lg border border-border bg-background text-start overflow-hidden">
+          {requiredBundles.map((candidate) => {
+            const isCandidateInstalling = !!installing[candidate.id];
+            const isCandidateQueued = queued.includes(candidate.id);
+            const isCandidateInstalled = candidate.status === "installed";
+            const statusClass = isCandidateInstalled
+              ? "bg-emerald-500/10 text-emerald-700 dark:text-emerald-300"
+              : isCandidateInstalling || isCandidateQueued
+                ? "bg-primary/10 text-primary"
+                : candidate.status === "error"
+                  ? "bg-destructive/10 text-destructive"
+                  : "bg-muted text-muted-foreground";
+            return (
+              <div
+                key={candidate.id}
+                className="flex items-center justify-between gap-3 px-4 py-3 border-b border-border last:border-b-0"
+              >
+                <div className="min-w-0">
+                  <p className="text-sm font-medium text-foreground truncate">{candidate.name}</p>
+                  <p className="text-xs text-muted-foreground">
+                    {candidate.downloadBytes
+                      ? formatFileSize(candidate.downloadBytes)
+                      : candidate.estimatedSize}
+                  </p>
+                </div>
+                <span
+                  className={`shrink-0 rounded-full px-2.5 py-1 text-xs font-medium ${statusClass}`}
+                >
+                  {bundleStatusLabel(candidate)}
+                </span>
+              </div>
+            );
+          })}
+        </div>
+      )}
+
       {(error || (isRepair && bundle.error)) && (
         <div className="flex items-center gap-2 bg-destructive/10 text-destructive rounded-lg px-4 py-3 max-w-md w-full">
           <AlertCircle className="h-4 w-4 shrink-0" />

apps/web/src/pages/tool-page.tsx

@@ -600,6 +600,7 @@ export function ToolPage() {
           <FeatureInstallPrompt
             bundle={featureBundle}
             isAdmin={isAdmin}
+            toolId={toolId}
             toolName={tool?.name}
             toolDescription={tool?.description}
           />

apps/web/src/stores/features-store.ts

@@ -20,6 +20,13 @@ interface BundleProgress {
   stage: string;
 }
 
+interface ToolBundleInstallResult {
+  bundleId: string;
+  jobId?: string;
+  queued?: boolean;
+  skipped?: boolean;
+}
+
 interface FeaturesState {
   bundles: FeatureBundleState[];
   loaded: boolean;
@@ -35,6 +42,7 @@ interface FeaturesState {
   isToolInstalled: (toolId: string) => boolean;
   getBundleForTool: (toolId: string) => FeatureBundleState | null;
   installBundle: (bundleId: string) => Promise<void>;
+  installTool: (toolId: string) => Promise<void>;
   uninstallBundle: (bundleId: string) => Promise<void>;
   reinstallBundle: (bundleId: string) => Promise<void>;
   installAll: () => Promise<void>;
@@ -346,6 +354,74 @@ export const useFeaturesStore = create<FeaturesState>((set, get) => {
       }
     },
 
+    installTool: async (toolId: string) => {
+      const required = requiredBundlesForTool(toolId);
+      const targets = required.filter(
+        (bundleId) => get().bundles.find((b) => b.id === bundleId)?.status !== "installed",
+      );
+      if (targets.length === 0) return;
+
+      const errors = { ...get().errors };
+      const installing = { ...get().installing };
+      const startTimes = { ...get().startTimes };
+      const now = Date.now();
+
+      for (const bundleId of targets) {
+        delete errors[bundleId];
+        if (!get().queued.includes(bundleId)) {
+          installing[bundleId] = installing[bundleId] ?? { percent: 5, stage: "Starting..." };
+        }
+        startTimes[bundleId] = startTimes[bundleId] ?? now;
+      }
+      set({ errors, installing, startTimes });
+
+      try {
+        const result = await apiPost<{ bundles: ToolBundleInstallResult[] }>(
+          `/v1/admin/tools/${toolId}/features/install`,
+          {},
+        );
+
+        for (const item of result.bundles) {
+          if (item.skipped) {
+            stopTracking(item.bundleId);
+            continue;
+          }
+
+          if (item.queued) {
+            const nextInstalling = { ...get().installing };
+            delete nextInstalling[item.bundleId];
+            set({
+              installing: nextInstalling,
+              queued: get().queued.includes(item.bundleId)
+                ? get().queued
+                : [...get().queued, item.bundleId],
+            });
+            startPolling(item.bundleId);
+            continue;
+          }
+
+          if (item.jobId) {
+            listenToProgress(item.bundleId, item.jobId);
+          }
+        }
+
+        if (result.bundles.every((item) => item.skipped)) {
+          await refreshBundles();
+        }
+      } catch (err) {
+        const message = err instanceof Error ? err.message : "Failed to start installation";
+        const nextErrors = { ...get().errors };
+
+        for (const bundleId of targets) {
+          stopTracking(bundleId);
+          nextErrors[bundleId] = message;
+        }
+
+        set({ errors: nextErrors });
+        maybeFinishInstallAll();
+      }
+    },
+
     uninstallBundle: async (bundleId: string) => {
       try {
         await apiPost(`/v1/admin/features/${bundleId}/uninstall`, {});