fix: reliable, self-healing AI feature-bundle installs (#472)

Make on-demand AI feature-bundle installs reliable and self-healing, closing
the failure modes behind most "some tool doesn't work" reports.

Multi-bundle installs: tools needing more than one bundle (Passport Photo,
Enhance Faces) install every required bundle from one action and stay
not-installed until all are present. Verified across all 19 AI tools.

Downloads: self-heal the accelerated Hugging Face (Xet) client so an upgraded
venv no longer silently falls back to slow urllib; restart instead of
corrupting a resumed partial when a proxy ignores Range and returns 200;
verify the completed size; fail fast on disk-full and HTTP 4xx; retry
transient errors five times; add hf_transfer fallback and document Xet egress.

Install integrity: crash-atomic venv writes so a killed or out-of-space
install can no longer tear the shared venv and break other tools; a boot
breadcrumb reseeds a torn venv to a clean state automatically; a post-install
smoke import test refuses to record a bundle whose libraries cannot load; an
install watchdog stops a wedged installer that would otherwise hold the venv
writer lock forever.

Adds unit and end-to-end tests for every failure mode above.

apps/api/src/lib/env.ts

@@ -108,6 +108,12 @@ const envSchema = z
     SYNC_WAIT_MS: z.coerce.number().default(8000),
     JOB_TIMEOUT_FAST_S: z.coerce.number().default(120),
     JOB_TIMEOUT_LONG_S: z.coerce.number().default(7200),
+    // AI bundle install watchdog. A wedged installer (dead download socket,
+    // hung pip) otherwise holds the venv writer lock forever and blocks every
+    // other install. STALL = max time with no progress frame before a kill;
+    // MAX = absolute wall-clock ceiling. 0 disables that check.
+    INSTALL_STALL_MS: z.coerce.number().default(1_200_000),
+    INSTALL_MAX_MS: z.coerce.number().default(7_200_000),
     JOBS_RETENTION_DAYS: z.coerce.number().default(30),
     AUDIT_RETENTION_DAYS: z.coerce.number().default(0),
     LOG_DIR: z.string().default("./data/logs"),

apps/api/src/lib/feature-status.ts

@@ -27,12 +27,16 @@ import { getQueuedBundleIds } from "./feature-install-queue.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PROJECT_ROOT = resolve(__dirname, "../../../..");
 
-const DATA_DIR = process.env.DATA_DIR || "/data";
+const DATA_DIR = process.env.DATA_DIR || "./data";
 const AI_DIR = join(DATA_DIR, "ai");
 const MODELS_DIR = join(AI_DIR, "models");
 const INSTALLED_PATH = join(AI_DIR, "installed.json");
 const INSTALLED_TMP_PATH = `${INSTALLED_PATH}.tmp`;
 const LOCK_PATH = join(AI_DIR, "install.lock");
+// Breadcrumb the installer drops right before it writes into the shared venv
+// site-packages and clears the instant that write completes. A survivor on boot
+// means the process died mid-write, so the venv may be torn (see move_tree).
+const VENV_WRITING_MARKER = join(AI_DIR, "venv.writing");
 const MANIFEST_PATH =
   process.env.FEATURE_MANIFEST_PATH || join(PROJECT_ROOT, "docker/feature-manifest.json");
 
@@ -61,10 +65,8 @@ export function ensureAiDirs(): void {
     mkdirSync(MODELS_DIR, { recursive: true });
     mkdirSync(join(AI_DIR, "pip-cache"), { recursive: true });
   } catch (err: unknown) {
-    // Never refuse to boot over the AI data dir. On native checkouts the
-    // default DATA_DIR (/data) is often uncreatable (ENOENT/EROFS on a
-    // sealed macOS root, EACCES on restrictive volumes); AI tools simply
-    // report as not installed until DATA_DIR points somewhere writable.
+    // Never refuse to boot over the AI data dir. AI tools simply report as not
+    // installed until DATA_DIR points somewhere writable.
     const code = (err as NodeJS.ErrnoException).code;
     console.error(
       `WARNING: Cannot create AI directories under "${AI_DIR}" (${code}). AI features will be unavailable. Set DATA_DIR to a writable path (or check volume permissions / PUID / PGID in Docker).`,
@@ -440,6 +442,37 @@ export function recoverInterruptedInstalls(): void {
     }
   }
 
+  // 4b. Heal a torn shared venv. If the venv-writing breadcrumb survived, an
+  // install died while rewriting the shared site-packages, which can leave a
+  // package half-replaced and break EVERY AI tool (not just the one installing).
+  // Model weight files under MODELS_DIR are unaffected, so the safe, automatic
+  // recovery is to reseed the venv from the image base and reset the install
+  // ledger; the user reinstalls bundles from a known-good state. This is the
+  // same repair as "Reset AI Environment", done automatically on next boot so a
+  // crash-broken install self-heals instead of leaving mysteriously dead tools.
+  if (existsSync(VENV_WRITING_MARKER)) {
+    if (isDockerEnvironment() && existsSync("/opt/venv")) {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write; reseeding the AI venv to a clean state and clearing installed bundles (reinstall to restore).",
+      );
+      try {
+        execFileSync("/usr/local/bin/reseed-ai-venv.sh", { stdio: "ignore", timeout: 120_000 });
+        writeInstalled({ bundles: {} });
+      } catch (err) {
+        console.error("[feature-status] Failed to reseed AI venv after interrupted install:", err);
+      }
+    } else {
+      console.warn(
+        "[feature-status] An install was interrupted mid venv-write (non-Docker); the AI venv may be inconsistent. Reinstall the affected bundle.",
+      );
+    }
+    try {
+      unlinkSync(VENV_WRITING_MARKER);
+    } catch {
+      // best-effort
+    }
+  }
+
   // 5. Verify installed bundles still have their model files
   const manifest = readManifest();
   if (manifest) {

apps/api/src/lib/install-watchdog.ts

@@ -0,0 +1,42 @@
+/**
+ * Pure decision for the AI-bundle install watchdog.
+ *
+ * A wedged installer (dead download socket, hung pip) holds the venv writer lock
+ * forever and blocks every other install with no exit but a server restart. The
+ * watchdog kills such a child so its close handler frees the locks and the user
+ * gets a retryable failure. This function holds the (timer-free, side-effect
+ * free) rule so it can be unit-tested exhaustively; features.ts owns the timers.
+ *
+ * @param now            current time (ms)
+ * @param lastProgressAt time of the last progress frame (ms)
+ * @param startedAt      time the install started (ms)
+ * @param stallMs        max time with no progress before a kill (0 disables)
+ * @param maxMs          absolute wall-clock ceiling (0 disables)
+ */
+export function evaluateInstallWatchdog(
+  now: number,
+  lastProgressAt: number,
+  startedAt: number,
+  stallMs: number,
+  maxMs: number,
+): { kill: boolean; reason: string | null } {
+  const overMax = maxMs > 0 && now - startedAt > maxMs;
+  if (overMax) {
+    return {
+      kill: true,
+      reason: `Installation exceeded the ${Math.round(
+        maxMs / 60_000,
+      )} minute time limit and was stopped. Please retry.`,
+    };
+  }
+  const stalled = stallMs > 0 && now - lastProgressAt > stallMs;
+  if (stalled) {
+    return {
+      kill: true,
+      reason: `Installation made no progress for ${Math.round(
+        stallMs / 60_000,
+      )} minutes and was stopped. Check your connection and retry.`,
+    };
+  }
+  return { kill: false, reason: null };
+}