fix: reliable, self-healing AI feature-bundle installs (#472)

Make on-demand AI feature-bundle installs reliable and self-healing, closing
the failure modes behind most "some tool doesn't work" reports.

Multi-bundle installs: tools needing more than one bundle (Passport Photo,
Enhance Faces) install every required bundle from one action and stay
not-installed until all are present. Verified across all 19 AI tools.

Downloads: self-heal the accelerated Hugging Face (Xet) client so an upgraded
venv no longer silently falls back to slow urllib; restart instead of
corrupting a resumed partial when a proxy ignores Range and returns 200;
verify the completed size; fail fast on disk-full and HTTP 4xx; retry
transient errors five times; add hf_transfer fallback and document Xet egress.

Install integrity: crash-atomic venv writes so a killed or out-of-space
install can no longer tear the shared venv and break other tools; a boot
breadcrumb reseeds a torn venv to a clean state automatically; a post-install
smoke import test refuses to record a bundle whose libraries cannot load; an
install watchdog stops a wedged installer that would otherwise hold the venv
writer lock forever.

Adds unit and end-to-end tests for every failure mode above.

tests/unit/api/install-watchdog.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import { evaluateInstallWatchdog } from "../../../apps/api/src/lib/install-watchdog.js";
+
+const STALL = 20 * 60_000; // 20 min
+const MAX = 120 * 60_000; // 2 h
+
+describe("evaluateInstallWatchdog", () => {
+  it("does not kill an install making steady progress", () => {
+    const now = 1_000_000;
+    const v = evaluateInstallWatchdog(now, now - 5_000, now - 60_000, STALL, MAX);
+    expect(v.kill).toBe(false);
+    expect(v.reason).toBeNull();
+  });
+
+  it("kills when no progress frame has arrived within the stall budget", () => {
+    const now = 1_000_000;
+    const v = evaluateInstallWatchdog(now, now - (STALL + 1), now - (STALL + 1), STALL, MAX);
+    expect(v.kill).toBe(true);
+    expect(v.reason).toContain("no progress");
+  });
+
+  it("does not kill exactly at the stall boundary (strictly greater than)", () => {
+    const now = 1_000_000;
+    const v = evaluateInstallWatchdog(now, now - STALL, now - STALL, STALL, MAX);
+    expect(v.kill).toBe(false);
+  });
+
+  it("kills when the absolute time ceiling is exceeded even if progress is recent", () => {
+    const now = 1_000_000;
+    // Progress 1s ago (not stalled) but the install started > MAX ago.
+    const v = evaluateInstallWatchdog(now, now - 1_000, now - (MAX + 1), STALL, MAX);
+    expect(v.kill).toBe(true);
+    expect(v.reason).toContain("time limit");
+  });
+
+  it("prefers the absolute-ceiling reason when both conditions hold", () => {
+    const now = 1_000_000;
+    const v = evaluateInstallWatchdog(now, now - (STALL + 1), now - (MAX + 1), STALL, MAX);
+    expect(v.kill).toBe(true);
+    expect(v.reason).toContain("time limit");
+  });
+
+  it("treats 0 as disabled for each check independently", () => {
+    const now = 1_000_000;
+    // Stall disabled, max active: a long stall alone must not kill.
+    expect(evaluateInstallWatchdog(now, now - 10 * STALL, now - 1_000, 0, MAX).kill).toBe(false);
+    // Max disabled, stall active: an old start alone must not kill.
+    expect(evaluateInstallWatchdog(now, now - 1_000, now - 10 * MAX, STALL, 0).kill).toBe(false);
+    // Both disabled: never kills.
+    expect(evaluateInstallWatchdog(now, now - 10 * STALL, now - 10 * MAX, 0, 0).kill).toBe(false);
+  });
+});

tests/unit/api/tool-factory-route.test.ts

@@ -100,6 +100,7 @@ vi.mock("../../../apps/api/src/lib/svg-sanitize.js", () => ({
 }));
 
 vi.mock("../../../apps/api/src/lib/feature-status.js", () => ({
+  getFirstMissingBundleForTool: vi.fn(() => null),
   isToolInstalled: vi.fn(() => true),
 }));
 
@@ -139,7 +140,10 @@ vi.mock("sharp", () => ({
 
 import { apiToolPath } from "@snapotter/shared";
 import { enqueueToolJob, waitForJob } from "../../../apps/api/src/jobs/enqueue.js";
-import { isToolInstalled } from "../../../apps/api/src/lib/feature-status.js";
+import {
+  getFirstMissingBundleForTool,
+  isToolInstalled,
+} from "../../../apps/api/src/lib/feature-status.js";
 import { validateImageBuffer } from "../../../apps/api/src/lib/file-validation.js";
 import type { AnyToolRouteConfig } from "../../../apps/api/src/routes/tool-factory.js";
 import {
@@ -281,6 +285,8 @@ function createMockRequest(opts: {
 describe("createToolRoute", () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    vi.mocked(getFirstMissingBundleForTool).mockReset();
+    vi.mocked(getFirstMissingBundleForTool).mockReturnValue(null);
     vi.mocked(isToolInstalled).mockReset();
     vi.mocked(isToolInstalled).mockReturnValue(true);
   });
@@ -424,6 +430,31 @@ describe("createToolRoute", () => {
       // This validates the guard code path exists without false positives.
     });
 
+    it("returns 501 naming the first missing extra bundle for multi-bundle AI tools", async () => {
+      vi.mocked(isToolInstalled).mockReturnValueOnce(false);
+      vi.mocked(getFirstMissingBundleForTool).mockReturnValueOnce("face-detection");
+      const app = createMockApp();
+      const id = "enhance-faces";
+      createToolRoute(app as never, makeMockConfig(id));
+      const handler = app.routes[apiToolPath(id)];
+      const reply = createMockReply();
+      const req = createMockRequest({
+        fileBuffer: Buffer.from("png-data"),
+        settings: JSON.stringify({}),
+      });
+
+      await handler(req, reply);
+
+      expect(reply.status).toHaveBeenCalledWith(501);
+      expect(reply.send).toHaveBeenCalledWith(
+        expect.objectContaining({
+          code: "FEATURE_NOT_INSTALLED",
+          feature: "face-detection",
+          featureName: "Face Detection",
+        }),
+      );
+    });
+
     it("returns 200 success envelope when waitForJob resolves", async () => {
       const app = createMockApp();
       const id = "resize";