test: testing overhaul -- CI e2e gates, parallel suites, generated matrices, mutation testing (#215)

Closes the "e2e never runs in CI" hole. Adds per-PR e2e smoke gate,
nightly full-suite workflows, parallel vitest forks (per-fork DBs),
Playwright parallel/serial/visual projects against production builds,
metadata-generated test suites (drift guards, hostile inputs, format
matrix, pairwise settings, property-based fuzz), Stryker mutation
testing, Schemathesis API fuzz, coverage ratchet, and fixes for three
session-poisoning bugs that caused 200+ serial-bucket failures.

Bug fix included: favicon/split/bulk-rename could hang clients forever
when ZIP streaming failed after reply.hijack().

tests/integration/format-matrix-generated.test.ts

@@ -0,0 +1,146 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { TOOLS } from "@snapotter/shared";
+import sharp from "sharp";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getRegisteredToolIds, getToolConfig } from "../../apps/api/src/routes/tool-factory.js";
+import { defaultSettingsFor, TOOL_SETTINGS_OVERRIDES } from "../helpers/tool-default-settings.js";
+import { buildTestApp, createMultipartPayload, loginAsAdmin, type TestApp } from "./test-server.js";
+
+/**
+ * Registry-generated tool x format matrix.
+ *
+ * Every registered tool is exercised against every input format fixture with
+ * its minimal valid settings. The invariant is the factory's error contract:
+ * success (200/202), clean rejection (400/413/415/422), or AI-not-installed
+ * (501). A 500 or an undecodable "successful" output is a bug.
+ *
+ * PR runs use the core web formats; FULL_MATRIX=1 (nightly) unlocks all
+ * fixtures in tests/fixtures/formats/.
+ */
+const FORMATS_DIR = join(__dirname, "..", "fixtures", "formats");
+
+const CORE_FORMATS = [
+  "sample.png",
+  "sample.jpg",
+  "sample.webp",
+  "sample.gif",
+  "sample.svg",
+  "sample.heic",
+];
+
+const fixtureFiles = process.env.FULL_MATRIX
+  ? readdirSync(FORMATS_DIR).filter((f) => !f.startsWith("."))
+  : CORE_FORMATS;
+
+const ALLOWED_STATUSES = new Set([200, 202, 400, 413, 415, 422, 501]);
+
+/** Content types whose payloads are not raster images (skip pixel decode). */
+const NON_RASTER_OUTPUT = new Set([
+  "application/pdf",
+  "application/json",
+  "application/zip",
+  "image/svg+xml",
+  "text/plain",
+]);
+
+describe("tool x format matrix (generated)", () => {
+  let testApp: TestApp;
+  let adminToken: string;
+
+  beforeAll(async () => {
+    testApp = await buildTestApp();
+    adminToken = await loginAsAdmin(testApp.app);
+  }, 30_000);
+
+  afterAll(async () => {
+    await testApp.cleanup();
+  }, 10_000);
+
+  it("settings overrides only reference registered tools", () => {
+    const registered = new Set(getRegisteredToolIds());
+    for (const toolId of Object.keys(TOOL_SETTINGS_OVERRIDES)) {
+      expect(registered.has(toolId), `override for unknown tool "${toolId}"`).toBe(true);
+    }
+  });
+
+  it("default settings are valid for every registered tool", () => {
+    const invalid: string[] = [];
+    for (const toolId of getRegisteredToolIds()) {
+      const config = getToolConfig(toolId);
+      if (!config) continue;
+      const result = config.settingsSchema.safeParse(defaultSettingsFor(toolId));
+      if (!result.success) {
+        invalid.push(
+          `${toolId}: ${result.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}`,
+        );
+      }
+    }
+    expect(
+      invalid,
+      `tools needing TOOL_SETTINGS_OVERRIDES entries:\n${invalid.join("\n")}`,
+    ).toEqual([]);
+  });
+
+  for (const tool of TOOLS) {
+    const toolId = tool.id;
+    it(`${toolId} handles every input format cleanly`, async () => {
+      for (const fixture of fixtureFiles) {
+        const content = readFileSync(join(FORMATS_DIR, fixture));
+        const { body, contentType } = createMultipartPayload([
+          { name: "file", filename: fixture, contentType: "application/octet-stream", content },
+          { name: "settings", content: JSON.stringify(defaultSettingsFor(toolId)) },
+        ]);
+        const res = await testApp.app.inject({
+          method: "POST",
+          url: `/api/v1/tools/${toolId}`,
+          headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType },
+          body,
+        });
+
+        // Custom-route tools can 404 on the standard path; covered elsewhere.
+        if (res.statusCode === 404) return;
+
+        expect(
+          ALLOWED_STATUSES.has(res.statusCode),
+          `${toolId} x ${fixture}: status ${res.statusCode}: ${res.body.slice(0, 300)}`,
+        ).toBe(true);
+
+        if (res.statusCode === 200) {
+          const resType = (res.headers["content-type"]?.toString() ?? "").split(";")[0];
+          if (resType !== "application/json") {
+            // Tools like bulk-rename/favicon/split stream a ZIP directly.
+            if (resType === "application/zip") {
+              expect(
+                res.rawPayload.subarray(0, 2).toString("latin1"),
+                `${toolId} x ${fixture}: ZIP response is not a ZIP`,
+              ).toBe("PK");
+            }
+            continue;
+          }
+          const payload = JSON.parse(res.body) as { downloadUrl?: string };
+          if (!payload.downloadUrl) continue;
+          const dl = await testApp.app.inject({
+            method: "GET",
+            url: payload.downloadUrl,
+            headers: { authorization: `Bearer ${adminToken}` },
+          });
+          expect(dl.statusCode, `${toolId} x ${fixture}: download failed`).toBe(200);
+          const outType = dl.headers["content-type"]?.toString() ?? "";
+          const isRaster =
+            !NON_RASTER_OUTPUT.has(outType.split(";")[0]) && outType.startsWith("image/");
+          const sharpDecodable =
+            isRaster &&
+            !["image/heic", "image/heif", "image/x-icon", "image/qoi"].includes(
+              outType.split(";")[0],
+            );
+          if (sharpDecodable) {
+            // The processed output must actually decode; a corrupt "success" is a bug.
+            const meta = await sharp(dl.rawPayload).metadata();
+            expect(meta.width, `${toolId} x ${fixture}: output not decodable`).toBeGreaterThan(0);
+          }
+        }
+      }
+    }, 240_000);
+  }
+});

tests/integration/fuzz-settings.test.ts

@@ -0,0 +1,96 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { TOOLS } from "@snapotter/shared";
+import fc from "fast-check";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { z } from "zod";
+import { ZodFastCheck } from "zod-fast-check";
+import { getToolConfig } from "../../apps/api/src/routes/tool-factory.js";
+import { collectRegexStringSchemas } from "../helpers/zod-pict.js";
+import { buildTestApp, type TestApp } from "./test-server.js";
+
+/**
+ * Property-based settings fuzz: random VALID settings (derived from each
+ * tool's own Zod schema via zod-fast-check) must never produce crash-class
+ * failures. Complements the deterministic pairwise matrix with arbitrary
+ * strings/numbers that humans and AIs never think to write.
+ *
+ * Nightly-only (FUZZ=1); FUZZ_RUNS controls depth (default 25).
+ */
+const FUZZ = !!process.env.FUZZ;
+const NUM_RUNS = Number(process.env.FUZZ_RUNS ?? 25);
+const CRASH_PATTERN =
+  /TypeError|undefined is not|null is not|Cannot read propert|is not a function/i;
+
+describe.skipIf(!FUZZ)("settings fuzz (property-based)", () => {
+  let testApp: TestApp;
+  let inputPng: Buffer;
+
+  beforeAll(async () => {
+    testApp = await buildTestApp();
+    inputPng = readFileSync(join(__dirname, "..", "fixtures", "test-200x150.png"));
+  }, 30_000);
+
+  afterAll(async () => {
+    await testApp.cleanup();
+  }, 10_000);
+
+  // The registry is populated by buildTestApp() in beforeAll, so tool configs
+  // are looked up inside the test body; registry-exempt tools no-op here.
+  for (const tool of TOOLS) {
+    const toolId = tool.id;
+    it(`${toolId} never crashes on schema-valid settings`, async () => {
+      const config = getToolConfig(toolId);
+      if (!config) return;
+
+      let arbitrary: fc.Arbitrary<unknown>;
+      try {
+        let zfc = ZodFastCheck();
+        // zod-fast-check cannot generate regex-constrained strings (hex
+        // colors and friends); override every regex-checked string field
+        // with plausible color constants. Values that still fail the regex
+        // are discarded by the fc.pre() below.
+        for (const sub of collectRegexStringSchemas(config.settingsSchema)) {
+          zfc = zfc.override(
+            sub as z.ZodTypeAny,
+            fc.constantFrom("#ff0000", "#000000", "#ffffff", "#00ff7f", "#ff000080"),
+          );
+        }
+        arbitrary = zfc.inputOf(config.settingsSchema as z.ZodTypeAny);
+      } catch {
+        // Schema uses constructs zod-fast-check cannot derive (refinements over
+        // multiple fields, transforms); the pairwise matrix still covers it.
+        return;
+      }
+
+      try {
+        await fc.assert(
+          fc.asyncProperty(arbitrary, async (settings) => {
+            const parsed = config.settingsSchema.safeParse(settings);
+            fc.pre(parsed.success);
+            try {
+              await config.process(inputPng, parsed.data, "test-200x150.png");
+            } catch (err) {
+              if (!(err instanceof Error)) {
+                throw new Error(`${toolId} threw a non-Error: ${String(err)}`);
+              }
+              if (CRASH_PATTERN.test(err.message)) {
+                throw new Error(`${toolId} crashed on ${JSON.stringify(settings)}: ${err.message}`);
+              }
+              // Clean operational failure: acceptable.
+            }
+          }),
+          { numRuns: NUM_RUNS, interruptAfterTimeLimit: 180_000 },
+        );
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        // Generator dead-ends (un-derivable sub-schema or every value failing
+        // a refinement) mean this tool cannot be fuzzed generically; the
+        // pairwise matrix still covers it. Real property failures rethrow.
+        if (/Unable to generate valid values|precondition/i.test(message)) return;
+        throw err;
+      }
+      expect(true).toBe(true);
+    }, 240_000);
+  }
+});

tests/integration/hostile-inputs.test.ts

@@ -0,0 +1,105 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { TOOLS } from "@snapotter/shared";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { TOOL_DISPLAY_MODES } from "../../apps/web/src/lib/tool-display-modes.js";
+import { buildTestApp, createMultipartPayload, loginAsAdmin, type TestApp } from "./test-server.js";
+
+/**
+ * Hostile-input matrix: every tool route must reject malformed, truncated,
+ * lying, or bomb-shaped files with a clean 4xx (or 501 for uninstalled AI
+ * bundles). A 500, a hang, or a success response for garbage is a bug in the
+ * tool, not in this test.
+ *
+ * Fixtures come from scripts/generate-hostile-fixtures.mjs (committed).
+ */
+const HOSTILE_DIR = join(__dirname, "..", "fixtures", "hostile");
+
+/** Fixtures that are unreadable garbage: the server must NOT report success. */
+const GARBAGE_FIXTURES = ["truncated.jpg", "zero-byte.png", "garbage.jpg", "bomb-50000x50000.png"];
+
+/** Valid PNG bytes behind a lying .jpg extension: success or clean 4xx are both
+ * fine (tools sniff content, some require a specific input type); 5xx is not. */
+const MISMATCH_FIXTURE = "png-bytes.jpg";
+
+const REJECT_STATUSES = new Set([400, 413, 415, 422, 501]);
+
+// Tools that never decode the uploaded pixel data: no-dropzone generators take
+// input from settings, bulk-rename zips bytes verbatim, and the metadata tools
+// operate on metadata segments only. Succeeding on a file with a valid header
+// but broken pixel data is correct behavior for them; everything else must
+// reject.
+const INPUT_AGNOSTIC = new Set(
+  TOOLS.filter((t) => TOOL_DISPLAY_MODES[t.id] === "no-dropzone").map((t) => t.id),
+);
+INPUT_AGNOSTIC.add("bulk-rename");
+INPUT_AGNOSTIC.add("edit-metadata");
+INPUT_AGNOSTIC.add("strip-metadata");
+INPUT_AGNOSTIC.add("info");
+INPUT_AGNOSTIC.add("image-to-base64");
+
+/** Server-error statuses; 501 (feature not installed) is a clean rejection. */
+const SERVER_ERRORS = [500, 502, 503, 504];
+
+describe("hostile input matrix", () => {
+  let testApp: TestApp;
+  let adminToken: string;
+
+  beforeAll(async () => {
+    testApp = await buildTestApp();
+    adminToken = await loginAsAdmin(testApp.app);
+  }, 30_000);
+
+  afterAll(async () => {
+    await testApp.cleanup();
+  }, 10_000);
+
+  async function postFile(toolId: string, fixtureName: string) {
+    const content = readFileSync(join(HOSTILE_DIR, fixtureName));
+    const { body, contentType } = createMultipartPayload([
+      { name: "file", filename: fixtureName, contentType: "application/octet-stream", content },
+      { name: "settings", content: "{}" },
+    ]);
+    const started = Date.now();
+    const res = await testApp.app.inject({
+      method: "POST",
+      url: `/api/v1/tools/${toolId}`,
+      headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType },
+      body,
+    });
+    return { res, elapsedMs: Date.now() - started };
+  }
+
+  for (const tool of TOOLS) {
+    const toolId = tool.id;
+    it(`${toolId} rejects hostile files cleanly`, async () => {
+      for (const fixture of GARBAGE_FIXTURES) {
+        const { res, elapsedMs } = await postFile(toolId, fixture);
+
+        expect(
+          SERVER_ERRORS.includes(res.statusCode),
+          `${toolId} returned ${res.statusCode} for ${fixture}: ${res.body.slice(0, 300)}`,
+        ).toBe(false);
+        expect(elapsedMs, `${toolId} took ${elapsedMs}ms on ${fixture}`).toBeLessThan(15_000);
+
+        if (INPUT_AGNOSTIC.has(toolId)) continue;
+
+        expect(
+          REJECT_STATUSES.has(res.statusCode),
+          `${toolId} did not reject ${fixture} (got ${res.statusCode})`,
+        ).toBe(true);
+
+        // Error responses must be structured JSON, not stack traces
+        const parsed = JSON.parse(res.body) as { error?: string };
+        expect(parsed.error, `${toolId} 4xx body has no error field for ${fixture}`).toBeTruthy();
+      }
+
+      // Lying extension with valid content: anything but a server error is fine
+      const { res } = await postFile(toolId, MISMATCH_FIXTURE);
+      expect(
+        SERVER_ERRORS.includes(res.statusCode),
+        `${toolId} returned ${res.statusCode} for ${MISMATCH_FIXTURE}: ${res.body.slice(0, 300)}`,
+      ).toBe(false);
+    }, 120_000);
+  }
+});

tests/integration/settings-pairwise.test.ts

@@ -0,0 +1,108 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { TOOLS } from "@snapotter/shared";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getToolConfig } from "../../apps/api/src/routes/tool-factory.js";
+import { pairwise } from "../helpers/pairwise.js";
+import { defaultSettingsFor } from "../helpers/tool-default-settings.js";
+import { compactCase, deriveAxes } from "../helpers/zod-pict.js";
+import { buildTestApp, type TestApp } from "./test-server.js";
+
+/**
+ * Pairwise settings matrix: a covering array over each tool's settings schema
+ * (every pair of axis values appears at least once), filtered through the
+ * schema's own refinements, with each survivor run through the tool's process
+ * function directly.
+ *
+ * Invariant: a tool either succeeds or fails with a real, descriptive Error.
+ * TypeErrors and undefined-access crashes are the AI-written-code failure
+ * class this suite exists to catch.
+ *
+ * PR runs cover the core tools; FULL_MATRIX=1 (nightly) covers every tool.
+ */
+const CORE_TOOLS = [
+  "resize",
+  "crop",
+  "rotate",
+  "convert",
+  "compress",
+  "adjust-colors",
+  "watermark-text",
+  "border",
+];
+
+const MAX_CASES_PER_TOOL = 40;
+const CRASH_PATTERN =
+  /TypeError|undefined is not|null is not|Cannot read propert|is not a function/i;
+
+describe("pairwise settings matrix", () => {
+  let testApp: TestApp;
+  let inputPng: Buffer;
+
+  beforeAll(async () => {
+    testApp = await buildTestApp();
+    inputPng = readFileSync(join(__dirname, "..", "fixtures", "test-200x150.png"));
+  }, 30_000);
+
+  afterAll(async () => {
+    await testApp.cleanup();
+  }, 10_000);
+
+  // The registry is populated by buildTestApp() in beforeAll, so the
+  // FULL_MATRIX tool list comes from the static TOOLS catalog and configs are
+  // looked up inside the test body; registry-exempt tools no-op here.
+  const toolIds = process.env.FULL_MATRIX ? TOOLS.map((t) => t.id) : CORE_TOOLS;
+
+  for (const toolId of toolIds) {
+    it(`${toolId} survives its pairwise settings matrix`, async () => {
+      const config = getToolConfig(toolId);
+      if (!config) {
+        expect(process.env.FULL_MATRIX, `core tool "${toolId}" is not registered`).toBeTruthy();
+        return;
+      }
+
+      const axes = deriveAxes(config.settingsSchema);
+      if (axes.length < 2) {
+        // Not enough enumerable axes for pair coverage; fuzz covers this tool.
+        return;
+      }
+
+      // Merge combos over the tool's minimal valid settings so required
+      // fields that are not enumerable axes (e.g. watermark text) are present.
+      const base = defaultSettingsFor(toolId) as Record<string, unknown>;
+      const combos = pairwise(axes);
+      const cases = combos
+        .map((combo) => ({ ...base, ...compactCase(combo) }))
+        .map((combo) => config.settingsSchema.safeParse(combo))
+        .filter((parsed): parsed is { success: true; data: unknown } => parsed.success)
+        .slice(0, MAX_CASES_PER_TOOL);
+
+      expect(
+        cases.length,
+        `${toolId}: every pairwise combo was rejected by the schema`,
+      ).toBeGreaterThan(0);
+
+      for (const parsed of cases) {
+        try {
+          const result = await config.process(inputPng, parsed.data, "test-200x150.png");
+          expect(
+            result.buffer.length,
+            `${toolId} produced empty output for ${JSON.stringify(parsed.data)}`,
+          ).toBeGreaterThan(0);
+        } catch (err) {
+          // Clean operational failures (e.g. crop area outside image) are
+          // acceptable; crash-class errors are not.
+          expect(
+            err,
+            `${toolId} threw a non-Error for ${JSON.stringify(parsed.data)}`,
+          ).toBeInstanceOf(Error);
+          const message = (err as Error).message;
+          expect(
+            CRASH_PATTERN.test(message),
+            `${toolId} crashed on ${JSON.stringify(parsed.data)}: ${message}`,
+          ).toBe(false);
+        }
+      }
+    }, 240_000);
+  }
+});

tests/integration/tool-route-drift.test.ts

@@ -0,0 +1,104 @@
+import { TOOLS } from "@snapotter/shared";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getRegisteredToolIds, getToolConfig } from "../../apps/api/src/routes/tool-factory.js";
+import { buildTestApp, loginAsAdmin, type TestApp } from "./test-server.js";
+
+/**
+ * Drift guards between the shared TOOLS catalog and the API.
+ *
+ * Two intentional asymmetries exist and are pinned exactly:
+ *  - REGISTRY_EXEMPT: tools whose contract does not fit the single-buffer
+ *    process fn (multi-file, ZIP/JSON output, no-input generators, custom AI
+ *    routes). They expose an HTTP route but are not in the pipeline/batch
+ *    registry. If one of these gains registry support, remove it here.
+ *  - LEGACY_ALIASES: extra registered toolIds kept for backwards-compatible
+ *    URLs (consolidated into adjust-colors).
+ */
+const REGISTRY_EXEMPT = new Set([
+  "barcode-read",
+  "bulk-rename",
+  "collage",
+  "color-palette",
+  "compare",
+  "compose",
+  "erase-object",
+  "favicon",
+  "find-duplicates",
+  "html-to-image",
+  "image-to-base64",
+  "image-to-pdf",
+  "info",
+  "ocr",
+  "pdf-to-image",
+  "qr-generate",
+  "stitch",
+  "svg-to-raster",
+  "watermark-image",
+]);
+
+const LEGACY_ALIASES = new Set([
+  "brightness-contrast",
+  "saturation",
+  "color-channels",
+  "color-effects",
+]);
+
+describe("tool route drift", () => {
+  let testApp: TestApp;
+  let adminToken: string;
+
+  beforeAll(async () => {
+    testApp = await buildTestApp();
+    adminToken = await loginAsAdmin(testApp.app);
+  }, 30_000);
+
+  afterAll(async () => {
+    await testApp.cleanup();
+  }, 10_000);
+
+  it("every non-exempt TOOLS entry has a registered process fn", () => {
+    const registered = new Set(getRegisteredToolIds());
+    const missing = TOOLS.filter((t) => !REGISTRY_EXEMPT.has(t.id) && !registered.has(t.id)).map(
+      (t) => t.id,
+    );
+    expect(missing, `tools not registered on the API: ${missing.join(", ")}`).toEqual([]);
+  });
+
+  it("registry-exempt list is not stale", () => {
+    const registered = new Set(getRegisteredToolIds());
+    for (const id of REGISTRY_EXEMPT) {
+      expect(
+        registered.has(id),
+        `"${id}" is in REGISTRY_EXEMPT but IS registered now; remove it from the exempt list`,
+      ).toBe(false);
+    }
+  });
+
+  it("every registered tool exposes a settings schema and process fn", () => {
+    for (const id of getRegisteredToolIds()) {
+      const config = getToolConfig(id);
+      expect(config?.settingsSchema, `tool "${id}" has no settings schema`).toBeTruthy();
+      expect(typeof config?.process, `tool "${id}" has no process fn`).toBe("function");
+    }
+  });
+
+  it("no orphan registrations (registered but missing from TOOLS, excluding legacy aliases)", () => {
+    const ids = new Set(TOOLS.map((t) => t.id));
+    for (const id of getRegisteredToolIds()) {
+      if (LEGACY_ALIASES.has(id)) continue;
+      expect(ids.has(id), `registered tool "${id}" has no TOOLS definition`).toBe(true);
+    }
+  });
+
+  it("every TOOLS entry answers on POST /api/v1/tools/:toolId (no dead routes)", async () => {
+    for (const tool of TOOLS) {
+      const res = await testApp.app.inject({
+        method: "POST",
+        url: `/api/v1/tools/${tool.id}`,
+        headers: { authorization: `Bearer ${adminToken}`, "content-type": "application/json" },
+        payload: {},
+      });
+      expect(res.statusCode, `tool "${tool.id}" has no live POST route (got 404)`).not.toBe(404);
+    }
+  }, 60_000);
+});