test: split the three oversized format-matrix specs (#651)

Vitest shards by file and runs a file's tests serially in one fork, so a single spec set the floor for the whole Integration job no matter how many shards or forks it got. Cost-aware sharding (#650) balanced the shards but could not get under that floor. Split the three specs that exceeded it: format-matrix-comprehensive (1365s) into 4 by describe, format-matrix (1130s) into 4 with Cross-format matrix striped over FORMAT_SAMPLES, and format-matrix-generated (779s) into 3 striped over TOOLS. Largest spec is now 370s. Each preamble moved verbatim into a sibling .shared.ts exposing setupMatrixApp(). Integration shards went from 20m59s/17m55s/16m33s/9m19s to 11m44s/12m31s/10m37s/11m1s. Coverage checked, not assumed: the set of test names collected by vitest list is byte-identical across the split, 2151 before and 2151 after. Per-shard totals matched the baseline exactly at 9903 tests, 9435 passed, 468 skipped.
2026-08-03 07:46:42 +02:00 · 2026-07-27 00:36:51 +08:00
parent d91abc8a23
commit d9978525fe
19 changed files with 3365 additions and 2947 deletions
@@ -0,0 +1,191 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { apiToolPath, TOOLS } from "@snapotter/shared";
+import sharp from "sharp";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getRegisteredToolIds, getToolConfig } from "../../../apps/api/src/routes/tool-factory.js";
+import { fixtureDir } from "../../fixtures/index.js";
+import {
+  defaultSettingsFor,
+  TOOL_SETTINGS_OVERRIDES,
+} from "../../helpers/tool-default-settings.js";
+import { cancelAcceptedJobAndWait } from "../settle-job.js";
+import {
+  buildTestApp,
+  createMultipartPayload,
+  loginAsAdmin,
+  type TestApp,
+} from "../test-server.js";
+
+/**
+ * Registry-generated tool x format matrix.
+ *
+ * Every registered tool is exercised against every input format fixture with
+ * its minimal valid settings. The invariant is the factory's error contract:
+ * success (200/202), clean rejection (400/413/415/422), or AI-not-installed
+ * (501). A 500 or an undecodable "successful" output is a bug.
+ *
+ * PR runs use the core web formats; FULL_MATRIX=1 (nightly) unlocks all
+ * fixtures in tests/fixtures/image/formats/.
+ *
+ * Split across format-matrix-generated-{1,2,3}.test.ts because vitest shards by
+ * file and runs a file's tests serially in one fork, so a single 800s spec set
+ * the floor for the whole Integration job. `toolsForPart` stripes TOOLS by
+ * modulo, which partitions them by construction: every tool lands in exactly
+ * one part, and the describe name is identical in each so the full test-name
+ * set is unchanged.
+ */
+
+const CORE_FORMATS = [
+  "sample.png",
+  "sample.jpg",
+  "sample.webp",
+  "sample.gif",
+  "sample.svg",
+  "sample.heic",
+];
+
+const fixtureFiles = process.env.FULL_MATRIX
+  ? readdirSync(fixtureDir.formats).filter((f) => !f.startsWith("."))
+  : CORE_FORMATS;
+
+const ALLOWED_STATUSES = new Set([200, 202, 400, 413, 415, 422, 501]);
+
+/**
+ * Raster content types this libvips/Sharp build is guaranteed to decode. Used
+ * to decide whether a 200 image response should be pixel-verified. Anything
+ * else (PDF, JSON, ZIP, SVG, or a niche raster like BMP/PSD streamed back
+ * untouched) carries an honest content-type we don't attempt to decode.
+ */
+const SHARP_DECODABLE_TYPES = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "image/gif",
+  "image/tiff",
+  "image/avif",
+]);
+
+export const MATRIX_PART_COUNT = 3;
+
+/** Tools belonging to one part. Striped, so the parts partition TOOLS exactly. */
+export function toolsForPart(part: number): typeof TOOLS {
+  return TOOLS.filter((_, index) => index % MATRIX_PART_COUNT === part - 1);
+}
+
+/**
+ * Registers the matrix for one part. The registry-wide schema guards are
+ * global assertions rather than per-tool ones, so they run in part 1 only.
+ */
+export function registerToolFormatMatrix(part: number): void {
+  describe("tool x format matrix (generated)", () => {
+    let testApp: TestApp;
+    let adminToken: string;
+
+    beforeAll(async () => {
+      testApp = await buildTestApp();
+      adminToken = await loginAsAdmin(testApp.app);
+    }, 30_000);
+
+    afterAll(async () => {
+      await testApp.cleanup();
+    }, 10_000);
+
+    if (part === 1) {
+      it("settings overrides only reference registered tools", () => {
+        const registered = new Set(getRegisteredToolIds());
+        for (const toolId of Object.keys(TOOL_SETTINGS_OVERRIDES)) {
+          expect(registered.has(toolId), `override for unknown tool "${toolId}"`).toBe(true);
+        }
+      });
+
+      it("default settings are valid for every registered tool", () => {
+        const invalid: string[] = [];
+        for (const toolId of getRegisteredToolIds()) {
+          const config = getToolConfig(toolId);
+          if (!config) continue;
+          const result = config.settingsSchema.safeParse(defaultSettingsFor(toolId));
+          if (!result.success) {
+            invalid.push(
+              `${toolId}: ${result.error.issues.map((i) => `${i.path.join(".")} ${i.message}`).join("; ")}`,
+            );
+          }
+        }
+        expect(
+          invalid,
+          `tools needing TOOL_SETTINGS_OVERRIDES entries:\n${invalid.join("\n")}`,
+        ).toEqual([]);
+      });
+    }
+
+    for (const tool of toolsForPart(part)) {
+      const toolId = tool.id;
+      it(`${toolId} handles every input format cleanly`, async () => {
+        for (const fixture of fixtureFiles) {
+          const content = readFileSync(join(fixtureDir.formats, fixture));
+          const { body, contentType } = createMultipartPayload([
+            { name: "file", filename: fixture, contentType: "application/octet-stream", content },
+            { name: "settings", content: JSON.stringify(defaultSettingsFor(toolId)) },
+          ]);
+          const res = await testApp.app.inject({
+            method: "POST",
+            url: apiToolPath(toolId),
+            headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType },
+            body,
+          });
+
+          // Custom-route tools can 404 on the standard path; covered elsewhere.
+          if (res.statusCode === 404) return;
+
+          expect(
+            ALLOWED_STATUSES.has(res.statusCode),
+            `${toolId} x ${fixture}: status ${res.statusCode}: ${res.body.slice(0, 300)}`,
+          ).toBe(true);
+
+          if (res.statusCode === 200) {
+            const resType = (res.headers["content-type"]?.toString() ?? "").split(";")[0];
+            if (resType !== "application/json") {
+              // Tools like bulk-rename/favicon/split stream a ZIP directly.
+              if (resType === "application/zip") {
+                expect(
+                  res.rawPayload.subarray(0, 2).toString("latin1"),
+                  `${toolId} x ${fixture}: ZIP response is not a ZIP`,
+                ).toBe("PK");
+              }
+              continue;
+            }
+            const payload = JSON.parse(res.body) as { downloadUrl?: string };
+            if (!payload.downloadUrl) continue;
+            const dl = await testApp.app.inject({
+              method: "GET",
+              url: payload.downloadUrl,
+              headers: { authorization: `Bearer ${adminToken}` },
+            });
+            expect(dl.statusCode, `${toolId} x ${fixture}: download failed`).toBe(200);
+            const outType = (dl.headers["content-type"]?.toString() ?? "").split(";")[0];
+            // Allowlist of raster types this libvips build is guaranteed to
+            // decode. An allowlist (vs the old denylist) is robust to tools that
+            // legitimately stream back niche formats untouched (e.g. edit-metadata
+            // writing tags in place on a BMP/PSD): those carry an honest
+            // content-type we simply don't pixel-verify, rather than being
+            // misread as a corrupt JPEG.
+            const sharpDecodable = SHARP_DECODABLE_TYPES.has(outType);
+            if (sharpDecodable) {
+              // The processed output must actually decode; a corrupt "success" is a bug.
+              const meta = await sharp(dl.rawPayload).metadata();
+              expect(meta.width, `${toolId} x ${fixture}: output not decodable`).toBeGreaterThan(0);
+            }
+          }
+
+          if (res.statusCode === 202 && (toolId === "ocr" || toolId === "ocr-pdf")) {
+            const payload = JSON.parse(res.body) as { jobId?: string };
+            expect(payload.jobId).toBeDefined();
+            await cancelAcceptedJobAndWait(payload.jobId as string, "ai");
+          }
+        }
+        // The nightly avif converters run many slow encodes per test; a busy runner
+        // intermittently overran the old 240s cap, so allow the job's full budget.
+      }, 600_000);
+    }
+  });
+}