fix(security): close the gaps a full 2.0 re-audit left open (#620)

Follow-up to a full re-audit of the 2.0 tree. Most prior findings were already
fixed; this closes the ones that were not:

- SAML assertion replay: validateInResponseTo ifPresent plus a Redis-backed
  CacheProvider, so a captured signed assertion cannot be replayed. ifPresent
  keeps IdP-initiated SSO working.
- MFA login challenge burned after 5 wrong TOTP codes.
- api_keys.key_prefix indexed; the per-request lookup was a full table scan.
- MAX_AI_JOBS_PER_USER caps a user's in-flight single-file AI jobs (the AI pool
  runs at concurrency 1). Batch and pipeline AI stay uncapped.
- MAX_WORKSPACE_SIZE_GB enforced instead of being dead config.
- SUBPROCESS_MEMORY_LIMIT_MB (default off) for the native media and doc engines;
  not applied to the AI sidecar.
- SVG sanitizer closes unquoted and whitespace-prefixed javascript: hrefs and
  the animateTransform/animateMotion/handler/mpath elements.
- Windows-style paths stripped from error output to match the Sentry scrubber.
- Postgres and Redis compose services get cap_drop plus pids_limit and cpus.
- .env.example ships MAX_SVG_SIZE_MB=50 (0 disabled the cap).

Adds security-focused unit and integration tests. typecheck, biome, and the
full unit and integration suites pass.

packages/doc-engine/src/ghostscript.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolveGs } from "./binaries.js";
 
 export type PdfCompressionPreset = "screen" | "ebook" | "printer";
@@ -8,7 +9,8 @@ function runGs(args: string[], timeoutMs = 120_000): Promise<void> {
   const bin = resolveGs();
   if (!bin) throw new Error("gs binary not found (set GS_PATH or install ghostscript)");
   return new Promise<void>((resolvePromise, reject) => {
-    const child = spawn(bin, args, { stdio: ["ignore", "ignore", "pipe"] });
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, args);
+    const child = spawn(limBin, limArgs, { stdio: ["ignore", "ignore", "pipe"] });
     let err = "";
     let settled = false;
     const timer = setTimeout(() => {

packages/doc-engine/src/libreoffice.ts

@@ -4,6 +4,7 @@ import { readdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, extname, join } from "node:path";
 import { pathToFileURL } from "node:url";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolveSoffice } from "./binaries.js";
 
 export interface ConvertOptions {
@@ -48,22 +49,19 @@ export async function convertDocument(
   const { ext, convertTo } = parseConvertTarget(target);
   try {
     await new Promise<void>((resolvePromise, reject) => {
-      const child = spawn(
-        bin,
-        [
-          `-env:UserInstallation=${pathToFileURL(profileDir).href}`,
-          "--headless",
-          "--norestore",
-          "--nolockcheck",
-          "--nodefault",
-          "--convert-to",
-          convertTo,
-          "--outdir",
-          outDir,
-          inputPath,
-        ],
-        { stdio: ["ignore", "pipe", "pipe"] },
-      );
+      const [limBin, limArgs] = wrapWithMemoryLimit(bin, [
+        `-env:UserInstallation=${pathToFileURL(profileDir).href}`,
+        "--headless",
+        "--norestore",
+        "--nolockcheck",
+        "--nodefault",
+        "--convert-to",
+        convertTo,
+        "--outdir",
+        outDir,
+        inputPath,
+      ]);
+      const child = spawn(limBin, limArgs, { stdio: ["ignore", "pipe", "pipe"] });
       let err = "";
       let settled = false;
       const timer = setTimeout(() => {

packages/doc-engine/src/pandoc.ts

@@ -1,4 +1,5 @@
 import { spawn, spawnSync } from "node:child_process";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 
 /** Resolve the pandoc binary, honoring PANDOC_PATH for parity with the other doc-engine wrappers. */
 function pandocBin(): string {
@@ -69,7 +70,8 @@ export function runPandoc(
   const timeoutMs = opts.timeoutMs ?? 120_000;
   const args = buildPandocArgs(inPath, outPath, opts);
   return new Promise<void>((resolvePromise, reject) => {
-    const child = spawn(pandocBin(), args, { stdio: ["ignore", "pipe", "pipe"] });
+    const [limBin, limArgs] = wrapWithMemoryLimit(pandocBin(), args);
+    const child = spawn(limBin, limArgs, { stdio: ["ignore", "pipe", "pipe"] });
     let err = "";
     let settled = false;
     const timer = setTimeout(() => {

packages/doc-engine/src/pdfcpu.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolvePdfcpu } from "./binaries.js";
 
 /**
@@ -10,7 +11,8 @@ function runPdfcpu(args: string[], timeoutMs = 60_000): Promise<string> {
   const bin = resolvePdfcpu();
   if (!bin) throw new Error("pdfcpu binary not found (set PDFCPU_PATH or install pdfcpu)");
   return new Promise<string>((resolvePromise, reject) => {
-    const child = spawn(bin, ["-c", "disable", ...args], {
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, ["-c", "disable", ...args]);
+    const child = spawn(limBin, limArgs, {
       stdio: ["ignore", "pipe", "pipe"],
     });
     let out = "";

packages/doc-engine/src/qpdf.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolveQpdf } from "./binaries.js";
 
 /** @internal Shared qpdf CLI runner for doc-engine modules; not part of the public package API. */
@@ -6,7 +7,8 @@ export function runQpdf(args: string[], timeoutMs = 30_000): Promise<string> {
   const bin = resolveQpdf();
   if (!bin) throw new Error("qpdf binary not found (set QPDF_PATH or install qpdf)");
   return new Promise<string>((resolvePromise, reject) => {
-    const child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, args);
+    const child = spawn(limBin, limArgs, { stdio: ["ignore", "pipe", "pipe"] });
     let out = "";
     let err = "";
     let settled = false;
@@ -63,7 +65,8 @@ export async function qpdfRequiresPassword(filePath: string): Promise<boolean> {
   const bin = resolveQpdf();
   if (!bin) throw new Error("qpdf binary not found (set QPDF_PATH or install qpdf)");
   return new Promise<boolean>((resolvePromise, reject) => {
-    const child = spawn(bin, ["--requires-password", filePath], {
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, ["--requires-password", filePath]);
+    const child = spawn(limBin, limArgs, {
       stdio: ["ignore", "ignore", "pipe"],
     });
     let settled = false;

packages/media-engine/src/ffmpeg.ts

@@ -1,5 +1,5 @@
 import { spawn } from "node:child_process";
-import { markToolInputError, SafeError } from "@snapotter/shared";
+import { markToolInputError, SafeError, wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolveFfmpeg } from "./binaries.js";
 import { type FfmpegProgress, parseProgressBlock } from "./progress.js";
 
@@ -39,7 +39,15 @@ export async function runFfmpeg(args: string[], opts: RunFfmpegOptions = {}): Pr
   const bin = resolveFfmpeg();
   if (!bin) throw new Error("ffmpeg binary not found (set FFMPEG_PATH or install ffmpeg)");
   return new Promise<string>((resolvePromise, reject) => {
-    const child = spawn(bin, ["-hide_banner", "-nostdin", "-y", ...args, "-progress", "pipe:1"], {
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, [
+      "-hide_banner",
+      "-nostdin",
+      "-y",
+      ...args,
+      "-progress",
+      "pipe:1",
+    ]);
+    const child = spawn(limBin, limArgs, {
       stdio: ["ignore", "pipe", "pipe"],
     });
     let stderrTail = "";

packages/media-engine/src/ffprobe.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { wrapWithMemoryLimit } from "@snapotter/shared";
 import { resolveFfprobe } from "./binaries.js";
 import { markIfInputError } from "./ffmpeg.js";
 
@@ -42,7 +43,8 @@ export async function probeMedia(filePath: string, opts: ProbeOptions = {}): Pro
   ];
   const timeoutMs = opts.timeoutMs ?? 15_000;
   const stdout = await new Promise<string>((resolvePromise, reject) => {
-    const child = spawn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
+    const [limBin, limArgs] = wrapWithMemoryLimit(bin, args);
+    const child = spawn(limBin, limArgs, { stdio: ["ignore", "pipe", "pipe"] });
     let out = "";
     let err = "";
     let settled = false;

packages/shared/src/index.ts

@@ -14,5 +14,6 @@ export * from "./permissions.js";
 export * from "./pipeline-templates.js";
 export * from "./search/format-aliases.js";
 export * from "./section.js";
+export * from "./subprocess-limit.js";
 export * from "./tool-errors.js";
 export * from "./types.js";

packages/shared/src/subprocess-limit.ts

@@ -0,0 +1,26 @@
+/**
+ * Optional per-subprocess address-space cap (RLIMIT_AS) for the native media and
+ * document engines.
+ *
+ * When SUBPROCESS_MEMORY_LIMIT_MB is a positive integer, the command runs under
+ * /bin/sh, which sets `ulimit -v` and then `exec`s the real binary with its exact
+ * argv. `exec "$@"` does not re-parse the arguments through the shell, so this
+ * stays injection-safe. A decompression bomb or runaway filter graph is then
+ * killed at that ceiling instead of driving the whole container to the cgroup
+ * OOM-killer (which would take every in-flight job down with it).
+ *
+ * Disabled by default (unset or 0): the container memory limit remains the
+ * primary backstop, and `ulimit -v` is a blunt instrument (it caps virtual
+ * address space, not RSS). `|| true` makes it a no-op where `ulimit -v` is
+ * unsupported, e.g. macOS.
+ *
+ * Deliberately NOT applied to the Python AI sidecar: ML frameworks (torch, CUDA)
+ * reserve very large virtual address space without touching it, so an RLIMIT_AS
+ * cap would break legitimate model loads. Those rely on the container limit.
+ */
+export function wrapWithMemoryLimit(bin: string, args: string[]): [string, string[]] {
+  const mb = Number.parseInt(process.env.SUBPROCESS_MEMORY_LIMIT_MB ?? "", 10);
+  if (!Number.isFinite(mb) || mb <= 0) return [bin, args];
+  const script = 'ulimit -v "$1" 2>/dev/null || true; shift; exec "$@"';
+  return ["/bin/sh", ["-c", script, "sh", String(mb * 1024), bin, ...args]];
+}