fix: release QA hardening across processing, media, security, and CI gates (#649)

A release-readiness QA pass over the whole product. The commits split into defects a user would hit and gates that were reporting green while measuring nothing. ## Fixes that change behaviour Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so request.ip came from a client-set header and a forged X-Forwarded-For got past the login limiter. The default is now a private-network trust list. A transient Postgres outage stranded in-flight jobs, leaving finished output on disk with no row pointing at it. A reconciler now resolves those rows and adopts the bytes rather than dropping the work. A Redis connection that moved to a new address wedged every read-blocked consumer, so completions stopped signalling while health still answered 200. Socket timeouts plus subscriber pings recover it. Installing more than one AI bundle left the shared venv multi-versioned and silently broke three tools. The installer now reconciles distributions to one version each. Converting an image to JXL at quality 1 through 4 returned a 500, because libjxl 0.7 rejects the distance those values compute. The quality is floored at what the encoder honours. A missing ffmpeg was also reported to the user as a corrupt upload; it now says the engine is unavailable. RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at 0.22.2, and the release scan was split so it can fail on an unfixed critical instead of hiding it behind ignore-unfixed. ## Gates that could not fail Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs build; coverage discarded its whole report on any failing test; the lint gate skipped root tests, scripts, and two workspaces; and several generated matrices counted a host missing ffmpeg as a passing tool. Each now measures what it claims. Full evidence and the outstanding release items are tracked locally and are not part of this branch.
2026-08-03 07:46:42 +02:00 · 2026-07-27 15:37:30 +08:00
parent bc32f86a07
commit d10d0f544f
855 changed files with 54564 additions and 13092 deletions
@@ -0,0 +1,547 @@
+#!/usr/bin/env node
+import { readFile, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+import AdmZip from "adm-zip";
+import { assertOracle } from "./oracles.mjs";
+
+const MIN_ARTIFACT_BYTES = 16;
+const SUPPORTED_EXACT_MIMES = new Set([
+  "application/json",
+  "application/pdf",
+  "application/zip",
+  "application/xml",
+  "application/yaml",
+  "application/x-yaml",
+  "application/epub+zip",
+  "application/vnd.apple.mpegurl",
+  "application/vnd.ms-excel",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  "audio/aac",
+  "audio/flac",
+  "audio/mpeg",
+  "audio/mp4",
+  "audio/ogg",
+  "audio/wav",
+  "audio/webm",
+  "audio/x-wav",
+  "image/avif",
+  "image/bmp",
+  "image/gif",
+  "image/heic",
+  "image/heif",
+  "image/jpeg",
+  "image/png",
+  "image/svg+xml",
+  "image/tiff",
+  "image/vnd.microsoft.icon",
+  "image/webp",
+  "image/x-icon",
+  "video/mp4",
+  "video/ogg",
+  "video/quicktime",
+  "video/webm",
+  "video/x-msvideo",
+]);
+
+function normalizeMime(value) {
+  return String(value ?? "")
+    .split(";", 1)[0]
+    .trim()
+    .toLowerCase();
+}
+
+function startsWith(bytes, signature, offset = 0) {
+  if (bytes.length < offset + signature.length) return false;
+  return signature.every((value, index) => bytes[offset + index] === value);
+}
+
+function ascii(bytes, start, end) {
+  return bytes.subarray(start, end).toString("latin1");
+}
+
+function assertExpectedMime(actualMime, expectedMime) {
+  if (!expectedMime) return;
+  const allowed = String(expectedMime).split(",").map(normalizeMime).filter(Boolean);
+  const matches = allowed.some((candidate) =>
+    candidate.endsWith("/*")
+      ? actualMime.startsWith(candidate.slice(0, -1))
+      : actualMime === candidate,
+  );
+  if (!matches) throw new Error(`expected ${allowed.join(" or ")} but received ${actualMime}`);
+}
+
+function assertSuccessfulJson(payload, label) {
+  if (!payload || typeof payload !== "object") throw new Error(`${label} is not a JSON object`);
+  if (payload.success === false || payload.error !== undefined) {
+    throw new Error(`${label} reported failure: ${errorMessage(payload.error)}`);
+  }
+}
+
+function validatedZipEntries(bytes) {
+  try {
+    const archive = new AdmZip(bytes);
+    const entries = archive.getEntries();
+    if (entries.length === 0) throw new Error("archive has no entries");
+    for (const entry of entries) {
+      if (!entry.isDirectory) entry.getData();
+    }
+    return entries;
+  } catch (error) {
+    throw new Error(`ZIP is invalid: ${errorMessage(error)}`);
+  }
+}
+
+export function validateArtifact(output, contentType, options = {}) {
+  const bytes = Buffer.from(output);
+  let mime = normalizeMime(contentType);
+  if (bytes.length < MIN_ARTIFACT_BYTES) {
+    throw new Error(`artifact is trivial (${bytes.length} bytes)`);
+  }
+
+  if (!mime || mime === "application/octet-stream") {
+    if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47])) mime = "image/png";
+    else if (startsWith(bytes, [0xff, 0xd8, 0xff])) mime = "image/jpeg";
+    else if (ascii(bytes, 0, 4) === "%PDF") mime = "application/pdf";
+    else if (ascii(bytes, 0, 2) === "PK") mime = "application/zip";
+    else throw new Error("artifact MIME is missing or generic and magic is unknown");
+  }
+
+  if (!SUPPORTED_EXACT_MIMES.has(mime) && !mime.startsWith("text/")) {
+    throw new Error(`unsupported artifact MIME: ${mime}`);
+  }
+  assertExpectedMime(mime, options.expectedMime);
+
+  const magicChecks = [
+    [mime === "image/png", startsWith(bytes, [0x89, 0x50, 0x4e, 0x47]), "PNG"],
+    [mime === "image/jpeg", startsWith(bytes, [0xff, 0xd8, 0xff]), "JPEG"],
+    [mime === "image/gif", ascii(bytes, 0, 4) === "GIF8", "GIF"],
+    [
+      mime === "image/webp",
+      ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 12) === "WEBP",
+      "WebP",
+    ],
+    [mime === "image/bmp", ascii(bytes, 0, 2) === "BM", "BMP"],
+    [mime === "image/tiff", ["II*\0", "MM\0*"].includes(ascii(bytes, 0, 4)), "TIFF"],
+    [
+      mime === "image/x-icon" || mime === "image/vnd.microsoft.icon",
+      startsWith(bytes, [0, 0, 1, 0]),
+      "ICO",
+    ],
+    [mime === "application/pdf", ascii(bytes, 0, 4) === "%PDF", "PDF"],
+    [mime === "application/zip", ascii(bytes, 0, 2) === "PK", "ZIP"],
+    [
+      mime === "audio/wav" || mime === "audio/x-wav",
+      ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 12) === "WAVE",
+      "WAV",
+    ],
+    [mime === "audio/flac", ascii(bytes, 0, 4) === "fLaC", "FLAC"],
+    [mime === "audio/ogg" || mime === "video/ogg", ascii(bytes, 0, 4) === "OggS", "Ogg"],
+    [
+      mime === "video/webm" || mime === "audio/webm",
+      startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3]),
+      "WebM",
+    ],
+    [
+      mime === "video/mp4" ||
+        mime === "audio/mp4" ||
+        mime === "image/avif" ||
+        mime === "image/heic" ||
+        mime === "image/heif",
+      ascii(bytes, 4, 8) === "ftyp",
+      "ISO BMFF",
+    ],
+  ];
+  for (const [applies, valid, label] of magicChecks) {
+    if (applies && !valid) throw new Error(`${label} magic mismatch for ${mime}`);
+  }
+
+  if (mime === "image/jpeg" && !startsWith(bytes, [0xff, 0xd9], bytes.length - 2)) {
+    throw new Error("JPEG is truncated or missing its end marker");
+  }
+  if (mime === "image/png" && ascii(bytes, bytes.length - 8, bytes.length - 4) !== "IEND") {
+    throw new Error("PNG is truncated or missing its IEND chunk");
+  }
+  if (mime === "image/gif" && bytes[bytes.length - 1] !== 0x3b) {
+    throw new Error("GIF is truncated or missing its trailer");
+  }
+  if (mime === "application/pdf" && !/%%EOF\s*$/.test(bytes.toString("latin1"))) {
+    throw new Error("PDF is truncated or missing %%EOF");
+  }
+  if (mime === "application/zip" || mime.endsWith("+zip") || mime.includes("openxmlformats")) {
+    const entries = validatedZipEntries(bytes);
+    const files = entries.filter((entry) => !entry.isDirectory);
+    if (options.expectedZipEntries !== undefined && files.length !== options.expectedZipEntries) {
+      throw new Error(
+        `expected ${options.expectedZipEntries} ZIP entries but received ${files.length}`,
+      );
+    }
+    if (options.oracle?.zipEach) {
+      for (const entry of files) {
+        try {
+          assertOracle(entry.getData(), options.oracle.zipEach);
+        } catch (error) {
+          throw new Error(`ZIP entry ${entry.entryName}: ${errorMessage(error)}`);
+        }
+      }
+    }
+  }
+
+  if (mime === "audio/mpeg") {
+    const mp3 = ascii(bytes, 0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0);
+    if (!mp3) throw new Error("MP3 magic mismatch for audio/mpeg");
+  }
+  if (mime === "application/json") {
+    let payload;
+    try {
+      payload = JSON.parse(bytes.toString("utf8"));
+    } catch {
+      throw new Error("JSON artifact is not valid JSON");
+    }
+    assertSuccessfulJson(payload, "JSON artifact");
+  }
+  if (mime === "image/svg+xml" && !/^\s*(?:<\?xml[^>]*>\s*)?<svg\b/i.test(bytes.toString("utf8"))) {
+    throw new Error("SVG magic mismatch for image/svg+xml");
+  }
+  if (mime.startsWith("text/") && bytes.toString("utf8").trim().length === 0) {
+    throw new Error(`text artifact is empty for ${mime}`);
+  }
+
+  assertOracle(bytes, options.oracle);
+
+  return { output: bytes, outputMime: mime, outputSize: bytes.length };
+}
+
+function parseJson(bytes, label) {
+  try {
+    return JSON.parse(Buffer.from(bytes).toString("utf8"));
+  } catch {
+    throw new Error(`${label} is not valid JSON`);
+  }
+}
+
+function sameOriginUrl(baseUrl, candidate, label) {
+  const base = new URL(baseUrl);
+  const resolved = new URL(candidate, base);
+  if (resolved.origin !== base.origin)
+    throw new Error(`cross-origin ${label} URL: ${resolved.href}`);
+  return resolved;
+}
+
+function assertFinalResponseOrigin(baseUrl, response, label) {
+  if (!response.url) return;
+  const base = new URL(baseUrl);
+  const final = new URL(response.url);
+  if (final.origin !== base.origin) {
+    throw new Error(`cross-origin final ${label} URL: ${final.href}`);
+  }
+}
+
+function errorMessage(value) {
+  if (typeof value === "string") return value;
+  if (value && typeof value === "object" && typeof value.message === "string") {
+    return value.message;
+  }
+  return value == null ? "unknown error" : JSON.stringify(value);
+}
+
+function terminalEventFromBuffer(buffer) {
+  let terminal;
+  for (const line of buffer.split(/\r?\n/)) {
+    if (!line.startsWith("data:")) continue;
+    let event;
+    try {
+      event = JSON.parse(line.slice(5).trim());
+    } catch {
+      continue;
+    }
+    if (event.phase === "failed" || event.status === "failed") {
+      return { kind: "failed", event };
+    }
+    if (event.phase === "complete" || event.status === "completed") {
+      terminal = { kind: "completed", event };
+    }
+  }
+  return terminal;
+}
+
+export async function waitForTerminalEvent({ baseUrl, token, jobId, timeoutMs, fetchImpl }) {
+  const progressUrl = sameOriginUrl(
+    baseUrl,
+    `/api/v1/jobs/${encodeURIComponent(jobId)}/progress`,
+    "progress",
+  );
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetchImpl(progressUrl, {
+      headers: token ? { authorization: `Bearer ${token}` } : {},
+      signal: controller.signal,
+    });
+    if (!response.ok || !response.body) {
+      throw new Error(`job ${jobId} progress returned HTTP ${response.status}`);
+    }
+    assertFinalResponseOrigin(baseUrl, response, "progress");
+    const progressMime = normalizeMime(response.headers.get("content-type"));
+    if (progressMime !== "text/event-stream") {
+      throw new Error(`job ${jobId} progress returned unsupported content-type ${progressMime}`);
+    }
+
+    let buffer = "";
+    for await (const chunk of response.body) {
+      buffer += Buffer.from(chunk).toString("utf8");
+      const terminal = terminalEventFromBuffer(buffer);
+      if (!terminal) continue;
+      if (terminal.kind === "failed") {
+        throw new Error(`job ${jobId} failed: ${errorMessage(terminal.event.error)}`);
+      }
+      return terminal.event;
+    }
+    throw new Error(`job ${jobId} progress ended without a terminal event`);
+  } catch (error) {
+    if (controller.signal.aborted) throw new Error(`job ${jobId} timed out after ${timeoutMs}ms`);
+    throw error;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function fetchArtifact({
+  baseUrl,
+  token,
+  downloadUrl,
+  timeoutMs,
+  fetchImpl,
+  expectedMime,
+  expectedZipEntries,
+  oracle,
+}) {
+  const url = sameOriginUrl(baseUrl, downloadUrl, "artifact");
+  const response = await fetchImpl(url, {
+    headers: token ? { authorization: `Bearer ${token}` } : {},
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!response.ok) throw new Error(`artifact download returned HTTP ${response.status}`);
+  assertFinalResponseOrigin(baseUrl, response, "artifact");
+  const output = Buffer.from(await response.arrayBuffer());
+  return validateArtifact(output, response.headers.get("content-type"), {
+    expectedMime,
+    expectedZipEntries,
+    oracle,
+  });
+}
+
+async function fallbackDownloadUrl({ baseUrl, token, jobId, timeoutMs, fetchImpl }) {
+  const metaUrl = sameOriginUrl(
+    baseUrl,
+    `/api/v1/download/${encodeURIComponent(jobId)}/output-meta.json`,
+    "output metadata",
+  );
+  const response = await fetchImpl(metaUrl, {
+    headers: token ? { authorization: `Bearer ${token}` } : {},
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!response.ok) throw new Error(`job ${jobId} completed without a downloadable artifact`);
+  assertFinalResponseOrigin(baseUrl, response, "output metadata");
+  if (normalizeMime(response.headers.get("content-type")) !== "application/json") {
+    throw new Error(`job ${jobId} output metadata is not application/json`);
+  }
+  const metadata = await response.json();
+  if (!metadata || typeof metadata.filename !== "string" || metadata.filename.length === 0) {
+    throw new Error(`job ${jobId} output metadata has no filename`);
+  }
+  return `/api/v1/download/${encodeURIComponent(jobId)}/${encodeURIComponent(metadata.filename)}`;
+}
+
+export async function resolveBenchmarkResponse({
+  baseUrl,
+  token = "",
+  admissionStatus,
+  admissionMime,
+  admissionBody,
+  admissionLatencyS = 0,
+  timeoutMs = 300_000,
+  fetchImpl = globalThis.fetch,
+  expectedMime,
+  expectedZipEntries,
+  oracle,
+}) {
+  const started = performance.now();
+  let artifact;
+
+  if (admissionStatus === 200) {
+    if (normalizeMime(admissionMime) === "application/json") {
+      const payload = parseJson(admissionBody, "200 response");
+      assertSuccessfulJson(payload, "200 response");
+      if (typeof payload.downloadUrl === "string") {
+        artifact = await fetchArtifact({
+          baseUrl,
+          token,
+          downloadUrl: payload.downloadUrl,
+          timeoutMs,
+          fetchImpl,
+          expectedMime,
+          expectedZipEntries,
+          oracle,
+        });
+      } else {
+        artifact = validateArtifact(admissionBody, admissionMime, {
+          expectedMime,
+          expectedZipEntries,
+          oracle,
+        });
+      }
+    } else {
+      artifact = validateArtifact(admissionBody, admissionMime, {
+        expectedMime,
+        expectedZipEntries,
+        oracle,
+      });
+    }
+  } else if (admissionStatus === 202) {
+    const payload = parseJson(admissionBody, "202 response");
+    if (typeof payload.jobId !== "string" || payload.jobId.length === 0) {
+      throw new Error("202 response has no jobId");
+    }
+    const terminal = await waitForTerminalEvent({
+      baseUrl,
+      token,
+      jobId: payload.jobId,
+      timeoutMs,
+      fetchImpl,
+    });
+    const failedFiles = Number(terminal.failedFiles ?? 0);
+    const errors = Array.isArray(terminal.errors) ? terminal.errors : [];
+    if (failedFiles > 0 || errors.length > 0) {
+      throw new Error(
+        `batch ${payload.jobId} completed with ${Math.max(failedFiles, errors.length)} failed file(s)`,
+      );
+    }
+    const totalFiles = Number(terminal.totalFiles);
+    const completedFiles = Number(terminal.completedFiles);
+    if (
+      Number.isFinite(totalFiles) &&
+      totalFiles > 0 &&
+      (!Number.isFinite(completedFiles) || completedFiles !== totalFiles)
+    ) {
+      throw new Error(
+        `batch ${payload.jobId} completed partially (${completedFiles}/${totalFiles} files)`,
+      );
+    }
+    const nestedResult =
+      terminal.result && typeof terminal.result === "object" ? terminal.result : {};
+    assertSuccessfulJson(nestedResult, `job ${payload.jobId} terminal result`);
+    const artifactJobId =
+      typeof payload.artifactJobId === "string" && payload.artifactJobId.length > 0
+        ? payload.artifactJobId
+        : payload.jobId;
+    const downloadUrl =
+      typeof terminal.downloadUrl === "string"
+        ? terminal.downloadUrl
+        : typeof nestedResult.downloadUrl === "string"
+          ? nestedResult.downloadUrl
+          : await fallbackDownloadUrl({
+              baseUrl,
+              token,
+              jobId: artifactJobId,
+              timeoutMs,
+              fetchImpl,
+            });
+    artifact = await fetchArtifact({
+      baseUrl,
+      token,
+      downloadUrl,
+      timeoutMs,
+      fetchImpl,
+      expectedMime,
+      expectedZipEntries:
+        expectedZipEntries ??
+        (Number.isFinite(totalFiles) && totalFiles > 0 ? totalFiles : undefined),
+      oracle,
+    });
+  } else {
+    throw new Error(`admission returned HTTP ${admissionStatus}`);
+  }
+
+  return {
+    admissionStatus,
+    completionStatus: "completed",
+    completionLatencyS: Number(admissionLatencyS) + (performance.now() - started) / 1_000,
+    ...artifact,
+  };
+}
+
+function parseArgs(argv) {
+  const args = {};
+  for (let index = 0; index < argv.length; index += 2) {
+    const key = argv[index];
+    const value = argv[index + 1];
+    if (!key?.startsWith("--") || value === undefined)
+      throw new Error(`invalid argument ${key ?? ""}`);
+    args[key.slice(2)] = value;
+  }
+  return args;
+}
+
+function safeField(value) {
+  return String(value ?? "-").replace(/[\t\r\n]/g, " ");
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const started = performance.now();
+  const admissionStatus = Number(args.status);
+  const admissionLatencyS = Number(args["admission-latency"] ?? 0);
+  try {
+    const result = await resolveBenchmarkResponse({
+      baseUrl: args["base-url"],
+      token: args.token ?? "",
+      admissionStatus,
+      admissionMime: args.mime,
+      admissionBody: await readFile(args.body),
+      admissionLatencyS,
+      timeoutMs: Number(args["timeout-ms"] ?? 300_000),
+      expectedMime: args["expected-mime"],
+      expectedZipEntries:
+        args["expected-zip-entries"] === undefined
+          ? undefined
+          : Number(args["expected-zip-entries"]),
+      oracle: args.oracle === undefined ? undefined : JSON.parse(args.oracle),
+    });
+    if (args.output) await writeFile(args.output, result.output);
+    process.stdout.write(
+      `${[
+        "true",
+        result.admissionStatus,
+        result.completionStatus,
+        result.completionLatencyS.toFixed(3),
+        result.outputSize,
+        result.outputMime,
+        "-",
+      ]
+        .map(safeField)
+        .join("\t")}\n`,
+    );
+  } catch (error) {
+    const latency = admissionLatencyS + (performance.now() - started) / 1_000;
+    process.stdout.write(
+      `${[
+        "false",
+        admissionStatus || 0,
+        "failed",
+        latency.toFixed(3),
+        0,
+        "unknown",
+        errorMessage(error),
+      ]
+        .map(safeField)
+        .join("\t")}\n`,
+    );
+    process.exitCode = 1;
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  await main();
+}
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+JOB_AWARE_LIB_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+JOB_AWARE_NODE="${JOB_AWARE_LIB_DIR}/job-aware.mjs"
+BENCH_JOB_TIMEOUT_MS="${BENCH_JOB_TIMEOUT_MS:-300000}"
+BENCHMARK_FAILURES=0
+BENCH_CHILD_EXIT_FAILURES=0
+BENCH_OBSERVED_ROWS=0
+BENCH_EXPECTED_ROWS=0
+
+resolve_benchmark_response() {
+  local base_url="$1" token="$2" status="$3" mime="$4" body_file="$5" output_file="$6"
+  local admission_latency="$7" timeout_ms="${8:-$BENCH_JOB_TIMEOUT_MS}"
+  local expected_mime="${9:-}" expected_zip_entries="${10:-}" oracle="${11:-}"
+  local resolution rc
+  local resolver_args=(
+    --base-url "$base_url"
+    --token "$token"
+    --status "$status"
+    --mime "${mime:-application/octet-stream}"
+    --body "$body_file"
+    --output "$output_file"
+    --admission-latency "$admission_latency"
+    --timeout-ms "$timeout_ms"
+  )
+  if [ -n "$expected_mime" ]; then
+    resolver_args+=(--expected-mime "$expected_mime")
+  fi
+  if [ -n "$expected_zip_entries" ]; then
+    resolver_args+=(--expected-zip-entries "$expected_zip_entries")
+  fi
+  if [ -n "$oracle" ]; then
+    resolver_args+=(--oracle "$oracle")
+  fi
+
+  if resolution=$(node "$JOB_AWARE_NODE" "${resolver_args[@]}"); then
+    rc=0
+  else
+    rc=$?
+  fi
+
+  IFS=$'\t' read -r \
+    BENCH_PASS \
+    BENCH_ADMISSION_STATUS \
+    BENCH_COMPLETION_STATUS \
+    BENCH_COMPLETION_LATENCY_S \
+    BENCH_OUTPUT_SIZE \
+    BENCH_OUTPUT_MIME \
+    BENCH_ERROR <<< "$resolution"
+
+  if [ "$rc" -ne 0 ]; then
+    BENCHMARK_FAILURES=$((BENCHMARK_FAILURES + 1))
+    return "$rc"
+  fi
+  return 0
+}
+
+benchmark_assert_success() {
+  if [ "$BENCHMARK_FAILURES" -gt 0 ]; then
+    echo "Benchmark failed: ${BENCHMARK_FAILURES} request(s) timed out or returned invalid output" >&2
+    return 1
+  fi
+}
+
+wait_for_benchmark_children() {
+  local expected_rows="$1" results_file="$2"
+  shift 2
+  local pid
+  BENCH_CHILD_EXIT_FAILURES=0
+  BENCH_EXPECTED_ROWS="$expected_rows"
+
+  for pid in "$@"; do
+    if ! wait "$pid"; then
+      BENCH_CHILD_EXIT_FAILURES=$((BENCH_CHILD_EXIT_FAILURES + 1))
+    fi
+  done
+
+  BENCH_OBSERVED_ROWS=$(wc -l < "$results_file" | tr -d '[:space:]')
+  if [ "$BENCH_CHILD_EXIT_FAILURES" -gt 0 ] || [ "$BENCH_OBSERVED_ROWS" -ne "$expected_rows" ]; then
+    return 1
+  fi
+  return 0
+}
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Container resource readings shared by the benchmark scripts.
+#
+# `docker stats` prints MemUsage with a unit ("512MiB / 6GiB", "1.68GiB / 6GiB").
+# Stripping the unit and calling the number MB reports a 1.68 GiB container as
+# 1.68 MB, which is exactly how a memory leak hides inside a green benchmark row.
+
+# Memory used by a container, in MiB. Emits 0 when the reference is empty or
+# `docker stats` produces nothing, so a caller can always treat it as a number.
+docker_mem_mb() {
+  local ref="$1"
+  if [ -z "$ref" ]; then echo "0"; return; fi
+  docker stats "$ref" --no-stream --format "{{.MemUsage}}" 2>/dev/null \
+    | awk -F/ '{
+        value = $1
+        unit = "MiB"
+        if (match(value, /[KMGT]i?B/)) unit = substr(value, RSTART, RLENGTH)
+        gsub(/[^0-9.]/, "", value)
+        if (value + 0 <= 0) { print 0; exit }
+        factor = 1
+        if (unit ~ /^G/) factor = 1024
+        else if (unit ~ /^T/) factor = 1048576
+        else if (unit ~ /^K/) factor = 1 / 1024
+        else if (unit ~ /^B/) factor = 1 / 1048576
+        printf "%.2f\n", value * factor
+      }' \
+    | { read -r reading || reading=""; echo "${reading:-0}"; }
+}
+
+# CPU percentage for a container as a bare number.
+docker_cpu_pct() {
+  local ref="$1"
+  if [ -z "$ref" ]; then echo "0"; return; fi
+  docker stats "$ref" --no-stream --format "{{.CPUPerc}}" 2>/dev/null | tr -d '%' \
+    | { read -r reading || reading=""; echo "${reading:-0}"; }
+}
@@ -0,0 +1,266 @@
+/**
+ * Semantic oracles for benchmark artifacts.
+ *
+ * `validateArtifact` in job-aware.mjs proves an artifact is a structurally
+ * intact file of the declared type. That still lets a fast wrong answer pass:
+ * a resize that ignores its width, a page extraction that returns the whole
+ * document, a trim that returns the untrimmed audio are all valid PNG/PDF/WAV
+ * bytes. These oracles read the property the operation was asked to change out
+ * of the output itself, so a benchmark row can only be green when the work was
+ * actually done.
+ *
+ * Everything here is pure and dependency-free (node:zlib only), so the oracles
+ * run on the measuring host without needing ffprobe, qpdf or Sharp.
+ */
+import { inflateSync } from "node:zlib";
+
+function ascii(bytes, start, end) {
+  return bytes.subarray(start, end).toString("latin1");
+}
+
+/** PNG: the IHDR chunk always sits at byte 16 and is big-endian. */
+function pngDimensions(bytes) {
+  if (ascii(bytes, 12, 16) !== "IHDR") return null;
+  return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
+}
+
+/**
+ * JPEG: walk the marker chain to the first SOFn frame header. SOF4 (0xC4),
+ * SOF8 (0xC8) and SOF12 (0xCC) are DHT/JPG/DAC, not frame headers.
+ */
+function jpegDimensions(bytes) {
+  let offset = 2;
+  while (offset + 9 < bytes.length) {
+    if (bytes[offset] !== 0xff) {
+      offset += 1;
+      continue;
+    }
+    const marker = bytes[offset + 1];
+    if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
+      offset += 2;
+      continue;
+    }
+    const length = bytes.readUInt16BE(offset + 2);
+    const isFrameHeader =
+      marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
+    if (isFrameHeader) {
+      return { height: bytes.readUInt16BE(offset + 5), width: bytes.readUInt16BE(offset + 7) };
+    }
+    if (marker === 0xda) return null;
+    offset += 2 + length;
+  }
+  return null;
+}
+
+/** GIF: logical screen descriptor, little-endian, immediately after the header. */
+function gifDimensions(bytes) {
+  return { width: bytes.readUInt16LE(6), height: bytes.readUInt16LE(8) };
+}
+
+/** WebP: simple lossy (VP8), lossless (VP8L) and extended (VP8X) all differ. */
+function webpDimensions(bytes) {
+  const chunk = ascii(bytes, 12, 16);
+  if (chunk === "VP8X") {
+    return {
+      width: 1 + (bytes[24] | (bytes[25] << 8) | (bytes[26] << 16)),
+      height: 1 + (bytes[27] | (bytes[28] << 8) | (bytes[29] << 16)),
+    };
+  }
+  if (chunk === "VP8 ") {
+    return { width: bytes.readUInt16LE(26) & 0x3fff, height: bytes.readUInt16LE(28) & 0x3fff };
+  }
+  if (chunk === "VP8L") {
+    const bits = bytes.readUInt32LE(21);
+    return { width: (bits & 0x3fff) + 1, height: ((bits >> 14) & 0x3fff) + 1 };
+  }
+  return null;
+}
+
+/** BMP: DIB header dimensions are signed (a negative height is top-down). */
+function bmpDimensions(bytes) {
+  return { width: bytes.readInt32LE(18), height: Math.abs(bytes.readInt32LE(22)) };
+}
+
+/**
+ * Pixel dimensions of an encoded image, or null when the format is one this
+ * module deliberately does not parse (AVIF, HEIC, TIFF, SVG).
+ */
+export function imageDimensions(input) {
+  const bytes = Buffer.from(input);
+  if (bytes.length < 32) return null;
+  if (ascii(bytes, 1, 4) === "PNG") return pngDimensions(bytes);
+  if (bytes[0] === 0xff && bytes[1] === 0xd8) return jpegDimensions(bytes);
+  if (ascii(bytes, 0, 3) === "GIF") return gifDimensions(bytes);
+  if (ascii(bytes, 0, 4) === "RIFF" && ascii(bytes, 8, 12) === "WEBP") return webpDimensions(bytes);
+  if (ascii(bytes, 0, 2) === "BM") return bmpDimensions(bytes);
+  return null;
+}
+
+const PAGE_OBJECT = /\/Type\s*\/Page(?![sA-Za-z])/g;
+
+function countPageObjects(text) {
+  return (text.match(PAGE_OBJECT) ?? []).length;
+}
+
+/**
+ * Number of pages in a PDF.
+ *
+ * Classic PDFs keep page objects in the clear; anything written through a
+ * cross-reference stream (qpdf and pdfcpu both do by default) hides them
+ * inside FlateDecode object streams, so raw regex counting silently returns 0
+ * there. Inflating every Flate stream first is what makes this oracle work on
+ * real tool output rather than only on hand-built fixtures.
+ */
+export function pdfPageCount(input) {
+  const bytes = Buffer.from(input);
+  let pages = countPageObjects(bytes.toString("latin1"));
+
+  const open = Buffer.from("stream", "latin1");
+  const close = Buffer.from("endstream", "latin1");
+  let cursor = 0;
+  while (cursor < bytes.length) {
+    const start = bytes.indexOf(open, cursor);
+    if (start === -1) break;
+    // "endstream" contains "stream"; matching it would desynchronise the scan.
+    if (start >= 3 && ascii(bytes, start - 3, start) === "end") {
+      cursor = start + open.length;
+      continue;
+    }
+    let payload = start + open.length;
+    if (bytes[payload] === 0x0d) payload += 1;
+    if (bytes[payload] === 0x0a) payload += 1;
+    const end = bytes.indexOf(close, payload);
+    if (end === -1) break;
+    cursor = end + close.length;
+    try {
+      pages += countPageObjects(inflateSync(bytes.subarray(payload, end)).toString("latin1"));
+    } catch {
+      // Not a Flate stream (JPEG image data, plain content). Skipping is
+      // correct: those streams never carry page objects.
+    }
+  }
+  return pages;
+}
+
+/** Duration in seconds of a RIFF/WAVE file, from its fmt and data chunks. */
+export function wavDurationS(input) {
+  const bytes = Buffer.from(input);
+  if (ascii(bytes, 0, 4) !== "RIFF" || ascii(bytes, 8, 12) !== "WAVE") return null;
+  let cursor = 12;
+  let byteRate = 0;
+  while (cursor + 8 <= bytes.length) {
+    const id = ascii(bytes, cursor, cursor + 4);
+    const size = bytes.readUInt32LE(cursor + 4);
+    if (id === "fmt ") byteRate = bytes.readUInt32LE(cursor + 16);
+    if (id === "data") return byteRate > 0 ? size / byteRate : null;
+    cursor += 8 + size + (size % 2);
+  }
+  return null;
+}
+
+/**
+ * Duration in seconds of an ISO base media file (MP4/MOV/M4A), from the mvhd
+ * box. Version 1 mvhd uses 64-bit times; the low 32 bits are enough for any
+ * duration a benchmark produces.
+ */
+export function isoDurationS(input) {
+  const bytes = Buffer.from(input);
+  const mvhd = bytes.indexOf(Buffer.from("mvhd", "latin1"));
+  if (mvhd === -1) return null;
+  const version = bytes[mvhd + 4];
+  if (version === 1) {
+    if (mvhd + 36 > bytes.length) return null;
+    const timescale = bytes.readUInt32BE(mvhd + 24);
+    const duration = Number(bytes.readBigUInt64BE(mvhd + 28));
+    return timescale > 0 ? duration / timescale : null;
+  }
+  if (mvhd + 24 > bytes.length) return null;
+  const timescale = bytes.readUInt32BE(mvhd + 16);
+  const duration = bytes.readUInt32BE(mvhd + 20);
+  return timescale > 0 ? duration / timescale : null;
+}
+
+/** Number of Matroska/WebM clusters, a cheap "this actually has content" probe. */
+export function webmHasClusters(input) {
+  return Buffer.from(input).includes(Buffer.from([0x1f, 0x43, 0xb6, 0x75]));
+}
+
+function fail(label, expected, actual) {
+  throw new Error(`oracle ${label}: expected ${expected}, measured ${actual}`);
+}
+
+function assertClose(label, actual, expected, tolerance) {
+  if (actual === null || !Number.isFinite(actual)) fail(label, expected, "unreadable");
+  if (Math.abs(actual - expected) > tolerance) {
+    fail(label, `${expected} +/- ${tolerance}`, actual.toFixed(3));
+  }
+}
+
+/**
+ * Apply a declarative oracle to artifact bytes. Every supported key names a
+ * property the tool under benchmark was explicitly asked to produce.
+ *
+ *   { width: 800 }                image output is exactly 800 px wide
+ *   { height: 600 }
+ *   { pages: 3 }                  PDF has exactly 3 pages
+ *   { durationS: 5, toleranceS: 0.4 }
+ *   { minBytes: 1024 }
+ *   { json: { key: "value" } }    JSON output contains these fields
+ *   { textIncludes: "hello" }
+ */
+export function assertOracle(input, oracle) {
+  if (!oracle) return;
+  const bytes = Buffer.from(input);
+
+  if (oracle.width !== undefined || oracle.height !== undefined) {
+    const dimensions = imageDimensions(bytes);
+    if (!dimensions) fail("dimensions", "a parseable raster header", "unreadable");
+    if (oracle.width !== undefined && dimensions.width !== oracle.width) {
+      fail("width", oracle.width, dimensions.width);
+    }
+    if (oracle.height !== undefined && dimensions.height !== oracle.height) {
+      fail("height", oracle.height, dimensions.height);
+    }
+  }
+
+  if (oracle.pages !== undefined) {
+    const pages = pdfPageCount(bytes);
+    if (pages !== oracle.pages) fail("pages", oracle.pages, pages);
+  }
+
+  if (oracle.durationS !== undefined) {
+    const tolerance = oracle.toleranceS ?? 0.5;
+    const duration = wavDurationS(bytes) ?? isoDurationS(bytes);
+    assertClose("durationS", duration, oracle.durationS, tolerance);
+  }
+
+  if (oracle.minBytes !== undefined && bytes.length < oracle.minBytes) {
+    fail("minBytes", `>= ${oracle.minBytes}`, bytes.length);
+  }
+
+  if (oracle.maxBytes !== undefined && bytes.length > oracle.maxBytes) {
+    fail("maxBytes", `<= ${oracle.maxBytes}`, bytes.length);
+  }
+
+  if (oracle.json !== undefined) {
+    let payload;
+    try {
+      payload = JSON.parse(bytes.toString("utf8"));
+    } catch {
+      fail("json", "parseable JSON", "unparseable");
+    }
+    for (const [key, expected] of Object.entries(oracle.json)) {
+      const actual = key.split(".").reduce((node, part) => node?.[part], payload);
+      if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+        fail(`json.${key}`, JSON.stringify(expected), JSON.stringify(actual));
+      }
+    }
+  }
+
+  if (oracle.textIncludes !== undefined) {
+    const text = bytes.toString("utf8");
+    if (!text.includes(oracle.textIncludes)) {
+      fail("textIncludes", JSON.stringify(oracle.textIncludes), "absent");
+    }
+  }
+}