// Campaign master-20260724, AI lane (cells AI-001 / AI-002 / PRIOR-001). // // Runs a real modality fixture through EVERY tool a feature bundle enables and // asserts a tool-specific oracle on the artifact, against a running container. // The bundle -> tool mapping is read from the live shared exports, never from a // hand-maintained list, so a new bundle or a new `enablesTools` entry shows up // here automatically. // // Usage: // QA_BASE_URL=http://localhost:13601 QA_USERNAME=admin QA_PASSWORD=... \ // QA_PROFILE=ubuntu-gpu-amd64 QA_OUT=/path/results.jsonl \ // [QA_BUNDLES=face-detection,transcription] [QA_EXPECT=gated|installed] \ // apps/api/node_modules/.bin/tsx tests/qa/verify-ai-tool-matrix.mts // // QA_EXPECT=gated inverts the assertion: every tool must be REFUSED because its // bundle is not installed. That is how the uninstall/reset path is proven. import { appendFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { apiToolPath } from "../../packages/shared/src/constants.js"; import { FEATURE_BUNDLES, getRequiredBundlesForTool } from "../../packages/shared/src/features.js"; import { buildCenteredRectMask, meanAbsoluteDifference } from "../helpers/ai-fixture-builders.js"; import { expectAnimatedFrames, expectBackgroundBlurEnergyReduced, expectBackgroundCutOut, expectCanvasExpanded, expectColorAdded, expectConfiguredBackground, expectForegroundPreserved, expectHighFrequencyEnergyReduced, expectJapaneseScript, expectKnownTranscript, expectNonDegenerateImage, expectObservablePixelChange, expectRecognizedTerms, expectRedPixelsReduced, expectRegionRewritten, expectSameSizeButChanged, expectSearchablePdf, expectSrtArtifact, expectUpscaled, } from "../helpers/installed-ai-output-oracles.js"; const BASE = (process.env.QA_BASE_URL ?? "").replace(/\/$/u, ""); const USERNAME = process.env.QA_USERNAME ?? "admin"; const PASSWORD = process.env.QA_PASSWORD ?? ""; const PROFILE = process.env.QA_PROFILE ?? "unknown-profile"; const OUT = process.env.QA_OUT ?? ""; const EXPECT = process.env.QA_EXPECT === "gated" ? "gated" : "installed"; const JOB_TIMEOUT_MS = Number(process.env.QA_JOB_TIMEOUT_MS ?? 20 * 60_000); const REQUEST_TIMEOUT_MS = Number(process.env.QA_REQUEST_TIMEOUT_MS ?? 180_000); if (!BASE || !PASSWORD) throw new Error("QA_BASE_URL and QA_PASSWORD are required"); const FIXTURES = "tests/fixtures"; /** Terminal or intermediate frame on the job SSE stream. */ interface ProgressFrame { error?: string; jobId?: string; phase?: string; result?: { downloadUrl?: string; processedSize?: number }; type?: string; } /** The body of a 200 or 202 tool submission. */ interface AcceptedPayload { downloadUrl?: string; jobId?: string; } interface ToolCase { /** * Tools whose real work sits behind a sub-route (passport-photo) or that need * a second request to form an oracle (smart-crop) supply their own runner. */ customRun?: (ctx: { input: Buffer; post: PostFn; token: string }) => Promise; /** Extra multipart parts beyond `file`, e.g. the eraser mask. */ extraFiles?: { contentType: string; field: string; path: string }[]; contentType: string; expectedMime?: string | string[]; fixture: string; /** Human note recorded in the evidence row. */ oracle: string; settings: Record; toolId: string; verify: (output: Buffer, input: Buffer) => Promise | void; } const PORTRAIT = `${FIXTURES}/image/valid/portrait-color.jpg`; const PORTRAIT_BW = `${FIXTURES}/image/valid/portrait-bw.jpeg`; const MULTI_FACE = `${FIXTURES}/image/valid/multi-face.webp`; const RED_EYE = `${FIXTURES}/image/valid/red-eye.jpg`; const SAMPLE = `${FIXTURES}/image/valid/sample-photo.jpg`; const ISOLATED = `${FIXTURES}/image/valid/portrait-isolated.png`; const OCR_CLEAN = `${FIXTURES}/image/valid/ocr-clean.png`; const OCR_JA = `${FIXTURES}/image/valid/ocr-japanese.png`; // animated.gif is a flat synthetic gradient with no subject, so a cut-out // legitimately erases every pixel. The Simpsons clip has a real subject. const ANIMATED = `${FIXTURES}/image/valid/animated-simpsons.gif`; const SPEECH_WAV = `${FIXTURES}/audio/valid/speech-10s.wav`; const SPEECH_MP4 = `${FIXTURES}/video/valid/speech-10s.mp4`; /** The English OCR fixture's known answer, checked word by word. */ const OCR_ENGLISH_TERMS = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog"] as const; const CASES: ToolCase[] = [ { toolId: "remove-background", fixture: PORTRAIT, contentType: "image/jpeg", settings: { backgroundType: "transparent", format: "png" }, expectedMime: "image/png", oracle: "border >50% transparent, subject centre >80% opaque, non-degenerate", verify: async (out) => { await expectNonDegenerateImage(out); await expectBackgroundCutOut(out); }, }, { toolId: "background-replace", fixture: PORTRAIT, contentType: "image/jpeg", settings: { backgroundType: "color", color: "#ff0000", format: "png" }, expectedMime: "image/png", oracle: "solid red visible at border, subject preserved, pixels changed", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectObservablePixelChange(input, out); await expectConfiguredBackground(out, "solid-red"); await expectForegroundPreserved(input, out); }, }, { toolId: "blur-background", fixture: PORTRAIT, contentType: "image/jpeg", settings: { intensity: 75, feather: 3, format: "webp" }, expectedMime: "image/webp", oracle: "background HF energy ratio < 0.7, subject preserved", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectObservablePixelChange(input, out); await expectBackgroundBlurEnergyReduced(input, out); await expectForegroundPreserved(input, out); }, }, { toolId: "transparency-fixer", fixture: ISOLATED, contentType: "image/png", settings: { defringe: 50, outputFormat: "png" }, expectedMime: "image/png", oracle: "decodes, non-degenerate, alpha retained", verify: async (out) => { const stats = await expectNonDegenerateImage(out); if (stats.transparentFraction < 0.01) { throw new Error("transparency-fixer output carries no transparent pixels at all"); } }, }, { // The AI work (face landmarks + background removal) lives behind /analyze; // the base route deliberately 400s with a pointer to the sub-routes. toolId: "passport-photo", fixture: PORTRAIT, contentType: "image/jpeg", settings: {}, oracle: "analyze returns >=4 landmarks and a non-degenerate cut-out preview", verify: () => {}, customRun: async ({ input, post }) => { const { status, body } = await post( "/api/v1/tools/image/passport-photo/analyze", input, "portrait-color.jpg", "image/jpeg", {}, ); if (status !== 200) throw new Error(`passport-photo analyze HTTP ${status}: ${body.slice(0, 300)}`); const payload = JSON.parse(body) as Record; const landmarks = payload.landmarks as Record | undefined; if (!landmarks || Object.keys(landmarks).length < 4) { throw new Error( `passport-photo analyze returned ${Object.keys(landmarks ?? {}).length} landmark groups`, ); } if (typeof payload.preview !== "string" || payload.preview.length < 1000) { throw new Error("passport-photo analyze returned no usable preview"); } const preview = Buffer.from(payload.preview, "base64"); await expectNonDegenerateImage(preview); await expectBackgroundCutOut(preview); }, }, { toolId: "remove-gif-background", fixture: ANIMATED, contentType: "image/gif", settings: { outputFormat: "webp", backgroundType: "transparent" }, oracle: "still animated (>=2 frames) and non-degenerate", verify: async (out) => { await expectNonDegenerateImage(out); await expectAnimatedFrames(out, 2); }, }, { toolId: "blur-faces", fixture: MULTI_FACE, contentType: "image/webp", settings: { blurRadius: 40, sensitivity: 0.5 }, oracle: "same size, >=1% of pixels rewritten, HF energy falls", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectSameSizeButChanged(input, out, 0.01); await expectHighFrequencyEnergyReduced(input, out, 0.99); }, }, { toolId: "red-eye-removal", fixture: RED_EYE, contentType: "image/jpeg", settings: { sensitivity: 50, strength: 80, quality: 95 }, oracle: "saturated-red pixel count strictly reduced", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectRedPixelsReduced(input, out); }, }, { // A bare `mode: face` crop can legitimately be LARGER than the input (the // route pads to the requested aspect), so size alone proves nothing. Pin an // explicit output size and prove the face-guided window differs from the // geometry-only trim window on the same fixture. toolId: "smart-crop", fixture: MULTI_FACE, contentType: "image/webp", settings: { mode: "face", width: 256, height: 256, padding: 10 }, oracle: "exactly 256x256, non-degenerate, and different from the mode=trim crop", verify: () => {}, customRun: async ({ input, post }) => { const run = async (settings: Record): Promise => { const { status, body } = await post( apiToolPath("smart-crop"), input, "multi-face.webp", "image/webp", settings, ); if (status !== 200 && status !== 202) { throw new Error(`smart-crop HTTP ${status}: ${body.slice(0, 300)}`); } return downloadArtifact(JSON.parse(body), status); }; const face = await run({ mode: "face", width: 256, height: 256, padding: 10 }); const trim = await run({ mode: "trim", width: 256, height: 256, padding: 10 }); const stats = await expectNonDegenerateImage(face); if (stats.width !== 256 || stats.height !== 256) { throw new Error( `smart-crop returned ${stats.width}x${stats.height}, not the requested 256x256`, ); } const delta = await meanAbsoluteDifference(face, trim); if (!(delta > 2)) { throw new Error( `face-guided and geometry-only crops are near identical (mean abs diff ${delta.toFixed(3)})`, ); } }, }, { toolId: "erase-object", fixture: SAMPLE, contentType: "image/jpeg", settings: { format: "png", quality: 95, qualityMode: "fast" }, expectedMime: "image/png", oracle: "masked rectangle >35% rewritten, same dimensions (fast LaMa and hq diffusion)", verify: () => {}, customRun: async ({ input, post }) => { const { mask, region } = await buildCenteredRectMask(input, 0.3); const artifacts: Record = {}; // The hq (diffusion) leg only applies where the optional inpaint-hq pack // is installed; elsewhere the route correctly refuses it. const modes = installedBundleIds.has("inpaint-hq") ? (["fast", "hq"] as const) : (["fast"] as const); for (const qualityMode of modes) { const { status, body } = await post( apiToolPath("erase-object"), input, "sample-photo.jpg", "image/jpeg", { format: "png", quality: 95, qualityMode }, [{ field: "mask", bytes: mask, filename: "mask.png", contentType: "image/png" }], true, ); if (status !== 200 && status !== 202) { throw new Error(`erase-object[${qualityMode}] HTTP ${status}: ${body.slice(0, 300)}`); } const artifact = await downloadArtifact(JSON.parse(body), status); await expectNonDegenerateImage(artifact); await expectRegionRewritten(input, artifact, region); artifacts[qualityMode] = artifact; } // A diffusion result that is byte-identical to the LaMa result means the // hq path silently no-opped, which the API would otherwise report as a // clean success. if (artifacts.hq) { const hqDelta = await meanAbsoluteDifference(artifacts.fast, artifacts.hq); if (!(hqDelta > 1)) { throw new Error( `erase-object qualityMode=hq is indistinguishable from fast (mean abs diff ${hqDelta.toFixed(3)}), so the diffusion path did not run`, ); } } }, }, { toolId: "colorize", fixture: PORTRAIT_BW, contentType: "image/jpeg", settings: { intensity: 1, model: "auto" }, oracle: "input chroma < 12, output chroma > input + 6", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectColorAdded(input, out); }, }, { toolId: "ai-canvas-expand", fixture: PORTRAIT, contentType: "image/jpeg", settings: { extendTop: 0, extendRight: 64, extendBottom: 0, extendLeft: 64, tier: "fast", format: "png", quality: 95, }, oracle: "width == input + 128, height unchanged", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectCanvasExpanded(input, out, { top: 0, right: 64, bottom: 0, left: 64 }); }, }, { toolId: "upscale", fixture: `${FIXTURES}/image/valid/test-100x100.jpg`, contentType: "image/jpeg", settings: { scale: 2 }, oracle: "output width >= 2x input width, aspect preserved", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectUpscaled(input, out, 2); }, }, { toolId: "enhance-faces", fixture: PORTRAIT, contentType: "image/jpeg", settings: { model: "auto", strength: 0.8, onlyCenterFace: false, sensitivity: 0.5 }, oracle: "same size, >=0.5% pixels rewritten", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectSameSizeButChanged(input, out, 0.005); }, }, { toolId: "noise-removal", fixture: SAMPLE, contentType: "image/jpeg", settings: { tier: "balanced", strength: 60, format: "png" }, expectedMime: "image/png", oracle: "high-frequency energy ratio < 0.95", verify: async (out, input) => { await expectNonDegenerateImage(out); await expectHighFrequencyEnergyReduced(input, out, 0.95); }, }, { toolId: "restore-photo", fixture: PORTRAIT_BW, contentType: "image/jpeg", settings: { scratchRemoval: true, faceEnhancement: true, denoise: true, colorize: false }, oracle: "non-degenerate, >=0.5% pixels rewritten at the same or larger size", verify: async (out, input) => { const after = await expectNonDegenerateImage(out); const before = await expectNonDegenerateImage(input); if (after.width === before.width && after.height === before.height) { await expectSameSizeButChanged(input, out, 0.005); } }, }, { toolId: "ocr", fixture: OCR_CLEAN, contentType: "image/png", settings: { quality: "fast", language: "en", enhance: false }, oracle: "recognizes >=6 of the 8 known English fixture words", verify: (out) => { const text = out.toString("utf8"); expectRecognizedTerms(text, OCR_ENGLISH_TERMS, 6); }, }, { toolId: "ocr-pdf", fixture: `${FIXTURES}/document/valid/ocr-scanned.pdf`, contentType: "application/pdf", settings: { quality: "fast", language: "en", enhance: false }, oracle: "returns a PDF carrying text-drawing operators", verify: (out) => expectSearchablePdf(out), }, { toolId: "transcribe-audio", fixture: SPEECH_WAV, contentType: "audio/wav", settings: { language: "auto", outputFormat: "txt" }, expectedMime: "text/plain", oracle: ">=3 known fixture terms in the transcript", verify: (out) => expectKnownTranscript(out.toString("utf8")), }, { toolId: "auto-subtitles", fixture: SPEECH_MP4, contentType: "video/mp4", settings: { language: "auto", format: "srt" }, expectedMime: "application/x-subrip", oracle: "valid first SRT cue plus >=3 known fixture terms", verify: (out) => { const text = out.toString("utf8"); expectSrtArtifact(text); expectKnownTranscript(text); }, }, ]; /** Extra OCR probes that only make sense once the tool itself is reachable. */ const OCR_EXTRA_CASES: ToolCase[] = [ { toolId: "ocr", fixture: OCR_JA, contentType: "image/png", settings: { quality: "fast", language: "ja", enhance: false }, oracle: ">=4 Japanese codepoints in the transcript", verify: (out) => expectJapaneseScript(out.toString("utf8"), 4), }, ]; function record(row: Record): void { const line = `${JSON.stringify({ ts: new Date().toISOString(), profile: PROFILE, ...row })}\n`; if (OUT) appendFileSync(OUT, line); process.stdout.write(line); } async function login(): Promise { const r = await fetch(`${BASE}/api/auth/login`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ username: USERNAME, password: PASSWORD }), }); if (!r.ok) throw new Error(`login failed: HTTP ${r.status}`); const body = (await r.json()) as { token?: string }; if (!body.token) throw new Error("login returned no token"); return body.token; } async function readTerminal( token: string, jobId: string, ): Promise<{ downloadUrl: string; processedSize?: number }> { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), JOB_TIMEOUT_MS); try { const response = await fetch(`${BASE}/api/v1/jobs/${encodeURIComponent(jobId)}/progress`, { headers: { authorization: `Bearer ${token}` }, signal: controller.signal, }); if (response.status !== 200 || !response.body) { throw new Error(`job progress HTTP ${response.status}`); } const reader = response.body.getReader(); const decoder = new TextDecoder(); let pending = ""; while (true) { const { done, value } = await reader.read(); pending += decoder.decode(value, { stream: !done }); const lines = pending.split("\n"); pending = done ? "" : (lines.pop() ?? ""); for (const raw of lines) { const line = raw.trim(); if (!line.startsWith("data:")) continue; const frame = JSON.parse(line.slice(5).trim()) as ProgressFrame; if (frame.type === "heartbeat") continue; if (frame.phase === "failed") throw new Error(`job failed: ${frame.error ?? "unknown"}`); if (frame.phase === "complete") { await reader.cancel().catch(() => {}); const url = frame.result?.downloadUrl; if (typeof url !== "string") throw new Error("terminal frame omitted downloadUrl"); return { downloadUrl: url, processedSize: frame.result?.processedSize }; } } if (done) throw new Error("progress stream ended without a terminal frame"); } } finally { clearTimeout(timer); } } /** Shared token for the module-level `post` / `downloadArtifact` helpers. */ let sessionToken = ""; /** Bundles the live API reports as installed, for conditional legs. */ const installedBundleIds = new Set(); export type PostFn = ( path: string, file: Buffer, filename: string, contentType: string, settings: Record, extras?: { bytes: Buffer; contentType: string; field: string; filename: string }[], /** * erase-object's hand-written route reads DISCRETE multipart fields and * ignores a `settings` JSON blob entirely, so a caller following the usual * convention silently gets defaults. Send both when this is set. */ discreteFields?: boolean, ) => Promise<{ body: string; status: number }>; const post: PostFn = async ( path, file, filename, contentType, settings, extras, discreteFields, ) => { const form = new FormData(); form.append("file", new Blob([new Uint8Array(file)], { type: contentType }), filename); for (const extra of extras ?? []) { form.append( extra.field, new Blob([new Uint8Array(extra.bytes)], { type: extra.contentType }), extra.filename, ); } if (discreteFields) { for (const [key, value] of Object.entries(settings)) form.append(key, String(value)); } form.append("settings", JSON.stringify(settings)); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); try { const response = await fetch(`${BASE}${path}`, { method: "POST", headers: { authorization: `Bearer ${sessionToken}` }, body: form, signal: controller.signal, }); return { status: response.status, body: await response.text() }; } finally { clearTimeout(timer); } }; /** Resolve a 200 or 202 submission into the artifact bytes. */ async function downloadArtifact(accepted: AcceptedPayload, status: number): Promise { const { bytes } = await downloadArtifactWithMime(accepted, status); return bytes; } async function downloadArtifactWithMime( accepted: AcceptedPayload, status: number, ): Promise<{ bytes: Buffer; mime: string }> { let downloadUrl: string; if (status === 202) { if (typeof accepted.jobId !== "string") throw new Error("202 response without a jobId"); ({ downloadUrl } = await readTerminal(sessionToken, accepted.jobId)); } else { downloadUrl = accepted.downloadUrl; if (typeof downloadUrl !== "string") throw new Error("200 response without a downloadUrl"); } const response = await fetch(new URL(downloadUrl, BASE), { headers: { authorization: `Bearer ${sessionToken}` }, }); if (!response.ok) throw new Error(`artifact download HTTP ${response.status}`); const mime = (response.headers.get("content-type") ?? "").split(";", 1)[0]; const bytes = Buffer.from(await response.arrayBuffer()); if (bytes.length === 0) throw new Error("artifact is empty"); return { bytes, mime }; } async function runCase(token: string, spec: ToolCase): Promise { const started = Date.now(); const input = await readFile(spec.fixture); if (EXPECT === "gated") { // A gated tool must refuse before doing any work, and say why. The refusal // is asserted on the tool's OWN route, including the base route of tools // whose real work sits behind a sub-route. const { status, body } = await post( apiToolPath(spec.toolId), input, spec.fixture.split("/").pop() ?? "input", spec.contentType, spec.settings, ); const gated = (status === 402 || status === 403 || status === 409 || status === 428 || status === 501) && /not installed|FEATURE_NOT_INSTALLED|FEATURE_INCOMPATIBLE/i.test(body); record({ cell: "gate", toolId: spec.toolId, bundles: getRequiredBundlesForTool(spec.toolId), httpStatus: status, body: body.slice(0, 400), status: gated ? "pass" : "fail", durationS: Number(((Date.now() - started) / 1000).toFixed(1)), }); if (!gated) throw new Error(`${spec.toolId} was NOT gated: HTTP ${status} ${body.slice(0, 200)}`); return; } if (spec.customRun) { await spec.customRun({ input, post, token }); record({ cell: "tool", toolId: spec.toolId, bundles: getRequiredBundlesForTool(spec.toolId), inputBytes: input.length, oracle: spec.oracle, settings: spec.settings, status: "pass", durationS: Number(((Date.now() - started) / 1000).toFixed(1)), }); return; } const { status, body } = await post( apiToolPath(spec.toolId), input, spec.fixture.split("/").pop() ?? "input", spec.contentType, spec.settings, ); if (status !== 200 && status !== 202) { throw new Error(`${spec.toolId} submission HTTP ${status}: ${body.slice(0, 400)}`); } const { bytes: artifact, mime } = await downloadArtifactWithMime(JSON.parse(body), status); if (spec.expectedMime) { const allowed = Array.isArray(spec.expectedMime) ? spec.expectedMime : [spec.expectedMime]; if (!allowed.includes(mime)) { throw new Error(`${spec.toolId} MIME ${mime || "missing"} not in ${allowed.join("/")}`); } } await spec.verify(artifact, input); record({ cell: "tool", toolId: spec.toolId, bundles: getRequiredBundlesForTool(spec.toolId), httpStatus: status, mime, inputBytes: input.length, outputBytes: artifact.length, oracle: spec.oracle, settings: spec.settings, status: "pass", durationS: Number(((Date.now() - started) / 1000).toFixed(1)), }); } async function main(): Promise { const token = await login(); sessionToken = token; const featuresResponse = await fetch(`${BASE}/api/v1/features`, { headers: { authorization: `Bearer ${token}` }, }); const inventory = (await featuresResponse.json()) as { bundles: { id: string; status: string }[]; }; const installed = new Set( inventory.bundles.filter((b) => b.status === "installed").map((b) => b.id), ); for (const id of installed) installedBundleIds.add(id); const requested = process.env.QA_BUNDLES ? process.env.QA_BUNDLES.split(",") .map((s) => s.trim()) .filter(Boolean) : EXPECT === "gated" ? Object.keys(FEATURE_BUNDLES) : [...installed]; const unknown = requested.filter((id) => !FEATURE_BUNDLES[id]); if (unknown.length) throw new Error(`unknown bundle id(s): ${unknown.join(", ")}`); // The bundle -> tool mapping comes from the live shared export. const wanted = new Set(); for (const bundleId of requested) { for (const toolId of FEATURE_BUNDLES[bundleId].enablesTools) wanted.add(toolId); } const runnable: ToolCase[] = []; const skipped: { reason: string; toolId: string }[] = []; for (const spec of [...CASES, ...(EXPECT === "installed" ? OCR_EXTRA_CASES : [])]) { if (!wanted.has(spec.toolId)) continue; const required = getRequiredBundlesForTool(spec.toolId); if (EXPECT === "gated" && required.length === 0) { // OCR's Fast tier ships inside the image and the `ocr` bundle is only an // OPTIONAL pack (TOOL_OPTIONAL_BUNDLE_MAP), so ocr/ocr-pdf must stay // reachable with nothing installed. Refusal of the optional Balanced and // Best tiers is asserted separately. skipped.push({ toolId: spec.toolId, reason: "no required bundle: tool is never gated" }); continue; } if (EXPECT === "installed") { const missing = required.filter((id) => !installed.has(id)); if (missing.length) { skipped.push({ toolId: spec.toolId, reason: `requires ${missing.join(", ")}` }); continue; } } runnable.push(spec); } const missingCases = [...wanted].filter((t) => !CASES.some((c) => c.toolId === t)); if (missingCases.length) { throw new Error( `no oracle case defined for tool(s): ${missingCases.join(", ")}. Add one before claiming coverage.`, ); } record({ cell: "plan", expect: EXPECT, requestedBundles: requested, installedBundles: [...installed], toolCount: runnable.length, skipped, }); let failures = 0; for (const spec of runnable) { try { await runCase(token, spec); } catch (error) { failures += 1; record({ cell: EXPECT === "gated" ? "gate" : "tool", toolId: spec.toolId, bundles: getRequiredBundlesForTool(spec.toolId), status: "fail", error: String(error).slice(0, 900), settings: spec.settings, }); } } record({ cell: "summary", expect: EXPECT, ran: runnable.length, failures, skipped }); if (failures > 0) process.exitCode = 1; } await main();