// Mutation-killing unit tests for apps/api/src/lib/object-storage.ts. // // These target Stryker survivors by asserting EXACT constructed keys/paths, the // precise STORAGE_MODE branch chosen, the workspace-cap arithmetic and its // boundary, path-segment validation, the operational-error code allowlist, and // the chunk-byte accounting. A mutated string template, regex atom, path // separator, or comparison changes the produced key or the branch taken, so the // exact-value expectations below flip red under mutation. // // Companion files: object-storage-s3-mutation.test.ts covers the S3 dispatch // branches (it vi.mocks the config module with STORAGE_MODE="s3", which cannot // coexist with the real-env local tests here) and object-storage-statfs.test.ts // covers the statfs free-space floor (it needs a node:fs/promises mock). // // Env note: WORKSPACE_PATH / MAX_WORKSPACE_SIZE_GB are mutated inside beforeAll, // never in a describe body. Describe bodies all evaluate at collection time, so // body-level assignment would let the last-collected describe clobber every // other's workspace before any test runs. import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { PassThrough, Readable } from "node:stream"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { env } from "../../../apps/api/src/config.js"; import { CAPACITY_CRITICAL_GB, computeWorkspaceUsedBytes, copyReadableToFile, deleteObject, getObjectSize, isBelowCapacity, isOverWorkspaceCap, objectExists, putObject, putObjectStream, } from "../../../apps/api/src/lib/object-storage.js"; /** * Repoint the storage module at a fresh throwaway workspace with the aggregate * cap disabled, restoring the originals afterwards. Returns the temp root. Used * from beforeAll so describe-body evaluation never mutates shared env. */ function useThrowawayWorkspace(): { root: () => string } { const originalWorkspace = env.WORKSPACE_PATH; const originalMaxGb = env.MAX_WORKSPACE_SIZE_GB; let root = ""; beforeAll(() => { root = mkdtempSync(join(tmpdir(), "snapotter-objstore-mut-")); (env as { WORKSPACE_PATH: string }).WORKSPACE_PATH = root; (env as { MAX_WORKSPACE_SIZE_GB: number }).MAX_WORKSPACE_SIZE_GB = 0; }); afterAll(() => { (env as { WORKSPACE_PATH: string }).WORKSPACE_PATH = originalWorkspace; (env as { MAX_WORKSPACE_SIZE_GB: number }).MAX_WORKSPACE_SIZE_GB = originalMaxGb; if (root) rmSync(root, { recursive: true, force: true }); }); return { root: () => root }; } // ── VALID_KEY regex (assertValidKey, source L25/L27-28) ────────────── // Every public key-taking call funnels through assertValidKey. Kill regex-atom // mutations by pinning the exact accept/reject verdict at each boundary the // pattern encodes: the two allowed prefixes, the first-char class, the interior // char class, a non-empty filename, and the null-byte / traversal rejections. // assertValidKey is not exported, so probe it through putObject: a valid key // reaches the write (resolves), an invalid one throws "Invalid object key". describe("object key validation (VALID_KEY / assertValidKey)", () => { useThrowawayWorkspace(); const accepted: Array<[string, string]> = [ ["uploads prefix", "uploads/job1/file.bin"], ["outputs prefix", "outputs/job1/file.bin"], ["digit-leading jobId", "uploads/1abc/file.bin"], ["jobId with dot/underscore/dash interior", "uploads/a.b_c-d/file.bin"], ["single-char jobId", "uploads/a/file.bin"], ["filename with spaces and dots", "uploads/job1/my file.final.bin"], ]; it.each(accepted)("accepts a valid key: %s", async (_label, key) => { await expect(putObject(key, Buffer.from("x"))).resolves.toBeUndefined(); }); const rejected: Array<[string, string]> = [ ["wrong prefix (uploads-like but not exact)", "upload/job1/file.bin"], ["wrong prefix entirely", "secrets/job1/file.bin"], ["prefix must be at the very start (anchor ^)", "x/uploads/job1/file.bin"], ["jobId first char cannot be a dot", "uploads/.hidden/file.bin"], ["jobId first char cannot be an underscore", "uploads/_job/file.bin"], ["jobId first char cannot be a dash", "uploads/-job/file.bin"], ["empty filename after the slash is rejected", "uploads/job1/"], ["missing filename segment entirely", "uploads/job1"], ["a slash inside the filename is rejected (extra segment)", "uploads/job1/sub/deep.bin"], ["parent traversal in filename passes regex but the .. guard rejects", "uploads/job1/a..b"], ["classic dot-dot traversal", "outputs/../../etc/passwd"], ]; it.each(rejected)("rejects an invalid key: %s", async (_label, key) => { await expect(putObject(key, Buffer.from("x"))).rejects.toThrow(/Invalid object key/); }); it("rejects a null byte embedded in the filename", async () => { await expect(putObject("uploads/job1/ab", Buffer.from("x"))).rejects.toThrow( /Invalid object key/, ); }); // Isolates the `|| key.includes("..")` disjunct (source L28): this key matches // VALID_KEY yet must still be rejected purely because of the "..". it("rejects a regex-valid key solely on the traversal guard", async () => { const key = "uploads/job1/a..b"; expect(/^(uploads|outputs)\/[A-Za-z0-9][A-Za-z0-9._-]*\/[^/\0]+$/.test(key)).toBe(true); await expect(putObject(key, Buffer.from("x"))).rejects.toThrow(/Invalid object key/); }); }); // ── normalizeOperationalWriteError code allowlist (source L272-273) ── // A streaming write that fails with a transient-storage errno must be re-tagged // statusCode 503; any other error passes through untouched. Each listed code is // asserted independently so a mutation dropping one array element survives no // longer, and two negative cases pin the guard's shape. describe("operational write-error classification", () => { useThrowawayWorkspace(); function streamThatFailsWith(code: string): Readable { return Readable.from( (async function* () { yield Buffer.from("partial"); throw Object.assign(new Error(`fault:${code}`), { code }); })(), ); } const transient = ["EACCES", "EDQUOT", "ENOSPC", "EROFS"]; it.each(transient)("re-tags a %s streaming-write failure as statusCode 503", async (code) => { const key = `outputs/oper-${code}-${process.pid}/x.bin`; await expect(putObjectStream(key, streamThatFailsWith(code))).rejects.toMatchObject({ code, statusCode: 503, }); }); it("leaves an unlisted errno untagged (no statusCode injected)", async () => { const key = `outputs/oper-eperm-${process.pid}/x.bin`; const rejection = await putObjectStream(key, streamThatFailsWith("EPERM")).then( () => null, (e) => e, ); expect(rejection).toBeInstanceOf(Error); expect((rejection as { code?: string }).code).toBe("EPERM"); expect((rejection as { statusCode?: number }).statusCode).toBeUndefined(); }); it("leaves an error with no errno code untagged", async () => { const key = `outputs/oper-plain-${process.pid}/x.bin`; const source = Readable.from( (async function* () { yield Buffer.from("partial"); throw new Error("plain failure with no code"); })(), ); const rejection = await putObjectStream(key, source).then( () => null, (e) => e, ); expect(rejection).toBeInstanceOf(Error); expect((rejection as { statusCode?: number }).statusCode).toBeUndefined(); expect((rejection as { message?: string }).message).toBe("plain failure with no code"); }); }); // ── copyReadableToFile chunk byte accounting (source L316-321) ─────── // The counter must credit string chunks by their UTF-8 byte length and binary // chunks by byteLength. Pin the exact returned total for each chunk shape so the // ternary and the += cannot be mutated silently. describe("copyReadableToFile byte accounting", () => { let scratch = ""; afterEach(() => { if (scratch) rmSync(scratch, { recursive: true, force: true }); scratch = ""; }); it("counts a multibyte string chunk by its UTF-8 byte length, not char count", async () => { scratch = mkdtempSync(join(tmpdir(), "snapotter-copy-str-")); const destination = join(scratch, "utf8.txt"); // "é" is 1 char but 2 UTF-8 bytes; "abc" is 3 bytes => 5 total, not 4. const written = await copyReadableToFile(Readable.from(["é", "abc"]), destination, { maxBytes: 100, }); expect(written).toBe(5); expect(readFileSync(destination).toString("utf8")).toBe("éabc"); expect(readFileSync(destination).byteLength).toBe(5); }); it("counts binary (Uint8Array) chunks by byteLength", async () => { scratch = mkdtempSync(join(tmpdir(), "snapotter-copy-bin-")); const destination = join(scratch, "bin.dat"); const written = await copyReadableToFile( Readable.from([new Uint8Array([1, 2, 3]), Buffer.from([4, 5])]), destination, { maxBytes: 100 }, ); expect(written).toBe(5); expect(readFileSync(destination)).toEqual(Buffer.from([1, 2, 3, 4, 5])); }); it("trips the cap the moment the running total exceeds maxBytes on a string stream", async () => { scratch = mkdtempSync(join(tmpdir(), "snapotter-copy-strcap-")); const destination = join(scratch, "over.txt"); // "ééé" = 6 bytes; cap 5 => 413 and no destination left behind. await expect( copyReadableToFile(Readable.from(["ééé"]), destination, { maxBytes: 5 }), ).rejects.toMatchObject({ statusCode: 413 }); expect(existsSync(destination)).toBe(false); }); it("allows a stream whose total exactly equals maxBytes (boundary is strict >)", async () => { scratch = mkdtempSync(join(tmpdir(), "snapotter-copy-eq-")); const destination = join(scratch, "exact.dat"); const written = await copyReadableToFile(Readable.from([Buffer.alloc(8, 7)]), destination, { maxBytes: 8, }); expect(written).toBe(8); expect(readFileSync(destination).byteLength).toBe(8); }); }); // ── pure capacity thresholds (source L78-79, L123-124) ─────────────── // Nail the exact GiB divisor and the strict comparison at each boundary so the // arithmetic mutants (1024 -> other base, < -> <=, > -> >=) die. describe("capacity threshold arithmetic", () => { it("exposes the documented 0.5 GB critical floor", () => { expect(CAPACITY_CRITICAL_GB).toBe(0.5); }); it("isBelowCapacity uses a GiB divisor and a strict < at the boundary", () => { const halfGiB = 0.5 * 1024 ** 3; expect(isBelowCapacity(halfGiB - 1)).toBe(true); // just under 0.5 GiB expect(isBelowCapacity(halfGiB)).toBe(false); // exactly 0.5 GiB is NOT below expect(isBelowCapacity(halfGiB + 1)).toBe(false); expect(isBelowCapacity(0)).toBe(true); }); it("isOverWorkspaceCap disables on maxGb<=0 and uses a strict > at the boundary", () => { const tenGiB = 10 * 1024 ** 3; expect(isOverWorkspaceCap(tenGiB + 1, 10)).toBe(true); // just over expect(isOverWorkspaceCap(tenGiB, 10)).toBe(false); // exactly at cap is NOT over expect(isOverWorkspaceCap(tenGiB - 1, 10)).toBe(false); expect(isOverWorkspaceCap(Number.MAX_SAFE_INTEGER, 0)).toBe(false); // disabled expect(isOverWorkspaceCap(Number.MAX_SAFE_INTEGER, -1)).toBe(false); // disabled }); }); // ── computeWorkspaceUsedBytes: only regular files count (source L113-115) ─ // A non-file entry (a directory) sitting where a file might be must contribute // 0. Pins the `if (s?.isFile())` guard so mutating it to always-true (which // would add directory sizes or NaN) fails. describe("computeWorkspaceUsedBytes file-only accounting", () => { let root = ""; afterEach(() => { if (root) rmSync(root, { recursive: true, force: true }); root = ""; }); it("counts only regular files and ignores a nested directory entry", async () => { root = mkdtempSync(join(tmpdir(), "snapotter-used-fileonly-")); await mkdir(join(root, "uploads", "job1"), { recursive: true }); await writeFile(join(root, "uploads", "job1", "real.bin"), Buffer.alloc(1234)); // A directory entry inside the job dir must not be added to the total. await mkdir(join(root, "uploads", "job1", "subdir"), { recursive: true }); expect(await computeWorkspaceUsedBytes(root)).toBe(1234); }); it("sums across both prefixes and returns the exact byte total", async () => { root = mkdtempSync(join(tmpdir(), "snapotter-used-sum-")); await mkdir(join(root, "uploads", "j1"), { recursive: true }); await mkdir(join(root, "outputs", "j2"), { recursive: true }); await writeFile(join(root, "uploads", "j1", "a.bin"), Buffer.alloc(100)); await writeFile(join(root, "outputs", "j2", "b.bin"), Buffer.alloc(23)); expect(await computeWorkspaceUsedBytes(root)).toBe(123); }); }); // ── putObjectStream abort-listener wiring (source L189-196, L232) ───── // A signal aborted with an Error reason must destroy the source with THAT error // (reason instanceof Error branch, L191-192). A signal aborted with a non-Error // reason must fall back to a synthesized AbortError (L193). Covering both pins // the reason-construction ternary that is otherwise no-cov. The source carries // a no-op error handler so the deliberate destroy() never surfaces as uncaught. describe("putObjectStream abort-reason propagation", () => { useThrowawayWorkspace(); // The rejection identity is the assertion here: it proves which branch of the // reason-construction ternary ran. The post-abort file cleanup is covered // deterministically by the S3 copy-stream suite, so it is not re-asserted here // (the local mid-stream unlink races the write under concurrent load). it("rejects with a synthesized AbortError when aborted mid-stream with no reason", async () => { const controller = new AbortController(); const source = new PassThrough(); source.on("error", () => {}); const key = `uploads/abort-noreason-${process.pid}/x.bin`; const uploading = putObjectStream(key, source, { signal: controller.signal }); source.write(Buffer.from("partial")); await new Promise((r) => setImmediate(r)); controller.abort(); // default reason is a DOMException, not an Error await expect(uploading).rejects.toMatchObject({ name: "AbortError" }); }); it("destroys the source with the caller's Error reason when aborted mid-stream", async () => { const controller = new AbortController(); const source = new PassThrough(); source.on("error", () => {}); const destroySpy = vi.spyOn(source, "destroy"); const key = `uploads/abort-reason-${process.pid}/x.bin`; const boom = new Error("caller cancelled the upload"); const uploading = putObjectStream(key, source, { signal: controller.signal }); source.write(Buffer.from("partial")); await new Promise((r) => setImmediate(r)); controller.abort(boom); // The abort handler synchronously destroys the source with the caller's Error // (the `instanceof Error` branch of the reason ternary). The propagated // rejection reason races the write path in CI, so assert the destroy reason // directly, which is deterministic. await expect(uploading).rejects.toThrow(); expect(destroySpy).toHaveBeenCalledWith(boom); }); it("rejects immediately when the signal is already aborted before any I/O", async () => { const controller = new AbortController(); controller.abort(); const key = `uploads/abort-pre-${process.pid}/x.bin`; await expect( putObjectStream(key, Readable.from([Buffer.from("x")]), { signal: controller.signal }), ).rejects.toMatchObject({ name: "AbortError" }); await expect(objectExists(key)).resolves.toBe(false); }); }); // ── local getObjectSize / deleteObject exact-path behavior ─────────── // Confirms the local backend stats/unlinks the key that VALID_KEY built. A // mutated join separator would stat the wrong path and change the observed size. describe("local getObjectSize and deleteObject", () => { const ws = useThrowawayWorkspace(); it("reports the exact stored byte size and deletes at the constructed path", async () => { const root = ws.root(); const key = `outputs/sizejob-${process.pid}/payload.bin`; await putObject(key, Buffer.alloc(4097, 9)); // The file must land exactly at /outputs//payload.bin. const onDisk = join(root, "outputs", `sizejob-${process.pid}`, "payload.bin"); expect(existsSync(onDisk)).toBe(true); expect(await getObjectSize(key)).toBe(4097); await deleteObject(key); expect(await objectExists(key)).toBe(false); expect(existsSync(onDisk)).toBe(false); }); });