mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: release QA hardening across processing, media, security, and CI gates (#649)
A release-readiness QA pass over the whole product. The commits split into defects a user would hit and gates that were reporting green while measuring nothing. ## Fixes that change behaviour Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so request.ip came from a client-set header and a forged X-Forwarded-For got past the login limiter. The default is now a private-network trust list. A transient Postgres outage stranded in-flight jobs, leaving finished output on disk with no row pointing at it. A reconciler now resolves those rows and adopts the bytes rather than dropping the work. A Redis connection that moved to a new address wedged every read-blocked consumer, so completions stopped signalling while health still answered 200. Socket timeouts plus subscriber pings recover it. Installing more than one AI bundle left the shared venv multi-versioned and silently broke three tools. The installer now reconciles distributions to one version each. Converting an image to JXL at quality 1 through 4 returned a 500, because libjxl 0.7 rejects the distance those values compute. The quality is floored at what the encoder honours. A missing ffmpeg was also reported to the user as a corrupt upload; it now says the engine is unavailable. RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at 0.22.2, and the release scan was split so it can fail on an unfixed critical instead of hiding it behind ignore-unfixed. ## Gates that could not fail Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs build; coverage discarded its whole report on any failing test; the lint gate skipped root tests, scripts, and two workspaces; and several generated matrices counted a host missing ffmpeg as a passing tool. Each now measures what it claims. Full evidence and the outstanding release items are tracked locally and are not part of this branch.
This commit is contained in:
SnapOtter
@@ -6,7 +6,6 @@
|
||||
* is disabled (the default).
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { db, schema } from "../../../apps/api/src/db/index.js";
|
||||
import { fixtures, readFixture } from "../../fixtures/index.js";
|
||||
import {
|
||||
buildTestApp,
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
* correctly without data corruption, crashes, or race conditions.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { apiToolPath } from "@snapotter/shared";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { fixtures, readFixture } from "../../fixtures/index.js";
|
||||
|
||||
@@ -83,6 +83,22 @@ describe.skipIf(!pdfcpuAvailable())("doc-engine pdfcpu (requires pdfcpu binary)"
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["%", "100%", "%x"])("textStamp safely preserves literal text %j", async (text) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "pdfcpu-"));
|
||||
try {
|
||||
const out = join(dir, "literal-percent.pdf");
|
||||
await pdfcpuTextStamp(
|
||||
PDF,
|
||||
{ text, position: "c", fontSize: 10, opacity: 1, rotation: 0 },
|
||||
out,
|
||||
);
|
||||
const bytes = await readFile(out);
|
||||
expect(bytes.subarray(0, 5).toString()).toBe("%PDF-");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// --- Ungated validation tests: run without the binary ---
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { apiToolPath, TOOLS } from "@snapotter/shared";
|
||||
import { apiToolPath, TOOLS, toolSection } from "@snapotter/shared";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { buildTestApp, type TestApp } from "../test-server";
|
||||
|
||||
@@ -120,11 +120,27 @@ describe("API docs", () => {
|
||||
const deployment = readFileSync(join(root, "apps/docs/guide/deployment.md"), "utf8");
|
||||
const architecture = readFileSync(join(root, "apps/docs/guide/architecture.md"), "utf8");
|
||||
|
||||
expect(gettingStarted).toContain("| **Image** | 107 |");
|
||||
expect(gettingStarted).toContain("| **Video** | 57 |");
|
||||
expect(gettingStarted).toContain("| **Audio** | 27 |");
|
||||
expect(gettingStarted).toContain("| **PDF / Document** | 42 |");
|
||||
expect(gettingStarted).toContain("| **Files** | 10 |");
|
||||
// The table lists what a reader sees in the UI, so it counts by section,
|
||||
// not by modality. Those agree for image, video and audio and diverge for
|
||||
// the rest: the document modality splits across the PDF and Files sections
|
||||
// by whether a tool accepts .pdf. Hard-coding both taxonomies is what let
|
||||
// this guard drift out of step with the page it guards, so derive them.
|
||||
const perSection = new Map<string, number>();
|
||||
for (const tool of TOOLS) {
|
||||
const section = toolSection(tool);
|
||||
perSection.set(section, (perSection.get(section) ?? 0) + 1);
|
||||
}
|
||||
const rows: Array<[string, string]> = [
|
||||
["Image", "image"],
|
||||
["Video", "video"],
|
||||
["Audio", "audio"],
|
||||
["PDF / Document", "pdf"],
|
||||
["Files", "files"],
|
||||
];
|
||||
for (const [label, section] of rows) {
|
||||
expect(gettingStarted).toContain(`| **${label}** | ${perSection.get(section)} |`);
|
||||
}
|
||||
expect([...perSection.values()].reduce((sum, count) => sum + count, 0)).toBe(TOOLS.length);
|
||||
expect(deployment).not.toContain("All 138 non-AI tools");
|
||||
expect(architecture).toContain("243 tool routes");
|
||||
});
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
* the factory (route registration, file collection, per-file prepare,
|
||||
* inputRefs on the durable row, and the too-many-files rejection).
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { apiToolPath, TOOLS } from "@snapotter/shared";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
@@ -150,9 +148,10 @@ describe("Factory multi-input (maxInputs)", () => {
|
||||
// Verify the durable DB row has 3 inputRefs
|
||||
const [row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, result.jobId));
|
||||
expect(row).toBeDefined();
|
||||
expect(row?.status).toBe("completed");
|
||||
expect(row?.inputRefs).toBeDefined();
|
||||
expect((row?.inputRefs as string[]).length).toBe(3);
|
||||
if (!row) throw new Error("job row not found after polling");
|
||||
expect(row.status).toBe("completed");
|
||||
expect(row.inputRefs).toBeDefined();
|
||||
expect((row.inputRefs as string[]).length).toBe(3);
|
||||
}, 30_000);
|
||||
|
||||
it("rejects 2 files on a tool without maxInputs (default 1)", async () => {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { mkdtempSync, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { ffmpegAvailable } from "@snapotter/media-engine";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { fixtureRoot } from "../../fixtures/index.js";
|
||||
|
||||
/**
|
||||
* A container missing ffmpeg is an operator problem, not a bad upload.
|
||||
*
|
||||
* media-input caught every probe failure and reported "may be corrupt or in an
|
||||
* unsupported format", so an operator whose image lacks ffprobe was told their
|
||||
* perfectly good MP4 was broken, with nothing pointing at the real cause. The
|
||||
* CI integration shards ship without ffmpeg by design, which is where this
|
||||
* surfaced: every media tool failed there for a reason no caller could see.
|
||||
*
|
||||
* These run without the ffmpegAvailable() gate the sibling media specs use,
|
||||
* because the whole point is the behaviour when the binary is gone.
|
||||
*/
|
||||
const MP4 = readFileSync(join(fixtureRoot, "video", "formats", "tiny.mp4"));
|
||||
|
||||
function scratch(): { scratchDir: string } {
|
||||
return { scratchDir: mkdtempSync(join(tmpdir(), "media-unavailable-")) };
|
||||
}
|
||||
|
||||
const originalFfprobe = process.env.FFPROBE_PATH;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalFfprobe === undefined) delete process.env.FFPROBE_PATH;
|
||||
else process.env.FFPROBE_PATH = originalFfprobe;
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
/**
|
||||
* media-engine resolves the binary once and caches it in a module variable, so
|
||||
* changing FFPROBE_PATH after any earlier call has no effect. Re-import through
|
||||
* a fresh module graph so each case actually gets the environment it sets.
|
||||
*/
|
||||
async function freshHandler(kind: "video") {
|
||||
vi.resetModules();
|
||||
const { MediaInputHandler } = await import("../../../apps/api/src/modality/media-input.js");
|
||||
return new MediaInputHandler(kind);
|
||||
}
|
||||
|
||||
describe("media validation when the engine is unavailable", () => {
|
||||
it("reports the missing engine instead of blaming the upload", async () => {
|
||||
process.env.FFPROBE_PATH = join(tmpdir(), "definitely-not-ffprobe");
|
||||
const handler = await freshHandler("video");
|
||||
|
||||
const error = await handler.prepare(MP4, "clip.mp4", scratch()).then(
|
||||
() => null,
|
||||
(caught: unknown) => caught,
|
||||
);
|
||||
|
||||
// Shape, not instanceof: vi.resetModules() gives the handler a different
|
||||
// copy of the error class than a static import here would hold.
|
||||
const failure = error as { name?: string; message: string; statusCode?: number };
|
||||
expect(failure.name, "a missing engine must still reject the request").toBe(
|
||||
"InputValidationError",
|
||||
);
|
||||
|
||||
// The upload is fine. Saying otherwise sends the operator looking at their
|
||||
// file instead of their container.
|
||||
expect(failure.message).not.toMatch(/corrupt/i);
|
||||
expect(failure.message).toMatch(/ffprobe|ffmpeg|unavailable|not installed/i);
|
||||
|
||||
// 400 tells the caller to fix their request, which cannot help here. This
|
||||
// is the server missing a dependency, so it belongs in the 5xx range.
|
||||
expect(failure.statusCode).toBeGreaterThanOrEqual(500);
|
||||
});
|
||||
|
||||
// Needs a working ffprobe to reach the parse failure at all. CI shards ship
|
||||
// without ffmpeg, where every media input is an engine gap instead.
|
||||
it.skipIf(!ffmpegAvailable())(
|
||||
"still rejects a genuinely corrupt upload as a client error",
|
||||
async () => {
|
||||
delete process.env.FFPROBE_PATH;
|
||||
const handler = await freshHandler("video");
|
||||
const notMedia = Buffer.from("this is not a video file, it is prose");
|
||||
|
||||
const error = await handler.prepare(notMedia, "clip.mp4", scratch()).then(
|
||||
() => null,
|
||||
(caught: unknown) => caught,
|
||||
);
|
||||
|
||||
const failure = error as { name?: string; message: string; statusCode?: number };
|
||||
expect(failure.name).toBe("InputValidationError");
|
||||
expect(failure.statusCode).toBe(400);
|
||||
expect(failure.message).toMatch(/corrupt|unsupported/i);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { qpdfAvailable } from "@snapotter/doc-engine";
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { resolveGs } from "@snapotter/doc-engine";
|
||||
import { ffmpegAvailable } from "@snapotter/media-engine";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* A terminal job row must survive a late progress frame.
|
||||
*
|
||||
* Tool progress is published fire and forget (`void updateSingleFileProgress`
|
||||
* in the worker), so a nonterminal frame can still be in flight when the job
|
||||
* reaches a terminal state. persistSingleFileProgress wrote
|
||||
* `status: "processing", completedAt: null, error: null` unconditionally, so a
|
||||
* frame that landed late resurrected a finished row.
|
||||
*
|
||||
* Observed on the release container: cancelling a stabilize-video job just
|
||||
* after submit left the row permanently `status = processing`,
|
||||
* `completed_at = null`, `error = null`, `duration_ms = 59`, with
|
||||
* `progress = {"stage":"Analyzing","percent":5}` -- the tool's first frame,
|
||||
* written after the worker's cancel row. One in three attempts reproduced it,
|
||||
* and the row never recovered because nothing else revisits it.
|
||||
*
|
||||
* The completion path is already safe: it goes through
|
||||
* updateSingleFileProgressAtomically, whose per-job queue drains earlier
|
||||
* nonterminal writes first. The cancel and failure paths in the worker use a
|
||||
* plain db.update outside that queue, so the guard has to live in the write
|
||||
* itself.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { db, schema } from "../../../apps/api/src/db/index.js";
|
||||
import { updateSingleFileProgress } from "../../../apps/api/src/routes/progress.js";
|
||||
import { buildTestApp, type TestApp } from "../test-server.js";
|
||||
|
||||
let testApp: TestApp;
|
||||
|
||||
beforeAll(async () => {
|
||||
testApp = await buildTestApp();
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await testApp.cleanup();
|
||||
}, 10_000);
|
||||
|
||||
async function seedJob(
|
||||
status: "canceled" | "failed" | "completed" | "processing",
|
||||
): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db.insert(schema.jobs).values({
|
||||
id,
|
||||
type: "single",
|
||||
status,
|
||||
inputRefs: [],
|
||||
completedAt: status === "processing" ? null : new Date(),
|
||||
durationMs: 59,
|
||||
error: status === "failed" ? { message: "boom" } : null,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
async function readJob(id: string) {
|
||||
const [row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, id));
|
||||
return row;
|
||||
}
|
||||
|
||||
describe("late progress frames against a terminal job row", () => {
|
||||
for (const terminal of ["canceled", "failed", "completed"] as const) {
|
||||
it(`leaves a ${terminal} row terminal`, async () => {
|
||||
const id = await seedJob(terminal);
|
||||
|
||||
// The frame the worker had already queued before the job ended.
|
||||
await updateSingleFileProgress({
|
||||
jobId: id,
|
||||
phase: "processing",
|
||||
percent: 5,
|
||||
stage: "Analyzing",
|
||||
});
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status, `${terminal} row was rewritten to ${row.status}`).toBe(terminal);
|
||||
expect(row.completedAt, "completedAt was cleared by a late progress frame").not.toBeNull();
|
||||
if (terminal === "failed") {
|
||||
expect(row.error, "the failure reason was cleared by a late progress frame").not.toBeNull();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it("still advances a job that is genuinely still running", async () => {
|
||||
const id = await seedJob("processing");
|
||||
|
||||
await updateSingleFileProgress({
|
||||
jobId: id,
|
||||
phase: "processing",
|
||||
percent: 42,
|
||||
stage: "Working",
|
||||
});
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status).toBe("processing");
|
||||
expect((row.progress as { percent?: number } | null)?.percent).toBe(42);
|
||||
});
|
||||
|
||||
it("still records a terminal frame on a running job", async () => {
|
||||
const id = await seedJob("processing");
|
||||
|
||||
await updateSingleFileProgress({ jobId: id, phase: "complete", percent: 100 });
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status).toBe("completed");
|
||||
expect(row.completedAt).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* A Redis connection can stop working without either end noticing.
|
||||
*
|
||||
* When the server comes back at a different address, a connection parked on a
|
||||
* blocking read has nothing left to send, so it never draws a reset and ioredis
|
||||
* never sees a disconnect. Connections that keep issuing commands heal
|
||||
* themselves; the consumers, the BullMQ QueueEvents streams and the pub/sub
|
||||
* subscribers, sat on dead sockets until the process was restarted. Jobs still
|
||||
* ran and still wrote correct output, so nothing else noticed: only the
|
||||
* completion signal was gone, which cost every synchronous request the full
|
||||
* sync-wait window and left attached progress streams on heartbeats forever
|
||||
* (PERF-20260726-007).
|
||||
*
|
||||
* The proxy below reproduces that shape faithfully. Freezing forwards nothing
|
||||
* in either direction and closes nothing, so both ends keep believing the
|
||||
* socket is fine, exactly like packets to an address whose owner has left. New
|
||||
* connections still work, which is what makes the failure so quiet.
|
||||
*
|
||||
* Recovery costs about one socket timeout, so this case is deliberately slow.
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createServer, type Server, type Socket, connect as tcpConnect } from "node:net";
|
||||
import { QueueEvents } from "bullmq";
|
||||
import type { Redis } from "ioredis";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { env } from "../../../apps/api/src/config.js";
|
||||
import {
|
||||
createBullMQConnection,
|
||||
createRedisConnection,
|
||||
createRedisSubscriberConnection,
|
||||
} from "../../../apps/api/src/jobs/connection.js";
|
||||
import { bullPrefix, queueName } from "../../../apps/api/src/jobs/types.js";
|
||||
|
||||
/** The 30 s socket timeout plus the 10 s probe interval, plus a loaded box. */
|
||||
const RECOVERY_BUDGET_MS = 75_000;
|
||||
const CASE_TIMEOUT_MS = 150_000;
|
||||
|
||||
interface RelayPair {
|
||||
client: Socket;
|
||||
upstream: Socket;
|
||||
frozen: boolean;
|
||||
}
|
||||
|
||||
/** A TCP relay that can stop forwarding without letting either end find out. */
|
||||
class BlackholeProxy {
|
||||
private readonly pairs: RelayPair[] = [];
|
||||
private server: Server | null = null;
|
||||
|
||||
async listen(upstream: URL): Promise<number> {
|
||||
const host = upstream.hostname;
|
||||
const port = Number(upstream.port || "6379");
|
||||
this.server = createServer((client) => {
|
||||
const forward = tcpConnect(port, host);
|
||||
const pair: RelayPair = { client, upstream: forward, frozen: false };
|
||||
this.pairs.push(pair);
|
||||
client.pipe(forward);
|
||||
forward.pipe(client);
|
||||
// A frozen pair must survive one end giving up on it: that is the whole
|
||||
// point of the fault, and tearing the other half down would be a signal.
|
||||
const drop = () => {
|
||||
if (pair.frozen) return;
|
||||
client.destroy();
|
||||
forward.destroy();
|
||||
};
|
||||
for (const socket of [client, forward]) {
|
||||
socket.on("error", drop);
|
||||
socket.on("close", drop);
|
||||
}
|
||||
});
|
||||
await new Promise<void>((resolve) => this.server?.listen(0, "127.0.0.1", resolve));
|
||||
const address = this.server?.address();
|
||||
if (!address || typeof address === "string") throw new Error("proxy did not bind a port");
|
||||
return address.port;
|
||||
}
|
||||
|
||||
/** Stop relaying on every socket open right now. Later ones are unaffected. */
|
||||
freeze(): void {
|
||||
for (const pair of this.pairs) {
|
||||
if (pair.frozen) continue;
|
||||
pair.frozen = true;
|
||||
pair.client.unpipe(pair.upstream);
|
||||
pair.upstream.unpipe(pair.client);
|
||||
pair.client.pause();
|
||||
pair.upstream.pause();
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
for (const pair of this.pairs) {
|
||||
pair.client.destroy();
|
||||
pair.upstream.destroy();
|
||||
}
|
||||
this.pairs.length = 0;
|
||||
const server = this.server;
|
||||
this.server = null;
|
||||
if (server) await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
const proxy = new BlackholeProxy();
|
||||
const originalRedisUrl = env.REDIS_URL;
|
||||
const channel = `${bullPrefix()}:consumer-recovery-test`;
|
||||
const eventsKey = `bull:${queueName("image")}:events`;
|
||||
const probeEvent = "consumer-recovery-probe";
|
||||
|
||||
let subscriber: Redis | null = null;
|
||||
let queueEvents: QueueEvents | null = null;
|
||||
let publisher: Redis | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
const port = await proxy.listen(new URL(originalRedisUrl));
|
||||
// Everything built while REDIS_URL points at the proxy dials through it; the
|
||||
// publisher is built afterwards so the far side of the break stays reachable.
|
||||
const proxied = new URL(originalRedisUrl);
|
||||
proxied.hostname = "127.0.0.1";
|
||||
proxied.port = String(port);
|
||||
env.REDIS_URL = proxied.toString().replace(/\/$/, "");
|
||||
|
||||
subscriber = createRedisSubscriberConnection();
|
||||
await subscriber.subscribe(channel);
|
||||
queueEvents = new QueueEvents(queueName("image"), { connection: createBullMQConnection() });
|
||||
// BullMQ turns a consumer-loop failure into an 'error' event; the loop retries
|
||||
// on its own, and an unlistened one only clutters the run.
|
||||
queueEvents.on("error", () => {});
|
||||
await queueEvents.waitUntilReady();
|
||||
|
||||
env.REDIS_URL = originalRedisUrl;
|
||||
publisher = createRedisConnection();
|
||||
await publisher.ping();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
env.REDIS_URL = originalRedisUrl;
|
||||
await queueEvents?.close().catch(() => {});
|
||||
subscriber?.disconnect();
|
||||
publisher?.disconnect();
|
||||
await proxy.close();
|
||||
});
|
||||
|
||||
describe("Redis consumers whose socket wedges", () => {
|
||||
it(
|
||||
"still deliver queue events and pub/sub frames after the sockets go silent",
|
||||
async () => {
|
||||
const frames: string[] = [];
|
||||
const events: string[] = [];
|
||||
subscriber?.on("message", (_channel, message) => frames.push(message));
|
||||
(
|
||||
queueEvents as unknown as {
|
||||
on(event: string, listener: (args: { token?: string }) => void): void;
|
||||
}
|
||||
).on(probeEvent, (args) => events.push(args.token ?? ""));
|
||||
|
||||
// Baseline: both paths work through the proxy while it is relaying. The
|
||||
// stream event is not only a warm-up. A consumer that has never read one
|
||||
// is still parked on the literal "$", which the server resolves at
|
||||
// command time, so a resend after a reconnect would resume at the new
|
||||
// tail. Reading one first gives it a concrete position to resume from,
|
||||
// which is the state every pool is in once it has run a job.
|
||||
const baselineFrame = randomUUID();
|
||||
const baselineEvent = randomUUID();
|
||||
await publisher?.publish(channel, baselineFrame);
|
||||
await publisher?.xadd(eventsKey, "*", "event", probeEvent, "token", baselineEvent);
|
||||
await expect.poll(() => frames, { timeout: 10_000 }).toContain(baselineFrame);
|
||||
await expect.poll(() => events, { timeout: 10_000 }).toContain(baselineEvent);
|
||||
|
||||
// The break. Nothing is closed and no reset is sent, so ioredis is told
|
||||
// nothing at all: without an inactivity timeout both consumers would wait
|
||||
// here for the life of the process.
|
||||
proxy.freeze();
|
||||
|
||||
const eventToken = randomUUID();
|
||||
const frameToken = randomUUID();
|
||||
// The stream entry is written once. A consumer that reconnects resumes
|
||||
// from its last id, so the event has to survive the outage rather than
|
||||
// merely arrive after it. Pub/sub has no such memory, so that side is
|
||||
// republished until the subscriber is listening again.
|
||||
await publisher?.xadd(eventsKey, "*", "event", probeEvent, "token", eventToken);
|
||||
const republish = setInterval(() => {
|
||||
void publisher?.publish(channel, frameToken).catch(() => {});
|
||||
}, 1000);
|
||||
|
||||
try {
|
||||
await expect
|
||||
.poll(() => events, { timeout: RECOVERY_BUDGET_MS, interval: 500 })
|
||||
.toContain(eventToken);
|
||||
// Receiving on the original channel also proves ioredis restored the
|
||||
// subscription rather than just the socket.
|
||||
await expect
|
||||
.poll(() => frames, { timeout: RECOVERY_BUDGET_MS, interval: 500 })
|
||||
.toContain(frameToken);
|
||||
} finally {
|
||||
clearInterval(republish);
|
||||
}
|
||||
},
|
||||
CASE_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,225 @@
|
||||
/**
|
||||
* A job row must never stay non-terminal once nothing will run it again, and
|
||||
* output bytes already on disk must never become unreachable because the row
|
||||
* was not updated.
|
||||
*
|
||||
* Measured on the release container (finding PERF-20260726-006): stopping
|
||||
* Postgres for 20 s with six jobs in flight left four rows stranded forever,
|
||||
* three `queued` and one `processing`, with attempts 0 or 1, error null and
|
||||
* output_refs null, while every BullMQ pool was drained. The worker marks a job
|
||||
* `processing` before doing the work; with the database refusing connections
|
||||
* that UPDATE fails outright, BullMQ exhausts its attempts, and the failure
|
||||
* path then cannot persist `failed` through the same dead database either. The
|
||||
* worst row was `processing` with its finished AVIF sitting in
|
||||
* outputs/<jobId>/ and output_refs null, so the bytes were unreachable through
|
||||
* the API and would have been swept by the 72-hour TTL.
|
||||
*
|
||||
* These cases pin the reconciler's contract: recover where the artifact
|
||||
* exists, fail only where nothing was produced, and never touch a row that a
|
||||
* live queue entry or a terminal status already accounts for.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { db, schema } from "../../../apps/api/src/db/index.js";
|
||||
import {
|
||||
reconcileStrandedJobs,
|
||||
UNRECOVERABLE_STRANDED_JOB_ERROR,
|
||||
} from "../../../apps/api/src/jobs/job-reconciliation.js";
|
||||
import { getQueue } from "../../../apps/api/src/jobs/queues.js";
|
||||
import { deletePrefix, putObject } from "../../../apps/api/src/lib/object-storage.js";
|
||||
import { buildTestApp, type TestApp } from "../test-server.js";
|
||||
|
||||
let testApp: TestApp;
|
||||
const seededPrefixes: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
testApp = await buildTestApp();
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
for (const prefix of seededPrefixes) await deletePrefix(prefix).catch(() => {});
|
||||
await testApp.cleanup();
|
||||
}, 15_000);
|
||||
|
||||
type Seed = {
|
||||
status?: "queued" | "processing" | "completed" | "failed" | "canceled";
|
||||
toolId?: string | null;
|
||||
type?: string;
|
||||
pool?: string | null;
|
||||
ageMs?: number;
|
||||
};
|
||||
|
||||
async function seedJob(seed: Seed = {}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db.insert(schema.jobs).values({
|
||||
id,
|
||||
toolId: seed.toolId === undefined ? "convert" : seed.toolId,
|
||||
pool: seed.pool === undefined ? "image" : seed.pool,
|
||||
type: seed.type ?? "tool",
|
||||
status: seed.status ?? "processing",
|
||||
inputRefs: [],
|
||||
createdAt: new Date(Date.now() - (seed.ageMs ?? 10 * 60_000)),
|
||||
completedAt: ["completed", "failed", "canceled"].includes(seed.status ?? "")
|
||||
? new Date()
|
||||
: null,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
async function seedOutput(jobId: string, name: string, body: Buffer): Promise<void> {
|
||||
const prefix = `outputs/${jobId}/`;
|
||||
if (!seededPrefixes.includes(prefix)) seededPrefixes.push(prefix);
|
||||
await putObject(`${prefix}${name}`, body);
|
||||
}
|
||||
|
||||
async function readJob(id: string) {
|
||||
const [row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, id));
|
||||
return row;
|
||||
}
|
||||
|
||||
function resultOf(row: Awaited<ReturnType<typeof readJob>>): Record<string, unknown> {
|
||||
return (row.progress?.result ?? {}) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("stranded-job reconciliation", () => {
|
||||
it("completes a processing row whose output is already on disk", async () => {
|
||||
const id = await seedJob({ status: "processing" });
|
||||
const bytes = Buffer.from("avif-bytes-that-were-really-produced");
|
||||
await seedOutput(id, "stress-large.avif", bytes);
|
||||
await seedOutput(id, "preview.webp", Buffer.from("preview"));
|
||||
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
expect(summary.outcomes.find((o) => o.jobId === id)?.resolution).toBe("recovered");
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status, "a finished job was not restored to completed").toBe("completed");
|
||||
expect(row.outputRefs, "the artifact on disk was not adopted").toEqual([
|
||||
`outputs/${id}/stress-large.avif`,
|
||||
]);
|
||||
expect(row.bytesOut).toBe(bytes.length);
|
||||
expect(row.completedAt).not.toBeNull();
|
||||
expect(resultOf(row).downloadUrl).toBe(`/api/v1/download/${id}/stress-large.avif`);
|
||||
expect(resultOf(row).previewUrl).toBe(`/api/v1/download/${id}/preview.webp`);
|
||||
});
|
||||
|
||||
it("serves the recovered artifact through the API", async () => {
|
||||
const id = await seedJob({ status: "processing" });
|
||||
await seedOutput(id, "recovered.bin", Buffer.from("0123456789"));
|
||||
|
||||
await reconcileStrandedJobs({ graceMs: 0 });
|
||||
|
||||
const row = await readJob(id);
|
||||
const res = await testApp.app.inject({
|
||||
method: "GET",
|
||||
url: String(resultOf(row).downloadUrl),
|
||||
});
|
||||
expect(res.statusCode, "the recovered bytes are still unreachable").toBe(200);
|
||||
expect(res.body).toBe("0123456789");
|
||||
});
|
||||
|
||||
it("fails a queued row that produced nothing", async () => {
|
||||
const id = await seedJob({ status: "queued" });
|
||||
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
expect(summary.outcomes.find((o) => o.jobId === id)?.resolution).toBe("failed");
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status).toBe("failed");
|
||||
expect(row.error?.message).toBe(UNRECOVERABLE_STRANDED_JOB_ERROR);
|
||||
expect(row.completedAt).not.toBeNull();
|
||||
expect(row.outputRefs).toBeNull();
|
||||
});
|
||||
|
||||
it("leaves a job alone while it still has a live queue entry", async () => {
|
||||
const id = await seedJob({ status: "processing" });
|
||||
// A long delay parks the job in "delayed": a live state no worker will
|
||||
// pick up during the test, which is exactly what an in-flight job looks
|
||||
// like to the reconciler.
|
||||
const queue = getQueue("image");
|
||||
await queue.add("convert", { jobId: id } as never, { jobId: id, delay: 10 * 60_000 });
|
||||
try {
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
expect(summary.outcomes.find((o) => o.jobId === id)?.resolution).toBe("live");
|
||||
expect((await readJob(id)).status, "a live job was reconciled out from under BullMQ").toBe(
|
||||
"processing",
|
||||
);
|
||||
} finally {
|
||||
await queue.remove(id).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
it("never resurrects a terminal row", async () => {
|
||||
const canceled = await seedJob({ status: "canceled" });
|
||||
const failed = await seedJob({ status: "failed" });
|
||||
const completed = await seedJob({ status: "completed" });
|
||||
// Output on disk must not tempt the reconciler into rewriting a cancel.
|
||||
await seedOutput(canceled, "partial.avif", Buffer.from("bytes"));
|
||||
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
for (const id of [canceled, failed, completed]) {
|
||||
expect(summary.outcomes.some((o) => o.jobId === id)).toBe(false);
|
||||
}
|
||||
expect((await readJob(canceled)).status).toBe("canceled");
|
||||
expect((await readJob(failed)).status).toBe("failed");
|
||||
expect((await readJob(completed)).status).toBe("completed");
|
||||
});
|
||||
|
||||
it("is idempotent across repeated sweeps", async () => {
|
||||
const recovered = await seedJob({ status: "processing" });
|
||||
const lost = await seedJob({ status: "queued" });
|
||||
await seedOutput(recovered, "out.avif", Buffer.from("bytes"));
|
||||
|
||||
await reconcileStrandedJobs({ graceMs: 0 });
|
||||
const afterFirst = await readJob(recovered);
|
||||
|
||||
const second = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
expect(second.outcomes.some((o) => o.jobId === recovered || o.jobId === lost)).toBe(false);
|
||||
|
||||
const afterSecond = await readJob(recovered);
|
||||
expect(afterSecond.outputRefs, "a second sweep duplicated the output set").toEqual(
|
||||
afterFirst.outputRefs,
|
||||
);
|
||||
expect(afterSecond.completedAt?.getTime()).toBe(afterFirst.completedAt?.getTime());
|
||||
expect((await readJob(lost)).status).toBe("failed");
|
||||
});
|
||||
|
||||
it("ignores rows younger than the grace window", async () => {
|
||||
const id = await seedJob({ status: "queued", ageMs: 0 });
|
||||
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 60_000 });
|
||||
expect(summary.outcomes.some((o) => o.jobId === id)).toBe(false);
|
||||
expect((await readJob(id)).status).toBe("queued");
|
||||
});
|
||||
|
||||
it("ignores rows whose id is not a BullMQ job id", async () => {
|
||||
// gdpr-export enqueues without a jobId, so BullMQ generates its own and the
|
||||
// queue lookup would wrongly report the row dead while the export runs.
|
||||
const systemJob = await seedJob({
|
||||
status: "processing",
|
||||
type: "system",
|
||||
toolId: "gdpr-export",
|
||||
});
|
||||
// An SSE-progress placeholder was never enqueued at all; the narrow startup
|
||||
// sweep in apps/api/src/index.ts owns those rows.
|
||||
const placeholder = await seedJob({ status: "processing", toolId: null, pool: null });
|
||||
|
||||
const summary = await reconcileStrandedJobs({ graceMs: 0 });
|
||||
expect(summary.outcomes.some((o) => o.jobId === systemJob)).toBe(false);
|
||||
expect(summary.outcomes.some((o) => o.jobId === placeholder)).toBe(false);
|
||||
expect((await readJob(systemJob)).status).toBe("processing");
|
||||
expect((await readJob(placeholder)).status).toBe("processing");
|
||||
});
|
||||
|
||||
it("skips a zero-byte output rather than calling it a result", async () => {
|
||||
const id = await seedJob({ status: "processing" });
|
||||
await seedOutput(id, "truncated.avif", Buffer.alloc(0));
|
||||
|
||||
await reconcileStrandedJobs({ graceMs: 0 });
|
||||
|
||||
const row = await readJob(id);
|
||||
expect(row.status).toBe("failed");
|
||||
expect(row.outputRefs).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user