diff --git a/tests/integration/generated/format-matrix-1.test.ts b/tests/integration/generated/format-matrix-1.test.ts index 3fc5f89b..aac9838f 100644 --- a/tests/integration/generated/format-matrix-1.test.ts +++ b/tests/integration/generated/format-matrix-1.test.ts @@ -11,13 +11,13 @@ import { join } from "node:path"; import { apiToolPath } from "@snapotter/shared"; import { describe, expect, it } from "vitest"; import { fixtureDir } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { ACCEPTABLE_FALLBACK_CODES, adminToken, app, buildPayload, formatSamplesForPart, - isAsyncFallback, needsFallback, setupMatrixApp, TOOLS, @@ -68,7 +68,7 @@ describe("Cross-format matrix", () => { // Assert status code // ------------------------------------------------------------------ // A heavy encode may fall back to async (202) under CI load -- accept it. - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { // Formats with optional decoders: accept success or graceful error diff --git a/tests/integration/generated/format-matrix-2.test.ts b/tests/integration/generated/format-matrix-2.test.ts index 19f4edb9..42325047 100644 --- a/tests/integration/generated/format-matrix-2.test.ts +++ b/tests/integration/generated/format-matrix-2.test.ts @@ -11,13 +11,13 @@ import { join } from "node:path"; import { apiToolPath } from "@snapotter/shared"; import { describe, expect, it } from "vitest"; import { fixtureDir } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { ACCEPTABLE_FALLBACK_CODES, adminToken, app, buildPayload, formatSamplesForPart, - isAsyncFallback, needsFallback, setupMatrixApp, TOOLS, @@ -68,7 +68,7 @@ describe("Cross-format matrix", () => { // Assert status code // ------------------------------------------------------------------ // A heavy encode may fall back to async (202) under CI load -- accept it. - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { // Formats with optional decoders: accept success or graceful error diff --git a/tests/integration/generated/format-matrix-3.test.ts b/tests/integration/generated/format-matrix-3.test.ts index 1582cb45..a694489d 100644 --- a/tests/integration/generated/format-matrix-3.test.ts +++ b/tests/integration/generated/format-matrix-3.test.ts @@ -11,13 +11,13 @@ import { join } from "node:path"; import { apiToolPath } from "@snapotter/shared"; import { describe, expect, it } from "vitest"; import { fixtureDir, fixtures } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { createMultipartPayload } from "../test-server.js"; import { adminToken, app, buildPayload, FORMAT_SAMPLES, - isAsyncFallback, setupMatrixApp, TOOLS, } from "./format-matrix.shared.js"; @@ -125,7 +125,7 @@ describe("Exotic format error resilience", () => { }); // Must not crash (500) — either succeed or return a clean error - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).not.toBe(500); expect([200, 202, 400, 422]).toContain(res.statusCode); diff --git a/tests/integration/generated/format-matrix-4.test.ts b/tests/integration/generated/format-matrix-4.test.ts index 06a4aecb..15298832 100644 --- a/tests/integration/generated/format-matrix-4.test.ts +++ b/tests/integration/generated/format-matrix-4.test.ts @@ -10,13 +10,13 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { fixtureDir, fixtures } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { createMultipartPayload } from "../test-server.js"; import { ACCEPTABLE_FALLBACK_CODES, adminToken, app, FORMAT_SAMPLES, - isAsyncFallback, needsFallback, setupMatrixApp, } from "./format-matrix.shared.js"; @@ -70,7 +70,7 @@ describe("Cross-format conversion matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); @@ -140,7 +140,7 @@ describe("Watermark-image cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { expect(ACCEPTABLE_FALLBACK_CODES).toContain(res.statusCode); } else { @@ -210,7 +210,7 @@ describe("Watermark-image cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { expect(ACCEPTABLE_FALLBACK_CODES).toContain(res.statusCode); } else { @@ -279,7 +279,7 @@ describe("Watermark-image cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.downloadUrl).toBeDefined(); @@ -335,7 +335,7 @@ describe("Image-to-PDF cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { expect(ACCEPTABLE_FALLBACK_CODES).toContain(res.statusCode); } else { @@ -407,7 +407,7 @@ describe("Image-to-PDF cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.downloadUrl).toBeDefined(); @@ -466,7 +466,7 @@ describe("Image-to-PDF cross-format matrix", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.pages).toBe(2); @@ -552,8 +552,8 @@ describe("Image-to-PDF cross-format matrix", () => { }); // Must not crash with 500 - if (isAsyncFallback(res)) return; - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).not.toBe(500); expect([200, 202, 400, 422]).toContain(res.statusCode); @@ -664,7 +664,7 @@ describe("Watermark-image exotic format error resilience", () => { body: payload, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).not.toBe(500); expect([200, 202, 400, 422]).toContain(res.statusCode); diff --git a/tests/integration/generated/format-matrix-comprehensive-1.test.ts b/tests/integration/generated/format-matrix-comprehensive-1.test.ts index 0d0689e6..084285bb 100644 --- a/tests/integration/generated/format-matrix-comprehensive-1.test.ts +++ b/tests/integration/generated/format-matrix-comprehensive-1.test.ts @@ -35,7 +35,7 @@ describe("Strip-metadata across all 16 primary formats", () => { async () => { const res = await callTool("strip-metadata", fmt, { stripAll: true }); if (!res) return; - const body = assertDownloadResponse(res, fmt); + const body = await assertDownloadResponse(res, fmt); // For core formats, verify that stripping metadata produces output // (it may or may not reduce size depending on whether the fixture @@ -109,7 +109,7 @@ describe("Image enhancement across all 16 primary formats", () => { async () => { const res = await callTool("image-enhancement", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt, "image-enhancement"), ); diff --git a/tests/integration/generated/format-matrix-comprehensive-2.test.ts b/tests/integration/generated/format-matrix-comprehensive-2.test.ts index c6f0f0bb..2a8d8b87 100644 --- a/tests/integration/generated/format-matrix-comprehensive-2.test.ts +++ b/tests/integration/generated/format-matrix-comprehensive-2.test.ts @@ -7,6 +7,7 @@ */ import { describe, expect, it, vi } from "vitest"; +import { settleAsyncFallback } from "../settle-job.js"; import { createMultipartPayload } from "../test-server.js"; import { ACCEPTABLE_FALLBACK_CODES, @@ -16,7 +17,6 @@ import { CORE_FORMATS, callTool, getTimeout, - isAsyncFallback, needsFallback, PRIMARY_FORMATS, setupMatrixApp, @@ -41,7 +41,7 @@ describe("Resize across all 16 primary formats", () => { async () => { const res = await callTool("resize", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -74,7 +74,7 @@ describe("Convert: 16 formats -> 3 output targets", () => { async () => { const res = await callTool("convert", fmt, { format: target.format }); if (!res) return; - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; if (needsFallback(fmt)) { expect(ACCEPTABLE_FALLBACK_CODES).toContain(res.statusCode); @@ -118,7 +118,7 @@ describe("Compress across all 16 primary formats", () => { async () => { const res = await callTool("compress", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -234,7 +234,7 @@ describe("No-crash matrix: 16 formats x 12 tools", () => { ); // A heavy encode may fall back to async (202) under CI load -- accept it. - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; // Must be a recognized status code if (needsFallback(fmt)) { diff --git a/tests/integration/generated/format-matrix-comprehensive-3.test.ts b/tests/integration/generated/format-matrix-comprehensive-3.test.ts index ddf8be5c..5514658e 100644 --- a/tests/integration/generated/format-matrix-comprehensive-3.test.ts +++ b/tests/integration/generated/format-matrix-comprehensive-3.test.ts @@ -50,7 +50,7 @@ describe("Color adjustments across all 16 primary formats", () => { async () => { const res = await callTool("adjust-colors", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -118,7 +118,7 @@ describe("Optimize-for-web across all 16 primary formats", () => { async () => { const res = await callTool("optimize-for-web", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); diff --git a/tests/integration/generated/format-matrix-comprehensive-4.test.ts b/tests/integration/generated/format-matrix-comprehensive-4.test.ts index a02e736a..8d4c46d1 100644 --- a/tests/integration/generated/format-matrix-comprehensive-4.test.ts +++ b/tests/integration/generated/format-matrix-comprehensive-4.test.ts @@ -7,13 +7,13 @@ */ import { describe, expect, it, vi } from "vitest"; +import { settleAsyncFallback } from "../settle-job.js"; import { assertDownloadResponse, CORE_FORMATS, callTool, type FormatDef, getTimeout, - isAsyncFallback, PRIMARY_FORMATS, setupMatrixApp, } from "./format-matrix-comprehensive.shared.js"; @@ -36,7 +36,7 @@ describe("Crop across all 16 primary formats", () => { async () => { const res = await callTool("crop", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -65,7 +65,7 @@ describe("Rotate across all 16 primary formats", () => { async () => { const res = await callTool("rotate", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -92,7 +92,7 @@ describe("Sharpening across all 16 primary formats", () => { async () => { const res = await callTool("sharpening", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -119,7 +119,7 @@ describe("Border across all 16 primary formats", () => { async () => { const res = await callTool("border", fmt, { ...cfg.settings }); if (!res) return; - assertDownloadResponse(res, fmt); + await assertDownloadResponse(res, fmt); }, getTimeout(fmt), ); @@ -150,7 +150,7 @@ describe("Extended conversion targets (core formats)", () => { it(`${fmt.name} -> ${target.format}`, { timeout: testTimeout }, async () => { const res = await callTool("convert", fmt, { format: target.format }); if (!res) return; - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); diff --git a/tests/integration/generated/format-matrix-comprehensive.shared.ts b/tests/integration/generated/format-matrix-comprehensive.shared.ts index 2fde67a1..357086a7 100644 --- a/tests/integration/generated/format-matrix-comprehensive.shared.ts +++ b/tests/integration/generated/format-matrix-comprehensive.shared.ts @@ -35,6 +35,7 @@ import { join } from "node:path"; import { apiToolPath } from "@snapotter/shared"; import { afterAll, beforeAll, expect } from "vitest"; import { fixtureDir } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { buildTestApp, createMultipartPayload, @@ -198,22 +199,6 @@ export function needsFallback(fmt: FormatDef): boolean { return fmt.needsCliDecoder || fmt.needsHeifDecoder || fmt.mayFailValidation; } -/** - * A CPU-heavy encode can exceed the sync window (SYNC_WAIT_MS, 30s in tests) - * under parallel CI load and fall back to async: 202 {jobId, async: true}. Per - * the documented 200-or-202 contract that is a legitimate "accepted & processing" - * outcome -- the worker runs the same process fn either way -- not a failure. - * Returns true (validating the async body shape) when the response is that - * fallback, so callers can treat it as a pass. - */ -export function isAsyncFallback(res: { statusCode: number; body: string }): boolean { - if (res.statusCode !== 202) return false; - const body = JSON.parse(res.body); - expect(body.async).toBe(true); - expect(body.jobId).toBeDefined(); - return true; -} - export function getTimeout(fmt: FormatDef, toolId?: string): number | undefined { if ((fmt.needsHeifDecoder || fmt.needsCliDecoder) && toolId === "image-enhancement") return 300_000; @@ -280,8 +265,11 @@ export async function callTool(toolId: string, fmt: FormatDef, settings: Record< * Assert a standard download response shape (used by most tools). * For fallback formats, accepts 200/400/422. For core formats, expects 200. */ -export function assertDownloadResponse(res: { statusCode: number; body: string }, fmt: FormatDef) { - if (isAsyncFallback(res)) return undefined; +export async function assertDownloadResponse( + res: { statusCode: number; body: string }, + fmt: FormatDef, +) { + if (await settleAsyncFallback(res)) return undefined; if (needsFallback(fmt)) { expect(ACCEPTABLE_FALLBACK_CODES).toContain(res.statusCode); } else { diff --git a/tests/integration/generated/format-matrix.shared.ts b/tests/integration/generated/format-matrix.shared.ts index f27dae97..cb3b007c 100644 --- a/tests/integration/generated/format-matrix.shared.ts +++ b/tests/integration/generated/format-matrix.shared.ts @@ -26,7 +26,7 @@ * holds the fixtures, tool table, and helpers they all share. */ -import { afterAll, beforeAll, expect } from "vitest"; +import { afterAll, beforeAll } from "vitest"; import { buildTestApp, createMultipartPayload, @@ -439,20 +439,6 @@ export function needsFallback(fmt: FormatSample): boolean { return fmt.needsCliDecoder || fmt.needsHeifDecoder || fmt.mayFailValidation; } -/** - * A heavy encode can exceed SYNC_WAIT_MS (30s in tests) under parallel CI load - * and fall back to 202 {jobId, async: true}. Per the 200-or-202 API contract - * that is a legitimate "accepted & processing" outcome, not a failure. - * Returns true (validating the async body shape) so callers can early-return. - */ -export function isAsyncFallback(res: { statusCode: number; body: string }): boolean { - if (res.statusCode !== 202) return false; - const body = JSON.parse(res.body); - expect(body.async).toBe(true); - expect(body.jobId).toBeDefined(); - return true; -} - /** * Build multipart payload for a tool request. * Info route does not use a "settings" field; image-to-base64 uses its own diff --git a/tests/integration/generated/new-formats.test.ts b/tests/integration/generated/new-formats.test.ts index c6efed87..c294a855 100644 --- a/tests/integration/generated/new-formats.test.ts +++ b/tests/integration/generated/new-formats.test.ts @@ -10,6 +10,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { fixtureDir, fixtures, readFixture } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { buildTestApp, createMultipartPayload, @@ -17,14 +18,6 @@ import { type TestApp, } from "../test-server.js"; -function isAsyncFallback(res: { statusCode: number; body: string }): boolean { - if (res.statusCode !== 202) return false; - const body = JSON.parse(res.body); - expect(body.async).toBe(true); - expect(body.jobId).toBeDefined(); - return true; -} - describe("New format support", () => { let testApp: TestApp; let app: TestApp["app"]; @@ -72,7 +65,7 @@ describe("New format support", () => { }); // Accept 200 (success) or 422 (encoder not available in test env) - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect([200, 422]).toContain(res.statusCode); if (res.statusCode === 200) { const json = JSON.parse(res.body); @@ -110,7 +103,7 @@ describe("New format support", () => { body, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect([200, 422]).toContain(res.statusCode); if (res.statusCode === 200) { const json = JSON.parse(res.body); @@ -198,7 +191,7 @@ describe("New format support", () => { headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType }, body, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect([200, 422]).toContain(res.statusCode); if (res.statusCode === 200) { const json = JSON.parse(res.body); @@ -240,7 +233,7 @@ describe("New format support", () => { headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType }, body, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect([200, 400, 422]).toContain(res.statusCode); if (res.statusCode === 200) { const json = JSON.parse(res.body); @@ -273,7 +266,7 @@ describe("New format support", () => { headers: { authorization: `Bearer ${adminToken}`, "content-type": contentType }, body, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect([200, 422]).toContain(res.statusCode); if (res.statusCode === 200) { const ct = res.headers["content-type"] as string; diff --git a/tests/integration/platform/cross-modality-smoke.test.ts b/tests/integration/platform/cross-modality-smoke.test.ts index 76cd3471..eea66fa3 100644 --- a/tests/integration/platform/cross-modality-smoke.test.ts +++ b/tests/integration/platform/cross-modality-smoke.test.ts @@ -18,6 +18,7 @@ import { spawnSync } from "node:child_process"; import { apiToolPath } from "@snapotter/shared"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { fixtures, readFixture } from "../../fixtures/index.js"; +import { settleAsyncFallback } from "../settle-job.js"; import { buildTestApp, createMultipartPayload, @@ -36,14 +37,6 @@ const HAS_QPDF = hasBinary("qpdf"); // ── Helpers ─────────────────────────────────────────────────────────── -function isAsyncFallback(res: { statusCode: number; body: string }): boolean { - if (res.statusCode !== 202) return false; - const body = JSON.parse(res.body); - expect(body.async).toBe(true); - expect(body.jobId).toBeDefined(); - return true; -} - async function postTool( app: TestApp["app"], token: string, @@ -110,7 +103,7 @@ describe("cross-modality launch smoke", () => { flipH: false, flipV: false, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -123,7 +116,7 @@ describe("cross-modality launch smoke", () => { async () => { const file = readFixture(fixtures.video.tiny("mp4")); const res = await postTool(app, adminToken, "mute-video", file, "tiny.mp4", "video/mp4", {}); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -141,7 +134,7 @@ describe("cross-modality launch smoke", () => { format: "mp3", bitrate: 128, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -164,7 +157,7 @@ describe("cross-modality launch smoke", () => { "application/pdf", { angle: 90, range: "1-z" }, ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -179,7 +172,7 @@ describe("cross-modality launch smoke", () => { const res = await postTool(app, adminToken, "csv-json", file, "tiny.csv", "text/csv", { direction: "csv-to-json", }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); diff --git a/tests/integration/settle-job.ts b/tests/integration/settle-job.ts index 79a5928e..ab50a6a5 100644 --- a/tests/integration/settle-job.ts +++ b/tests/integration/settle-job.ts @@ -1,5 +1,6 @@ import { setTimeout as delay } from "node:timers/promises"; import { eq } from "drizzle-orm"; +import { expect } from "vitest"; import { db, schema } from "../../apps/api/src/db/index.js"; import { requestCancel } from "../../apps/api/src/jobs/cancel.js"; import { waitForJob } from "../../apps/api/src/jobs/enqueue.js"; @@ -66,6 +67,64 @@ export async function cancelAcceptedJobAndWait( ); } +const TERMINAL_DATABASE_STATUSES = new Set(["completed", "failed", "canceled"]); + +/** + * Settle a 202 async-fallback response, and assert the job actually finished. + * + * A 202 means the sync window expired while the job was still running. Tests + * used to `return` here, which asserted nothing beyond the envelope and left + * the job running into whatever test came next (the leak + * `cancelAcceptedJobAndWait` exists to prevent). Because the window only + * expires under load, that made coverage depend on how busy the runner was: on + * CI 44 tests took this path and checked nothing, while the same tests on a dev + * machine finished inside the window and asserted in full. + * + * Waits for a terminal state instead and asserts what every caller actually + * cares about: the job finished, and if it failed it failed cleanly with a + * message rather than crashing. A clean failure is a legitimate outcome here, + * since the exotic-format fixtures are expected to be rejected. + * + * Returns true when the response was a 202 so callers keep their existing + * early-return shape; a 202 body carries no result to assert against. + */ +export async function settleAsyncFallback( + res: { statusCode: number; body: string }, + timeoutMs = 120_000, +): Promise { + if (res.statusCode !== 202) return false; + + const body = JSON.parse(res.body) as { async?: boolean; jobId?: string }; + expect(body.async).toBe(true); + expect(body.jobId).toBeDefined(); + const jobId = body.jobId as string; + + const deadline = Date.now() + timeoutMs; + let row: { status: string; pool: string | null; error: { message: string } | null } | undefined; + + while (Date.now() < deadline) { + [row] = await db + .select({ status: schema.jobs.status, pool: schema.jobs.pool, error: schema.jobs.error }) + .from(schema.jobs) + .where(eq(schema.jobs.id, jobId)); + if (row && TERMINAL_DATABASE_STATUSES.has(row.status)) break; + await delay(100); + } + + if (!row || !TERMINAL_DATABASE_STATUSES.has(row.status)) { + // Never leave it running: a stuck job starves every later test in the fork. + await cancelAcceptedJobAndWait(jobId, (row?.pool as Pool | undefined) ?? "image"); + throw new AcceptedJobTimeoutError(jobId, timeoutMs); + } + + if (row.status === "failed") { + expect(typeof row.error?.message, `job ${jobId} failed without a message`).toBe("string"); + expect(row.error?.message.length ?? 0).toBeGreaterThan(0); + } + + return true; +} + /** * Wait for an accepted job to succeed. If the observation window expires, * cancel and fully drain the job before failing the test so timed-out work can diff --git a/tests/integration/tools/image/image-enhancement.test.ts b/tests/integration/tools/image/image-enhancement.test.ts index 9917fa77..3f6a0a59 100644 --- a/tests/integration/tools/image/image-enhancement.test.ts +++ b/tests/integration/tools/image/image-enhancement.test.ts @@ -9,6 +9,7 @@ import sharp from "sharp"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { fixtures, readFixture } from "../../../fixtures/index.js"; +import { settleAsyncFallback } from "../../settle-job.js"; import { buildTestApp, createMultipartPayload, @@ -69,25 +70,11 @@ async function postTool( }); } -/** - * Under parallel CI load the sync window (30s) can expire before the worker - * finishes, returning 202 {jobId, async: true}. That is a legitimate "accepted - * & processing" outcome per the API contract. Return true so tests can skip - * assertions that only apply to the synchronous 200 path. - */ -function isAsyncFallback(res: { statusCode: number; body: string }): boolean { - if (res.statusCode !== 202) return false; - const body = JSON.parse(res.body); - expect(body.async).toBe(true); - expect(body.jobId).toBeDefined(); - return true; -} - // ── Auto mode (default) ─────────────────────────────────────────── describe("Auto mode", () => { it("enhances with default settings", async () => { const res = await postTool({}); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -96,7 +83,7 @@ describe("Auto mode", () => { it("enhances with explicit auto mode", async () => { const res = await postTool({ mode: "auto" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -107,7 +94,7 @@ describe("Auto mode", () => { describe("Enhancement modes", () => { it("enhances in portrait mode", async () => { const res = await postTool({ mode: "portrait" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -115,7 +102,7 @@ describe("Enhancement modes", () => { it("enhances in landscape mode", async () => { const res = await postTool({ mode: "landscape" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -123,7 +110,7 @@ describe("Enhancement modes", () => { it("enhances in low-light mode", async () => { const res = await postTool({ mode: "low-light" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -131,7 +118,7 @@ describe("Enhancement modes", () => { it("enhances in food mode", async () => { const res = await postTool({ mode: "food" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -139,7 +126,7 @@ describe("Enhancement modes", () => { it("enhances in document mode", async () => { const res = await postTool({ mode: "document" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -150,7 +137,7 @@ describe("Enhancement modes", () => { describe("Intensity parameter", () => { it("enhances at minimum intensity (0)", async () => { const res = await postTool({ intensity: 0 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -158,7 +145,7 @@ describe("Intensity parameter", () => { it("enhances at maximum intensity (100)", async () => { const res = await postTool({ intensity: 100 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -166,7 +153,7 @@ describe("Intensity parameter", () => { it("enhances at mid intensity (50, default)", async () => { const res = await postTool({ intensity: 50 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -184,7 +171,7 @@ describe("Selective corrections", () => { denoise: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -201,7 +188,7 @@ describe("Selective corrections", () => { denoise: true, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -218,7 +205,7 @@ describe("Selective corrections", () => { denoise: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -227,7 +214,7 @@ describe("Selective corrections", () => { describe("Output verification", () => { it("output differs from input", async () => { const res = await postTool({ mode: "auto", intensity: 80 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -242,7 +229,7 @@ describe("Output verification", () => { it("preserves image dimensions", async () => { const res = await postTool({ mode: "auto" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -272,7 +259,7 @@ describe("Analyze endpoint", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); // Analysis should return corrections object @@ -300,7 +287,7 @@ describe("Analyze endpoint", () => { describe("Multiple input formats", () => { it("enhances JPEG input", async () => { const res = await postTool({ mode: "auto" }, JPG, "test.jpg", "image/jpeg"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -308,7 +295,7 @@ describe("Multiple input formats", () => { it("enhances WebP input", async () => { const res = await postTool({ mode: "auto" }, WEBP, "test.webp", "image/webp"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -319,7 +306,7 @@ describe("Multiple input formats", () => { describe("Mode and intensity combinations", () => { it("applies portrait mode at high intensity", async () => { const res = await postTool({ mode: "portrait", intensity: 90 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -333,7 +320,7 @@ describe("Mode and intensity combinations", () => { it("applies low-light mode at low intensity", async () => { const res = await postTool({ mode: "low-light", intensity: 10 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.processedSize).toBeGreaterThan(0); @@ -341,7 +328,7 @@ describe("Mode and intensity combinations", () => { it("applies food mode at zero intensity (no-op)", async () => { const res = await postTool({ mode: "food", intensity: 0 }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -361,7 +348,7 @@ describe("Analyze endpoint details", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.corrections).toBeDefined(); @@ -381,7 +368,7 @@ describe("Analyze endpoint details", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.corrections).toBeDefined(); @@ -403,7 +390,7 @@ describe("Full corrections suite", () => { denoise: true, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -415,7 +402,7 @@ describe("Full corrections suite", () => { describe("Format preservation", () => { it("preserves JPEG format for JPEG input", async () => { const res = await postTool({ mode: "auto" }, JPG, "test.jpg", "image/jpeg"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -594,7 +581,7 @@ describe("Alpha channel preservation", () => { "rgba.png", "image/png", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -630,7 +617,7 @@ describe("Alpha channel preservation", () => { "semi.png", "image/png", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -679,7 +666,7 @@ describe("Tiny file handling", () => { it("enhances a 1x1 pixel image", async () => { const tiny = readFixture(fixtures.image.edge.px1); const res = await postTool({ mode: "auto" }, tiny, "tiny.png", "image/png"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.processedSize).toBeGreaterThan(0); @@ -698,7 +685,7 @@ describe("Empty file handling", () => { describe("Document mode variations", () => { it("enhances JPEG in document mode at high intensity", async () => { const res = await postTool({ mode: "document", intensity: 90 }, JPG, "doc.jpg", "image/jpeg"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -711,7 +698,7 @@ describe("Document mode variations", () => { "landscape.webp", "image/webp", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -760,7 +747,7 @@ describe("SVG input", () => { describe("Animated GIF input", () => { it("enhances animated GIF input", async () => { const res = await postTool({ mode: "auto" }, GIF, "animated.gif", "image/gif"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -781,7 +768,7 @@ describe("Selective correction edge cases", () => { denoise: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); @@ -797,7 +784,7 @@ describe("Selective correction edge cases", () => { denoise: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -811,7 +798,7 @@ describe("Partial corrections object", () => { contrast: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -821,7 +808,7 @@ describe("Partial corrections object", () => { const res = await postTool({ corrections: {}, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -832,7 +819,7 @@ describe("Partial corrections object", () => { describe("Output format for different input formats", () => { it("preserves WebP format for WebP input", async () => { const res = await postTool({ mode: "auto" }, WEBP, "test.webp", "image/webp"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -847,7 +834,7 @@ describe("Output format for different input formats", () => { it("preserves PNG format for PNG input", async () => { const res = await postTool({ mode: "auto" }, PNG, "test.png", "image/png"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -865,7 +852,7 @@ describe("Output format for different input formats", () => { describe("Output dimension verification", () => { it("preserves JPEG input dimensions", async () => { const res = await postTool({ mode: "auto" }, JPG, "test.jpg", "image/jpeg"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -881,7 +868,7 @@ describe("Output dimension verification", () => { it("preserves WebP input dimensions", async () => { const res = await postTool({ mode: "landscape" }, WEBP, "test.webp", "image/webp"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -900,7 +887,7 @@ describe("Output dimension verification", () => { describe("Response structure", () => { it("returns all expected fields in 200 response", async () => { const res = await postTool({ mode: "auto" }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -931,7 +918,7 @@ describe("Analyze endpoint format coverage", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.corrections).toBeDefined(); @@ -957,7 +944,7 @@ describe("Analyze endpoint format coverage", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.corrections).toBeDefined(); @@ -980,7 +967,7 @@ describe("Mode with selective corrections", () => { denoise: false, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -999,7 +986,7 @@ describe("Mode with selective corrections", () => { denoise: true, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); }); }); @@ -1027,7 +1014,7 @@ describe("Large file with modes", () => { "stress-large.jpg", "image/jpeg", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.processedSize).toBeGreaterThan(0); @@ -1041,7 +1028,7 @@ describe("Large file with modes", () => { "stress-large.jpg", "image/jpeg", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.processedSize).toBeGreaterThan(0); @@ -1052,7 +1039,7 @@ describe("Large file with modes", () => { describe("Deep Enhance", () => { it("accepts deepEnhance setting and returns 200", async () => { const res = await postTool({ deepEnhance: true }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -1061,7 +1048,7 @@ describe("Deep Enhance", () => { it("works without deepEnhance (default false)", async () => { const res = await postTool({}); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -1095,7 +1082,7 @@ describe("Darkening regression", () => { "midgray.jpg", "image/jpeg", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -1131,7 +1118,7 @@ describe("Darkening regression", () => { const originalMean = originalStats.channels[0].mean; const res = await postTool({ mode: "auto", intensity: 50 }, bright, "bright.jpg", "image/jpeg"); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); @@ -1158,7 +1145,7 @@ describe("Portrait image enhancement", () => { "portrait.jpg", "image/jpeg", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.downloadUrl).toBeDefined(); @@ -1182,7 +1169,7 @@ describe("Portrait image enhancement", () => { "portrait-color.jpg", "image/jpeg", ); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.processedSize).toBeGreaterThan(0); @@ -1212,7 +1199,7 @@ describe("Batch processing", () => { }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); expect(res.headers["content-type"]).toBe("application/zip"); @@ -1261,7 +1248,7 @@ describe("Analyze endpoint response structure", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result).toHaveProperty("scores"); @@ -1321,7 +1308,7 @@ describe("Low-light image analysis", () => { authorization: `Bearer ${adminToken}`, }, }); - if (isAsyncFallback(res)) return; + if (await settleAsyncFallback(res)) return; expect(res.statusCode).toBe(200); const result = JSON.parse(res.body); expect(result.suggestedMode).toBe("low-light"); diff --git a/tests/setup/per-fork-env.ts b/tests/setup/per-fork-env.ts index 9eda2ffe..df70f3c3 100644 --- a/tests/setup/per-fork-env.ts +++ b/tests/setup/per-fork-env.ts @@ -25,13 +25,17 @@ process.env.BULLMQ_PREFIX = `snapotter_test_${suffix}`; // Heavy format conversions can exceed the 8s production default under parallel // test forks; 30s keeps tool routes synchronous (200) in tests while production -// stays at 8s. The constrained docker test image (macOS Docker VM, where Sharp -// and FFmpeg run ~2-3x slower) can request a larger window via SYNC_WAIT_MS; -// honor it rather than clobbering, but never drop below the 30s test floor. -const requestedSyncWait = Number(process.env.SYNC_WAIT_MS); -process.env.SYNC_WAIT_MS = String( - Number.isFinite(requestedSyncWait) && requestedSyncWait > 30000 ? requestedSyncWait : 30000, -); +// stays at 8s. +// +// An explicit SYNC_WAIT_MS is now honored verbatim rather than floored. The +// constrained docker test image (macOS Docker VM, where Sharp and FFmpeg run +// ~2-3x slower) still widens the window, and forcing it *down* (SYNC_WAIT_MS=0) +// drives every tool through its 202 path, which is the only way to exercise +// that branch on a machine fast enough to never hit it naturally. +const requestedSyncWait = process.env.SYNC_WAIT_MS?.trim(); +const hasExplicitSyncWait = + Boolean(requestedSyncWait) && Number.isFinite(Number(requestedSyncWait)); +process.env.SYNC_WAIT_MS = hasExplicitSyncWait ? (requestedSyncWait as string) : "30000"; const dbName = `snapotter_test_${suffix}`; // pid digits + uuid hex: identifier-safe const admin = new pg.Client({ connectionString: baseUrl }); await admin.connect();