mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: make OCR portable and reliable across AMD64 and ARM64 (#519)
* fix: make OCR portable and reliable * fix: harden OCR installation portability * fix: pin OCR partials across downloads * fix: make OCR execution reliably asynchronous * fix: harden OCR portability and docs routes * fix: preserve decoder and docs safeguards
This commit is contained in:
@@ -20,6 +20,8 @@ const {
|
||||
mockUnlink,
|
||||
mockRm,
|
||||
mockExecFile,
|
||||
mockRunOcrRuntime,
|
||||
mockRunTesseract,
|
||||
} = vi.hoisted(() => {
|
||||
const mockRunPythonWithProgress = vi.fn();
|
||||
const mockParseStdoutJson = vi.fn();
|
||||
@@ -31,6 +33,7 @@ const {
|
||||
chain.jpeg = vi.fn().mockReturnValue(chain);
|
||||
chain.resize = vi.fn().mockReturnValue(chain);
|
||||
chain.toBuffer = vi.fn().mockResolvedValue(Buffer.from("mock-png"));
|
||||
chain.toFile = vi.fn().mockResolvedValue({});
|
||||
chain.metadata = vi.fn().mockResolvedValue({
|
||||
width: 800,
|
||||
height: 600,
|
||||
@@ -54,6 +57,8 @@ const {
|
||||
mockUnlink: vi.fn().mockResolvedValue(undefined),
|
||||
mockRm: vi.fn().mockResolvedValue(undefined),
|
||||
mockExecFile: vi.fn(),
|
||||
mockRunOcrRuntime: vi.fn(),
|
||||
mockRunTesseract: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -63,6 +68,19 @@ vi.mock("../../../packages/ai/src/bridge.js", () => ({
|
||||
isGpuAvailable: mockIsGpuAvailable,
|
||||
}));
|
||||
|
||||
vi.mock("../../../packages/ai/src/ocr-runtime-dispatcher.js", () => ({
|
||||
runOcrRuntime: mockRunOcrRuntime,
|
||||
}));
|
||||
|
||||
vi.mock("../../../packages/ai/src/tesseract.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../../../packages/ai/src/tesseract.js")>();
|
||||
return {
|
||||
...actual,
|
||||
runAdaptiveTesseract: mockRunTesseract,
|
||||
runTesseract: mockRunTesseract,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("sharp", () => ({ default: mockSharp }));
|
||||
|
||||
vi.mock("node:fs/promises", () => ({
|
||||
@@ -118,6 +136,33 @@ beforeEach(() => {
|
||||
mockReadFile.mockResolvedValue(Buffer.from("output-buffer"));
|
||||
mockWriteFile.mockResolvedValue(undefined);
|
||||
mockRunPythonWithProgress.mockResolvedValue({ stdout: "", stderr: "" });
|
||||
mockRunTesseract.mockResolvedValue({
|
||||
text: "Sample OCR text",
|
||||
engine: "tesseract",
|
||||
device: "cpu",
|
||||
provider: "native",
|
||||
});
|
||||
mockRunOcrRuntime.mockResolvedValue({
|
||||
result: {
|
||||
success: true,
|
||||
text: "Accurate OCR text",
|
||||
engine: "rapidocr-onnx",
|
||||
requestedQuality: "best",
|
||||
actualQuality: "best",
|
||||
device: "cpu",
|
||||
provider: "CPUExecutionProvider",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
},
|
||||
stderr: "",
|
||||
runtime: {
|
||||
generation: "test",
|
||||
artifactVersion: "2.1.0",
|
||||
target: "linux-amd64-cpu-py312",
|
||||
providers: ["CPUExecutionProvider"],
|
||||
models: {},
|
||||
},
|
||||
});
|
||||
mockParseStdoutJson.mockReturnValue({ success: true });
|
||||
mockIsGpuAvailable.mockReturnValue(false);
|
||||
});
|
||||
@@ -817,74 +862,72 @@ describe("noiseRemoval", () => {
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe("extractText (OCR)", () => {
|
||||
it("calls ocr.py", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: true, text: "hello" });
|
||||
|
||||
it("uses built-in Tesseract for the default Fast tier", async () => {
|
||||
await extractText(INPUT_BUFFER, OUTPUT_DIR);
|
||||
|
||||
const [script] = mockRunPythonWithProgress.mock.calls[0];
|
||||
expect(script).toBe("ocr.py");
|
||||
expect(mockRunTesseract).toHaveBeenCalledWith(
|
||||
expect.stringContaining("input_ocr.png"),
|
||||
expect.objectContaining({ timeoutMs: expect.any(Number) }),
|
||||
);
|
||||
expect(mockRunPythonWithProgress).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes options as JSON", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: true, text: "" });
|
||||
|
||||
it("passes accurate options to the isolated runtime", async () => {
|
||||
await extractText(INPUT_BUFFER, OUTPUT_DIR, { quality: "best", language: "en" });
|
||||
|
||||
const [, args] = mockRunPythonWithProgress.mock.calls[0];
|
||||
const [, args] = mockRunOcrRuntime.mock.calls[0];
|
||||
const optsArg = JSON.parse(args[1]);
|
||||
expect(optsArg.quality).toBe("best");
|
||||
expect(optsArg.language).toBe("en");
|
||||
});
|
||||
|
||||
it("returns text and engine", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({
|
||||
success: true,
|
||||
text: "Sample OCR text",
|
||||
engine: "paddleocr",
|
||||
});
|
||||
|
||||
it("returns text and truthful engine metadata", async () => {
|
||||
const result = await extractText(INPUT_BUFFER, OUTPUT_DIR);
|
||||
|
||||
expect(result.text).toBe("Sample OCR text");
|
||||
expect(result.engine).toBe("paddleocr");
|
||||
expect(result.engine).toBe("tesseract");
|
||||
expect(result.actualQuality).toBe("fast");
|
||||
});
|
||||
|
||||
it("resizes image to max 2048px", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: true, text: "" });
|
||||
|
||||
it("preserves source resolution", async () => {
|
||||
const chain = createSharpChain();
|
||||
mockSharp.mockReturnValue(chain);
|
||||
|
||||
await extractText(INPUT_BUFFER, OUTPUT_DIR);
|
||||
|
||||
expect(chain.resize).toHaveBeenCalledWith({
|
||||
width: 2048,
|
||||
height: 2048,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
expect(chain.resize).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws on Fast failure", async () => {
|
||||
mockRunTesseract.mockRejectedValueOnce(new Error("Tesseract failed"));
|
||||
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("Tesseract failed");
|
||||
});
|
||||
|
||||
it("rejects incomplete accurate metadata without falling back", async () => {
|
||||
mockRunOcrRuntime.mockResolvedValueOnce({
|
||||
result: { success: true, text: "incomplete" },
|
||||
stderr: "",
|
||||
runtime: {
|
||||
generation: "test",
|
||||
artifactVersion: "2.1.0",
|
||||
target: "linux-amd64-cpu-py312",
|
||||
providers: ["CPUExecutionProvider"],
|
||||
models: {},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("throws on failure", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: false, error: "PaddleOCR init failed" });
|
||||
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("PaddleOCR init failed");
|
||||
});
|
||||
|
||||
it("provides fallback error message", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: false });
|
||||
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("OCR failed");
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR, { quality: "best" })).rejects.toThrow(
|
||||
"invalid metadata",
|
||||
);
|
||||
expect(mockRunTesseract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calculates timeout based on megapixels", async () => {
|
||||
mockParseStdoutJson.mockReturnValue({ success: true, text: "" });
|
||||
|
||||
await extractText(INPUT_BUFFER, OUTPUT_DIR);
|
||||
|
||||
const [, , opts] = mockRunPythonWithProgress.mock.calls[0];
|
||||
expect(opts.timeout).toBeGreaterThanOrEqual(600_000);
|
||||
const [, opts] = mockRunTesseract.mock.calls[0];
|
||||
expect(opts.timeoutMs).toBeGreaterThanOrEqual(600_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1382,7 +1425,11 @@ describe("cross-cutting tool patterns", () => {
|
||||
await expect(removeRedEye(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("timed out");
|
||||
await expect(restorePhoto(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("timed out");
|
||||
await expect(upscale(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("timed out");
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR)).rejects.toThrow("timed out");
|
||||
|
||||
mockRunOcrRuntime.mockRejectedValueOnce(new Error("OCR runtime timed out"));
|
||||
await expect(extractText(INPUT_BUFFER, OUTPUT_DIR, { quality: "balanced" })).rejects.toThrow(
|
||||
"timed out",
|
||||
);
|
||||
});
|
||||
|
||||
it("all Python tools convert input to PNG via sharp", async () => {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { env } from "../../../apps/api/src/config.js";
|
||||
|
||||
const qpdf = vi.hoisted(() => ({
|
||||
available: vi.fn(),
|
||||
check: vi.fn(),
|
||||
pageCount: vi.fn(),
|
||||
requiresPassword: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@snapotter/doc-engine", () => ({
|
||||
qpdfAvailable: qpdf.available,
|
||||
qpdfCheck: qpdf.check,
|
||||
qpdfPageCount: qpdf.pageCount,
|
||||
qpdfRequiresPassword: qpdf.requiresPassword,
|
||||
}));
|
||||
|
||||
import {
|
||||
DocumentInputHandler,
|
||||
validatePdfPath,
|
||||
} from "../../../apps/api/src/modality/document-input.js";
|
||||
|
||||
let scratchDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
scratchDir = mkdtempSync(join(tmpdir(), "snapotter-document-input-"));
|
||||
qpdf.available.mockReturnValue(true);
|
||||
qpdf.requiresPassword.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(scratchDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("DocumentInputHandler password policy", () => {
|
||||
it("rejects an encrypted PDF when the consuming tool cannot accept a password", async () => {
|
||||
const handler = new DocumentInputHandler();
|
||||
|
||||
await expect(
|
||||
handler.prepare(Buffer.from("%PDF-encrypted"), "scan.pdf", {
|
||||
scratchDir,
|
||||
rejectPasswordProtected: true,
|
||||
}),
|
||||
).rejects.toThrow(/password-protected/i);
|
||||
expect(qpdf.check).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves the existing policy for tools such as unlock-pdf", async () => {
|
||||
const handler = new DocumentInputHandler();
|
||||
const input = Buffer.from("%PDF-encrypted");
|
||||
|
||||
await expect(handler.prepare(input, "scan.pdf", { scratchDir })).resolves.toEqual({
|
||||
buffer: input,
|
||||
filename: "scan.pdf",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires PDF magic for a PDF-only consumer regardless of the client filename", async () => {
|
||||
const handler = new DocumentInputHandler();
|
||||
|
||||
await expect(
|
||||
handler.prepare(Buffer.from("not a PDF"), "renamed.txt", {
|
||||
scratchDir,
|
||||
rejectPasswordProtected: true,
|
||||
}),
|
||||
).rejects.toThrow(/PDF header/i);
|
||||
expect(qpdf.requiresPassword).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("path-backed PDF validation", () => {
|
||||
it("runs structural, encryption, and page validation directly against the file path", async () => {
|
||||
const inputPath = join(scratchDir, "scan.pdf");
|
||||
writeFileSync(inputPath, "%PDF-path-backed");
|
||||
qpdf.requiresPassword.mockResolvedValueOnce(false);
|
||||
qpdf.pageCount.mockResolvedValueOnce(3);
|
||||
const originalMaxPages = env.MAX_PDF_PAGES;
|
||||
env.MAX_PDF_PAGES = 10;
|
||||
|
||||
try {
|
||||
await expect(
|
||||
validatePdfPath(inputPath, { rejectPasswordProtected: true }),
|
||||
).resolves.toBeUndefined();
|
||||
} finally {
|
||||
env.MAX_PDF_PAGES = originalMaxPages;
|
||||
}
|
||||
|
||||
expect(qpdf.requiresPassword).toHaveBeenCalledWith(inputPath);
|
||||
expect(qpdf.check).toHaveBeenCalledWith(inputPath);
|
||||
expect(qpdf.pageCount).toHaveBeenCalledWith(inputPath);
|
||||
});
|
||||
|
||||
it("enforces the PDF page cap without loading the file into a Buffer", async () => {
|
||||
const inputPath = join(scratchDir, "too-many-pages.pdf");
|
||||
writeFileSync(inputPath, "%PDF-path-backed");
|
||||
qpdf.requiresPassword.mockResolvedValueOnce(false);
|
||||
qpdf.pageCount.mockResolvedValueOnce(11);
|
||||
const originalMaxPages = env.MAX_PDF_PAGES;
|
||||
env.MAX_PDF_PAGES = 10;
|
||||
|
||||
try {
|
||||
await expect(validatePdfPath(inputPath, { rejectPasswordProtected: true })).rejects.toThrow(
|
||||
/11 pages.*maximum of 10/i,
|
||||
);
|
||||
} finally {
|
||||
env.MAX_PDF_PAGES = originalMaxPages;
|
||||
}
|
||||
});
|
||||
|
||||
it("reports qpdf structural failures as input validation errors", async () => {
|
||||
const inputPath = join(scratchDir, "damaged.pdf");
|
||||
writeFileSync(inputPath, "%PDF-path-backed");
|
||||
qpdf.requiresPassword.mockResolvedValueOnce(false);
|
||||
qpdf.check.mockRejectedValueOnce(new Error("xref table is corrupt"));
|
||||
|
||||
await expect(validatePdfPath(inputPath, { rejectPasswordProtected: true })).rejects.toThrow(
|
||||
/Damaged PDF.*xref table is corrupt/i,
|
||||
);
|
||||
expect(qpdf.pageCount).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops before invoking qpdf when path validation is canceled", async () => {
|
||||
const inputPath = join(scratchDir, "canceled.pdf");
|
||||
writeFileSync(inputPath, "%PDF-canceled");
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
validatePdfPath(inputPath, {
|
||||
rejectPasswordProtected: true,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(qpdf.requiresPassword).not.toHaveBeenCalled();
|
||||
expect(qpdf.check).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,11 @@ import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import sharp from "sharp";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { decodeToSharpCompat, needsCliDecode } from "../../../apps/api/src/lib/format-decoders.js";
|
||||
import {
|
||||
buildImageMagickResourceLimitArgs,
|
||||
decodeToSharpCompat,
|
||||
needsCliDecode,
|
||||
} from "../../../apps/api/src/lib/format-decoders.js";
|
||||
import { encodeQoi } from "../../../apps/api/src/lib/format-encoders.js";
|
||||
import { fixtures, readFixture } from "../../fixtures/index.js";
|
||||
|
||||
@@ -42,7 +46,8 @@ async function assertValidImage(buf: Buffer): Promise<{ width: number; height: n
|
||||
const meta = await sharp(buf).metadata();
|
||||
expect(meta.width).toBeGreaterThan(0);
|
||||
expect(meta.height).toBeGreaterThan(0);
|
||||
return { width: meta.width!, height: meta.height! };
|
||||
if (!meta.width || !meta.height) throw new Error("Decoded image has no dimensions");
|
||||
return { width: meta.width, height: meta.height };
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
@@ -146,6 +151,36 @@ describe("needsCliDecode", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildImageMagickResourceLimitArgs", () => {
|
||||
it("uses unitless width and height values that work in ImageMagick 6 and 7", () => {
|
||||
expect(
|
||||
buildImageMagickResourceLimitArgs({
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
}),
|
||||
).toEqual([
|
||||
"-limit",
|
||||
"width",
|
||||
"40000",
|
||||
"-limit",
|
||||
"height",
|
||||
"40000",
|
||||
"-limit",
|
||||
"area",
|
||||
"640000000B",
|
||||
"-limit",
|
||||
"memory",
|
||||
"640000000B",
|
||||
"-limit",
|
||||
"map",
|
||||
"640000000B",
|
||||
"-limit",
|
||||
"disk",
|
||||
"1280000000B",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeToSharpCompat", () => {
|
||||
it("returns buffer unchanged for unknown/native formats", async () => {
|
||||
const buf = Buffer.from("test data");
|
||||
@@ -189,10 +224,13 @@ describe("decodeToSharpCompat", () => {
|
||||
expect(result).toBe(buf);
|
||||
});
|
||||
|
||||
it("decodes BMP to valid PNG", async () => {
|
||||
it("decodes BMP to valid PNG with bounded ImageMagick 6/7 limits", async () => {
|
||||
try {
|
||||
const input = readFixture(fixtures.image.formats("bmp"));
|
||||
const result = await decodeToSharpCompat(input, "bmp");
|
||||
const result = await decodeToSharpCompat(input, "bmp", undefined, {
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
});
|
||||
expect(isPng(result)).toBe(true);
|
||||
await assertValidImage(result);
|
||||
} catch (err) {
|
||||
@@ -412,7 +450,71 @@ describe("decodeToSharpCompat - individual decoder verification", () => {
|
||||
}
|
||||
});
|
||||
|
||||
describe("decodeToSharpCompat - bounded preflight", () => {
|
||||
const formats = [
|
||||
["raw", "dng"],
|
||||
["ico", undefined],
|
||||
["tga", undefined],
|
||||
["psd", undefined],
|
||||
["exr", undefined],
|
||||
["hdr", undefined],
|
||||
["bmp", undefined],
|
||||
["jxl", undefined],
|
||||
["jp2", undefined],
|
||||
["eps", undefined],
|
||||
["dds", undefined],
|
||||
["cur", undefined],
|
||||
["dpx", undefined],
|
||||
["fits", undefined],
|
||||
["ppm", undefined],
|
||||
["pgm", undefined],
|
||||
["pbm", undefined],
|
||||
] as const;
|
||||
|
||||
it.each(formats)("rejects oversized %s metadata before pixel decode", async (format, ext) => {
|
||||
const fixtureFormat = format === "raw" ? "dng" : format;
|
||||
const input = readFixture(fixtures.image.formats(fixtureFormat));
|
||||
|
||||
await expect(decodeToSharpCompat(input, format, ext, { maxPixels: 1 })).rejects.toThrow(
|
||||
/pixel safety limit/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeToSharpCompat - QOI decoder", () => {
|
||||
it("rejects an extreme QOI side before allocating pixels", async () => {
|
||||
const input = Buffer.alloc(14);
|
||||
input.write("qoif", 0, 4, "ascii");
|
||||
input.writeUInt32BE(40_001, 4);
|
||||
input.writeUInt32BE(1, 8);
|
||||
input[12] = 3;
|
||||
|
||||
await expect(
|
||||
decodeToSharpCompat(input, "qoi", undefined, {
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
}),
|
||||
).rejects.toThrow(/dimension safety limit.*40,001x1/i);
|
||||
});
|
||||
|
||||
it("rejects QOI dimensions over a caller-supplied pixel cap before allocating pixels", async () => {
|
||||
const input = readFixture(fixtures.image.formats("qoi"));
|
||||
|
||||
await expect(decodeToSharpCompat(input, "qoi", undefined, { maxPixels: 1 })).rejects.toThrow(
|
||||
/pixel safety limit/i,
|
||||
);
|
||||
});
|
||||
|
||||
it("honors an already-aborted request", async () => {
|
||||
const input = readFixture(fixtures.image.formats("qoi"));
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
decodeToSharpCompat(input, "qoi", undefined, { signal: controller.signal }),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("decodes QOI fixture to valid PNG", async () => {
|
||||
const input = readFixture(fixtures.image.formats("qoi"));
|
||||
const result = await decodeToSharpCompat(input, "qoi");
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const source = readFileSync(resolve(root, "apps/api/src/index.ts"), "utf8");
|
||||
|
||||
describe("API graceful shutdown order", () => {
|
||||
it("drains BullMQ workers before force-stopping the Accurate OCR dispatcher", () => {
|
||||
const shutdownStart = source.indexOf("async function shutdown(signal: string)");
|
||||
const shutdownEnd = source.indexOf('process.on("SIGTERM"', shutdownStart);
|
||||
const shutdownSource = source.slice(shutdownStart, shutdownEnd);
|
||||
const closeWorkers = shutdownSource.indexOf("await closeWorkers();");
|
||||
const shutdownOcrDispatcher = shutdownSource.indexOf("shutdownOcrDispatcher");
|
||||
|
||||
expect(shutdownStart).toBeGreaterThanOrEqual(0);
|
||||
expect(shutdownEnd).toBeGreaterThan(shutdownStart);
|
||||
expect(closeWorkers).toBeGreaterThanOrEqual(0);
|
||||
expect(shutdownOcrDispatcher).toBeGreaterThanOrEqual(0);
|
||||
expect(closeWorkers).toBeLessThan(shutdownOcrDispatcher);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,3 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import sharp from "sharp";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { decodeHeic, ensureSharpCompat } from "../../../apps/api/src/lib/heic-converter.js";
|
||||
@@ -16,7 +15,55 @@ function makeHeicHeader(brand: string): Buffer {
|
||||
return buf;
|
||||
}
|
||||
|
||||
function makeIspeBox(width: number, height: number): Buffer {
|
||||
const box = Buffer.alloc(20);
|
||||
box.writeUInt32BE(20, 0);
|
||||
box.write("ispe", 4, 4, "ascii");
|
||||
box.writeUInt32BE(width, 12);
|
||||
box.writeUInt32BE(height, 16);
|
||||
return box;
|
||||
}
|
||||
|
||||
describe("decodeHeic", () => {
|
||||
it("rejects source dimensions over a caller-supplied pixel cap", async () => {
|
||||
const heicBuf = readFixture(fixtures.image.formats("heic"));
|
||||
|
||||
await expect(decodeHeic(heicBuf, { maxPixels: 1 })).rejects.toThrow(/pixel safety limit/i);
|
||||
});
|
||||
|
||||
it("checks every image extent instead of trusting a small first thumbnail", async () => {
|
||||
const multiImage = Buffer.concat([
|
||||
makeHeicHeader("heic"),
|
||||
makeIspeBox(1, 1),
|
||||
makeIspeBox(10_000, 10_000),
|
||||
]);
|
||||
|
||||
await expect(decodeHeic(multiImage, { maxPixels: 40_000_000 })).rejects.toThrow(
|
||||
/pixel safety limit/i,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an extreme image side from encoded metadata before starting a decoder", async () => {
|
||||
const extremeImage = Buffer.concat([
|
||||
makeHeicHeader("heic"),
|
||||
makeIspeBox(1_000, 1_000),
|
||||
makeIspeBox(40_001, 1),
|
||||
]);
|
||||
|
||||
await expect(
|
||||
decodeHeic(extremeImage, { maxDimension: 40_000, maxPixels: 40_000_000 }),
|
||||
).rejects.toThrow(/dimension safety limit.*40,001x1/i);
|
||||
});
|
||||
|
||||
it("honors an already-aborted request", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
decodeHeic(readFixture(fixtures.image.formats("heic")), { signal: controller.signal }),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("decodes sample.heic to a valid PNG buffer", async () => {
|
||||
const heicBuf = readFixture(fixtures.image.formats("heic"));
|
||||
const result = await decodeHeic(heicBuf);
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
autoOrient: vi.fn(),
|
||||
decodeAnyFormat: vi.fn(),
|
||||
decodeHeic: vi.fn(),
|
||||
decodeToSharpCompat: vi.fn(),
|
||||
decompressSvgz: vi.fn(),
|
||||
metadata: vi.fn(),
|
||||
raw: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
sanitizeSvg: vi.fn(),
|
||||
toBuffer: vi.fn(),
|
||||
validateImageBuffer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("sharp", () => ({
|
||||
default: vi.fn(() => ({
|
||||
metadata: mocks.metadata,
|
||||
raw: mocks.raw,
|
||||
resize: mocks.resize,
|
||||
toBuffer: mocks.toBuffer,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/auto-orient.js", () => ({
|
||||
autoOrient: mocks.autoOrient,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/file-validation.js", () => ({
|
||||
validateImageBuffer: mocks.validateImageBuffer,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/format-decoders.js", () => ({
|
||||
decodeAnyFormat: mocks.decodeAnyFormat,
|
||||
decodeToSharpCompat: mocks.decodeToSharpCompat,
|
||||
needsCliDecode: (format: string) => format === "raw",
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/heic-converter.js", () => ({
|
||||
decodeHeic: mocks.decodeHeic,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/svg-sanitize.js", () => ({
|
||||
decompressSvgz: mocks.decompressSvgz,
|
||||
sanitizeSvg: mocks.sanitizeSvg,
|
||||
}));
|
||||
|
||||
import { InputValidationError } from "../../../apps/api/src/modality/contract.js";
|
||||
import { ImageInputHandler } from "../../../apps/api/src/modality/image-input.js";
|
||||
|
||||
const RAW = Buffer.from("raw");
|
||||
const DECODED = Buffer.from("decoded");
|
||||
const ORIENTED = Buffer.from("oriented");
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.autoOrient.mockResolvedValue(ORIENTED);
|
||||
mocks.decodeAnyFormat.mockResolvedValue(DECODED);
|
||||
mocks.decodeHeic.mockResolvedValue(DECODED);
|
||||
mocks.decodeToSharpCompat.mockResolvedValue(DECODED);
|
||||
mocks.decompressSvgz.mockImplementation((value) => value);
|
||||
mocks.sanitizeSvg.mockImplementation((value) => value);
|
||||
mocks.metadata.mockResolvedValue({ width: 1_000, height: 1_000 });
|
||||
mocks.resize.mockReturnValue({ raw: mocks.raw });
|
||||
mocks.raw.mockReturnValue({ toBuffer: mocks.toBuffer });
|
||||
mocks.toBuffer.mockResolvedValue(Buffer.from("pixel"));
|
||||
});
|
||||
|
||||
describe("ImageInputHandler resource bounds", () => {
|
||||
it("rejects an extreme image side before decoding or auto-orientation", async () => {
|
||||
mocks.validateImageBuffer.mockResolvedValue({
|
||||
valid: true,
|
||||
format: "jpeg",
|
||||
width: 40_001,
|
||||
height: 1,
|
||||
});
|
||||
|
||||
await expect(
|
||||
new ImageInputHandler().prepare(RAW, "scan.jpg", {
|
||||
scratchDir: "/tmp/ocr",
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
}),
|
||||
).rejects.toThrow(/dimension safety limit.*40,001x1/i);
|
||||
|
||||
expect(mocks.autoOrient).not.toHaveBeenCalled();
|
||||
expect(mocks.decodeHeic).not.toHaveBeenCalled();
|
||||
expect(mocks.decodeToSharpCompat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a native image over the caller pixel cap before decoding", async () => {
|
||||
mocks.validateImageBuffer.mockResolvedValue({
|
||||
valid: true,
|
||||
format: "jpeg",
|
||||
width: 8_000,
|
||||
height: 6_000,
|
||||
});
|
||||
|
||||
await expect(
|
||||
new ImageInputHandler().prepare(RAW, "scan.jpg", {
|
||||
scratchDir: "/tmp/ocr",
|
||||
maxPixels: 40_000_000,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(InputValidationError);
|
||||
|
||||
expect(mocks.autoOrient).not.toHaveBeenCalled();
|
||||
expect(mocks.decodeToSharpCompat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes pixel and cancellation bounds into a CLI decoder and validates its output", async () => {
|
||||
mocks.validateImageBuffer.mockResolvedValue({
|
||||
valid: true,
|
||||
format: "raw",
|
||||
width: 0,
|
||||
height: 0,
|
||||
});
|
||||
mocks.metadata.mockResolvedValue({ width: 2_000, height: 1_500 });
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
await expect(
|
||||
new ImageInputHandler().prepare(RAW, "scan.nef", {
|
||||
scratchDir: "/tmp/ocr",
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 2_000_000,
|
||||
signal,
|
||||
}),
|
||||
).rejects.toThrow(/pixel safety limit/i);
|
||||
|
||||
expect(mocks.decodeToSharpCompat).toHaveBeenCalledWith(RAW, "raw", "nef", {
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 2_000_000,
|
||||
signal,
|
||||
});
|
||||
expect(mocks.autoOrient).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops before starting a CLI decoder when the request is already canceled", async () => {
|
||||
mocks.validateImageBuffer.mockResolvedValue({
|
||||
valid: true,
|
||||
format: "raw",
|
||||
width: 0,
|
||||
height: 0,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
new ImageInputHandler().prepare(RAW, "scan.dng", {
|
||||
scratchDir: "/tmp/ocr",
|
||||
maxPixels: 40_000_000,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
|
||||
expect(mocks.decodeToSharpCompat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes the same bounds to HEIF decoding and returns the normalized image", async () => {
|
||||
mocks.validateImageBuffer.mockResolvedValue({
|
||||
valid: true,
|
||||
format: "heif",
|
||||
width: 1_000,
|
||||
height: 1_000,
|
||||
});
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
const result = await new ImageInputHandler().prepare(RAW, "scan.heic", {
|
||||
scratchDir: "/tmp/ocr",
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
signal,
|
||||
});
|
||||
|
||||
expect(mocks.decodeHeic).toHaveBeenCalledWith(RAW, {
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
signal,
|
||||
});
|
||||
expect(result).toEqual({ buffer: ORIENTED, filename: "scan.png" });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,12 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const objectStorageMocks = vi.hoisted(() => ({
|
||||
copyObjectToFile: vi.fn(),
|
||||
getObjectBuffer: vi.fn(),
|
||||
getObjectSize: vi.fn(),
|
||||
putObject: vi.fn(),
|
||||
}));
|
||||
|
||||
async function loadWorker() {
|
||||
vi.resetModules();
|
||||
|
||||
@@ -13,6 +20,7 @@ async function loadWorker() {
|
||||
ANALYTICS_EVENTS: {},
|
||||
TOOLS: [],
|
||||
getBundleForTool: vi.fn(() => null),
|
||||
getOptionalBundleForTool: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
vi.doMock("bullmq", () => ({
|
||||
@@ -70,13 +78,13 @@ async function loadWorker() {
|
||||
}));
|
||||
|
||||
vi.doMock("../../../../apps/api/src/lib/object-storage.js", () => ({
|
||||
getObjectBuffer: vi.fn(),
|
||||
putObject: vi.fn(),
|
||||
...objectStorageMocks,
|
||||
}));
|
||||
|
||||
vi.doMock("../../../../apps/api/src/routes/progress.js", () => ({
|
||||
publishEphemeral: vi.fn(),
|
||||
updateSingleFileProgress: vi.fn(),
|
||||
updateSingleFileProgressAtomically: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.doMock("../../../../apps/api/src/routes/tool-factory.js", () => ({
|
||||
@@ -116,9 +124,84 @@ async function loadWorker() {
|
||||
|
||||
describe("worker result payload behavior", () => {
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("rejects an oversized OCR image object before buffering it", async () => {
|
||||
objectStorageMocks.getObjectSize.mockResolvedValueOnce(512 * 1024 * 1024 + 1);
|
||||
const { loadToolInputBuffer } = await loadWorker();
|
||||
|
||||
await expect(loadToolInputBuffer("ocr", "uploads/job-1/input.bin")).rejects.toMatchObject({
|
||||
name: "InputValidationError",
|
||||
statusCode: 413,
|
||||
});
|
||||
expect(objectStorageMocks.getObjectBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("maps an oversized streamed OCR PDF object to the OCR input limit", async () => {
|
||||
objectStorageMocks.copyObjectToFile.mockRejectedValueOnce(
|
||||
Object.assign(new Error("too large"), { statusCode: 413 }),
|
||||
);
|
||||
const { loadToolInputs } = await loadWorker();
|
||||
|
||||
await expect(
|
||||
loadToolInputs(
|
||||
"ocr-pdf",
|
||||
["uploads/job-1/scan.pdf"],
|
||||
"scan.pdf",
|
||||
"/tmp/job-1",
|
||||
new AbortController().signal,
|
||||
),
|
||||
).rejects.toMatchObject({ name: "InputValidationError", statusCode: 413 });
|
||||
expect(objectStorageMocks.getObjectBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("loads OCR PDF input as a bounded scratch path without buffering it", async () => {
|
||||
objectStorageMocks.copyObjectToFile.mockResolvedValueOnce(42);
|
||||
const controller = new AbortController();
|
||||
const { loadToolInputs } = await loadWorker();
|
||||
|
||||
await expect(
|
||||
loadToolInputs(
|
||||
"ocr-pdf",
|
||||
["uploads/job-1/scan.pdf"],
|
||||
"scan.pdf",
|
||||
"/tmp/job-1",
|
||||
controller.signal,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
inputs: [],
|
||||
pathInput: { path: "/tmp/job-1/input.pdf", size: 42 },
|
||||
originalSize: 42,
|
||||
});
|
||||
|
||||
expect(objectStorageMocks.copyObjectToFile).toHaveBeenCalledWith(
|
||||
"uploads/job-1/scan.pdf",
|
||||
"/tmp/job-1/input.pdf",
|
||||
{
|
||||
maxBytes: 512 * 1024 * 1024,
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
expect(objectStorageMocks.getObjectBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("loads OCR objects at the encoded-size boundary and leaves other tools unchanged", async () => {
|
||||
objectStorageMocks.getObjectSize.mockResolvedValueOnce(512 * 1024 * 1024);
|
||||
objectStorageMocks.getObjectBuffer.mockResolvedValue(Buffer.from("ocr"));
|
||||
const { loadToolInputBuffer } = await loadWorker();
|
||||
|
||||
await expect(loadToolInputBuffer("ocr", "uploads/job-1/scan.tiff")).resolves.toEqual(
|
||||
Buffer.from("ocr"),
|
||||
);
|
||||
await expect(loadToolInputBuffer("compress", "uploads/job-2/photo.png")).resolves.toEqual(
|
||||
Buffer.from("ocr"),
|
||||
);
|
||||
expect(objectStorageMocks.getObjectSize).toHaveBeenCalledTimes(1);
|
||||
expect(objectStorageMocks.getObjectBuffer).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("builds legacy download, preview, saved-file, and tool payload fields", async () => {
|
||||
const { buildLegacyResultPayload } = await loadWorker();
|
||||
|
||||
|
||||
@@ -128,4 +128,18 @@ describe("multipartParts", () => {
|
||||
|
||||
await generator.return(undefined);
|
||||
});
|
||||
|
||||
it("honors a route-specific two-file limit", async () => {
|
||||
const body = multipartBody([
|
||||
{ name: "index", filename: "ocr-runtime-index.json", content: "index" },
|
||||
{ name: "archive", filename: "ocr-runtime.tar.gz", content: "archive" },
|
||||
{ name: "extra", filename: "extra.bin", content: "unexpected" },
|
||||
]);
|
||||
|
||||
await expect(async () => {
|
||||
for await (const part of multipartParts(fakeRequest(body), { files: 2 })) {
|
||||
if (part.type === "file") await drain(part.file);
|
||||
}
|
||||
}).rejects.toThrow("files limit");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PassThrough, Readable } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const s3 = vi.hoisted(() => ({
|
||||
configure: vi.fn(),
|
||||
deleteObject: vi.fn(),
|
||||
getSize: vi.fn(),
|
||||
getStream: vi.fn(),
|
||||
putStream: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/config.js", () => ({
|
||||
env: {
|
||||
STORAGE_MODE: "s3",
|
||||
WORKSPACE_PATH: "/unused",
|
||||
S3_BUCKET: "test",
|
||||
S3_REGION: "us-east-1",
|
||||
S3_ENDPOINT: "",
|
||||
S3_ACCESS_KEY_ID: "test",
|
||||
S3_SECRET_ACCESS_KEY: "test",
|
||||
S3_FORCE_PATH_STYLE: true,
|
||||
S3_PREFIX: "",
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@snapotter/enterprise", () => ({
|
||||
loadS3Storage: vi.fn(async () => ({
|
||||
configureS3: s3.configure,
|
||||
deleteGenericObject: s3.deleteObject,
|
||||
getGenericObjectSize: s3.getSize,
|
||||
getGenericObjectStream: s3.getStream,
|
||||
putGenericObjectStream: s3.putStream,
|
||||
})),
|
||||
}));
|
||||
|
||||
import { copyObjectToFile, putObjectStream } from "../../../apps/api/src/lib/object-storage.js";
|
||||
|
||||
let scratchDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
s3.deleteObject.mockResolvedValue(undefined);
|
||||
scratchDir = mkdtempSync(join(tmpdir(), "snapotter-object-stream-copy-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(scratchDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("object-to-file streaming enforcement", () => {
|
||||
it("enforces the byte cap when the object stream is larger than its metadata", async () => {
|
||||
const destination = join(scratchDir, "oversized.pdf");
|
||||
s3.getSize.mockResolvedValueOnce(1);
|
||||
s3.getStream.mockResolvedValueOnce(
|
||||
Readable.from([Buffer.alloc(2, 0x41), Buffer.alloc(2, 0x42)]),
|
||||
);
|
||||
|
||||
await expect(
|
||||
copyObjectToFile("uploads/job-1/scan.pdf", destination, { maxBytes: 3 }),
|
||||
).rejects.toMatchObject({ statusCode: 413 });
|
||||
// Let a delayed write-stream open run. Cleanup must not be able to return
|
||||
// before an asynchronous open recreates the staging path.
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(s3.getStream).toHaveBeenCalledTimes(1);
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
expect(readdirSync(scratchDir)).toEqual([]);
|
||||
});
|
||||
|
||||
it("removes a partially-created S3 object when a streamed upload fails", async () => {
|
||||
s3.putStream.mockImplementationOnce(async (_key: string, source: AsyncIterable<Buffer>) => {
|
||||
for await (const _chunk of source) {
|
||||
throw new Error("multipart upload failed");
|
||||
}
|
||||
});
|
||||
|
||||
await expect(
|
||||
putObjectStream("uploads/job-3/scan.pdf", Readable.from([Buffer.from("partial")]), {
|
||||
maxBytes: 10,
|
||||
}),
|
||||
).rejects.toThrow("multipart upload failed");
|
||||
|
||||
expect(s3.deleteObject).toHaveBeenCalledWith("uploads/job-3/scan.pdf");
|
||||
});
|
||||
|
||||
it("cancels a stalled S3 upload and removes its partial object", async () => {
|
||||
let consumedFirstChunk!: () => void;
|
||||
const consumed = new Promise<void>((resolve) => {
|
||||
consumedFirstChunk = resolve;
|
||||
});
|
||||
s3.putStream.mockImplementationOnce(async (_key: string, source: AsyncIterable<Buffer>) => {
|
||||
for await (const _chunk of source) consumedFirstChunk();
|
||||
});
|
||||
const source = new PassThrough();
|
||||
const controller = new AbortController();
|
||||
const uploading = putObjectStream("uploads/job-4/scan.pdf", source, {
|
||||
maxBytes: 10,
|
||||
signal: controller.signal,
|
||||
});
|
||||
source.write(Buffer.from("partial"));
|
||||
await consumed;
|
||||
expect(s3.putStream).toHaveBeenCalledWith(
|
||||
"uploads/job-4/scan.pdf",
|
||||
expect.anything(),
|
||||
controller.signal,
|
||||
);
|
||||
controller.abort();
|
||||
|
||||
await expect(uploading).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(s3.deleteObject).toHaveBeenCalledWith("uploads/job-4/scan.pdf");
|
||||
});
|
||||
|
||||
it("aborts an in-flight object stream and removes its staging file", async () => {
|
||||
const destination = join(scratchDir, "canceled.pdf");
|
||||
const controller = new AbortController();
|
||||
let firstChunkConsumed!: () => void;
|
||||
let resumeSource!: () => void;
|
||||
const consumed = new Promise<void>((resolve) => {
|
||||
firstChunkConsumed = resolve;
|
||||
});
|
||||
const resume = new Promise<void>((resolve) => {
|
||||
resumeSource = resolve;
|
||||
});
|
||||
s3.getSize.mockResolvedValueOnce(2);
|
||||
s3.getStream.mockResolvedValueOnce(
|
||||
Readable.from(
|
||||
(async function* () {
|
||||
yield Buffer.from("a");
|
||||
firstChunkConsumed();
|
||||
await resume;
|
||||
yield Buffer.from("b");
|
||||
})(),
|
||||
),
|
||||
);
|
||||
|
||||
const copying = copyObjectToFile("uploads/job-2/scan.pdf", destination, {
|
||||
maxBytes: 2,
|
||||
signal: controller.signal,
|
||||
});
|
||||
await consumed;
|
||||
controller.abort();
|
||||
resumeSource();
|
||||
|
||||
await expect(copying).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
expect(readdirSync(scratchDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,10 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
copyObjectToFile,
|
||||
deleteObject,
|
||||
getObjectSize,
|
||||
getObjectStream,
|
||||
@@ -12,9 +16,15 @@ import {
|
||||
|
||||
describe("object-storage (local backend)", () => {
|
||||
const key = `outputs/test-${process.pid}/hello.txt`;
|
||||
const copyKey = `outputs/test-${process.pid}/copy-source.bin`;
|
||||
const unavailableKey = `outputs/test-${process.pid}/unavailable.bin`;
|
||||
const copyDir = mkdtempSync(join(tmpdir(), "snapotter-object-copy-"));
|
||||
|
||||
afterAll(async () => {
|
||||
await deleteObject(key).catch(() => {});
|
||||
await deleteObject(copyKey).catch(() => {});
|
||||
await deleteObject(unavailableKey).catch(() => {});
|
||||
rmSync(copyDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("round-trips buffers and streams with size and listing", async () => {
|
||||
@@ -52,4 +62,66 @@ describe("object-storage (local backend)", () => {
|
||||
/invalid/i,
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies operational streaming-write failures as temporary storage outages", async () => {
|
||||
const source = Readable.from(
|
||||
(async function* () {
|
||||
yield Buffer.from("partial");
|
||||
throw Object.assign(new Error("disk quota exhausted"), { code: "EDQUOT" });
|
||||
})(),
|
||||
);
|
||||
|
||||
await expect(putObjectStream(unavailableKey, source)).rejects.toMatchObject({
|
||||
code: "EDQUOT",
|
||||
statusCode: 503,
|
||||
});
|
||||
await expect(objectExists(unavailableKey)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("streams an object to a file without exceeding the hard byte cap", async () => {
|
||||
const source = Buffer.alloc(4096, 0x5a);
|
||||
const destination = join(copyDir, "bounded.bin");
|
||||
await putObject(copyKey, source);
|
||||
|
||||
await expect(copyObjectToFile(copyKey, destination, { maxBytes: source.length })).resolves.toBe(
|
||||
source.length,
|
||||
);
|
||||
expect(readFileSync(destination)).toEqual(source);
|
||||
});
|
||||
|
||||
it("removes a partial destination when the streamed object exceeds its cap", async () => {
|
||||
const destination = join(copyDir, "oversized.bin");
|
||||
await putObject(copyKey, Buffer.alloc(4096, 0x41));
|
||||
|
||||
await expect(copyObjectToFile(copyKey, destination, { maxBytes: 2048 })).rejects.toMatchObject({
|
||||
statusCode: 413,
|
||||
});
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not leave a destination behind when copying is canceled", async () => {
|
||||
const destination = join(copyDir, "canceled.bin");
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
copyObjectToFile(copyKey, destination, {
|
||||
maxBytes: 4096,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
});
|
||||
|
||||
it("atomically replaces a stale destination left by a crashed attempt", async () => {
|
||||
const destination = join(copyDir, "stale-retry.bin");
|
||||
const source = Buffer.from("fresh object bytes");
|
||||
writeFileSync(destination, "stale partial bytes");
|
||||
await putObject(copyKey, source);
|
||||
|
||||
await expect(copyObjectToFile(copyKey, destination, { maxBytes: source.length })).resolves.toBe(
|
||||
source.length,
|
||||
);
|
||||
expect(readFileSync(destination)).toEqual(source);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
import type { OcrRuntimeCapability } from "@snapotter/ai";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveOcrIngressSettings } from "../../../apps/api/src/lib/ocr-capability.js";
|
||||
|
||||
const missingCapability: OcrRuntimeCapability = {
|
||||
available: false,
|
||||
status: "missing",
|
||||
reason: "descriptor-missing",
|
||||
qualities: [],
|
||||
providers: [],
|
||||
};
|
||||
|
||||
const incompatibleCapability: OcrRuntimeCapability = {
|
||||
available: false,
|
||||
status: "invalid",
|
||||
reason: "descriptor-invalid",
|
||||
qualities: [],
|
||||
providers: [],
|
||||
};
|
||||
|
||||
function readyCapability(qualities: readonly ("balanced" | "best")[]): OcrRuntimeCapability {
|
||||
return {
|
||||
available: true,
|
||||
status: "ready",
|
||||
qualities,
|
||||
providers: ["CPUExecutionProvider"],
|
||||
descriptor: {} as never,
|
||||
};
|
||||
}
|
||||
|
||||
describe("resolveOcrIngressSettings", () => {
|
||||
it("leaves non-OCR tool settings alone without inspecting OCR capability", () => {
|
||||
const settings = { quality: 80 };
|
||||
const getCapability = vi.fn(() => missingCapability);
|
||||
|
||||
expect(
|
||||
resolveOcrIngressSettings("compress", settings, { readCapability: getCapability }),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings,
|
||||
});
|
||||
expect(getCapability).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["ocr", "ocr-pdf"])("admits an explicit Fast tier for non-Korean %s", (toolId) => {
|
||||
const getCapability = vi.fn(() => incompatibleCapability);
|
||||
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
toolId,
|
||||
{ quality: "fast", language: "en" },
|
||||
{ readCapability: getCapability },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { quality: "fast", language: "en" },
|
||||
});
|
||||
expect(getCapability).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
"ocr",
|
||||
"ocr-pdf",
|
||||
])("rejects explicit Fast Korean %s before reading or queueing an accurate runtime", (toolId) => {
|
||||
const getCapability = vi.fn(() => readyCapability(["balanced", "best"]));
|
||||
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
toolId,
|
||||
{ quality: "fast", language: "ko" },
|
||||
{ readCapability: getCapability },
|
||||
),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "fast-korean-unsupported",
|
||||
requestedQuality: "fast",
|
||||
guidance:
|
||||
"Fast OCR does not support Korean. Install the Accurate OCR bundle and choose Balanced or Best.",
|
||||
});
|
||||
expect(getCapability).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves an omitted tier to Best when the healthy runtime supports it", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ language: "auto" },
|
||||
{
|
||||
readCapability: () => readyCapability(["balanced", "best"]),
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "auto", quality: "best" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves an omitted non-Korean tier to Balanced when it is the best available tier", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr-pdf",
|
||||
{ language: "en", pages: "all" },
|
||||
{ readCapability: () => readyCapability(["balanced"]) },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "en", pages: "all", quality: "balanced" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves an omitted tier to Fast when no healthy runtime is active", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr-pdf",
|
||||
{ pages: "all" },
|
||||
{
|
||||
readCapability: () => missingCapability,
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { pages: "all", quality: "fast" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves omitted Korean to Best when Best is available", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ language: "ko" },
|
||||
{ readCapability: () => readyCapability(["balanced", "best"]) },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "ko", quality: "best" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves omitted Korean to Balanced when it is the available accurate tier", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr-pdf",
|
||||
{ language: "ko", pages: "all" },
|
||||
{ readCapability: () => readyCapability(["balanced"]) },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "ko", pages: "all", quality: "balanced" },
|
||||
});
|
||||
});
|
||||
|
||||
it("pins omitted Korean to an accurate tier when the pack is missing", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ language: "ko" },
|
||||
{ readCapability: () => missingCapability },
|
||||
),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
code: "FEATURE_NOT_INSTALLED",
|
||||
reason: "descriptor-missing",
|
||||
requestedQuality: "best",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the request to detect omission even if a schema supplied a quality default", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ quality: "balanced", language: "auto" },
|
||||
{
|
||||
requestedSettings: { language: "auto" },
|
||||
readCapability: () => readyCapability(["balanced", "best"]),
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { quality: "best", language: "auto" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["tesseract", "fast"],
|
||||
["paddleocr", "balanced"],
|
||||
] as const)("maps legacy engine %s only when quality is absent", (engine, quality) => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ engine, language: "en" },
|
||||
{
|
||||
readCapability: () => readyCapability(["balanced", "best"]),
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "en", quality },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects legacy Tesseract for Korean like explicit Fast", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ engine: "tesseract", language: "ko" },
|
||||
{ readCapability: () => readyCapability(["balanced", "best"]) },
|
||||
),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "fast-korean-unsupported",
|
||||
requestedQuality: "fast",
|
||||
});
|
||||
});
|
||||
|
||||
it("maps legacy PaddleOCR Korean requests to Balanced normally", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr-pdf",
|
||||
{ engine: "paddleocr", language: "ko", pages: "all" },
|
||||
{ readCapability: () => readyCapability(["balanced", "best"]) },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { language: "ko", pages: "all", quality: "balanced" },
|
||||
});
|
||||
});
|
||||
|
||||
it("lets an explicit quality override the legacy engine and removes the legacy field", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ quality: "best", engine: "tesseract", language: "en" },
|
||||
{ readCapability: () => readyCapability(["balanced", "best"]) },
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
settings: { quality: "best", language: "en" },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a missing accurate runtime as FEATURE_NOT_INSTALLED", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ quality: "balanced" },
|
||||
{
|
||||
readCapability: () => missingCapability,
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
code: "FEATURE_NOT_INSTALLED",
|
||||
reason: "descriptor-missing",
|
||||
requestedQuality: "balanced",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports an invalid runtime as FEATURE_INCOMPATIBLE with its reason", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr-pdf",
|
||||
{ quality: "best" },
|
||||
{
|
||||
readCapability: () => incompatibleCapability,
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "descriptor-invalid",
|
||||
requestedQuality: "best",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a healthy runtime missing the requested tier as incompatible", () => {
|
||||
expect(
|
||||
resolveOcrIngressSettings(
|
||||
"ocr",
|
||||
{ quality: "best" },
|
||||
{
|
||||
readCapability: () => readyCapability(["balanced"]),
|
||||
},
|
||||
),
|
||||
).toEqual({
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "quality-not-supported",
|
||||
requestedQuality: "best",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({ prepare: vi.fn() }));
|
||||
|
||||
vi.mock("@snapotter/ai", () => ({
|
||||
MAX_OCR_INPUT_DIMENSION: 40_000,
|
||||
MAX_OCR_INPUT_PIXELS: 40_000_000,
|
||||
}));
|
||||
vi.mock("../../../apps/api/src/modality/input-handler.js", () => ({
|
||||
inputHandlerFor: () => ({ prepare: mocks.prepare }),
|
||||
}));
|
||||
|
||||
import { prepareOcrIngressImage } from "../../../apps/api/src/lib/ocr-image-input.js";
|
||||
|
||||
describe("prepareOcrIngressImage", () => {
|
||||
beforeEach(() => {
|
||||
mocks.prepare.mockReset();
|
||||
mocks.prepare.mockResolvedValue({ buffer: Buffer.from("prepared"), filename: "scan.png" });
|
||||
});
|
||||
|
||||
it("uses the shared image handler with both OCR decoded-dimension ceilings", async () => {
|
||||
const raw = Buffer.from("raw");
|
||||
|
||||
await expect(prepareOcrIngressImage(raw, "scan.qoi", "/tmp/ocr-ingress")).resolves.toEqual({
|
||||
buffer: Buffer.from("prepared"),
|
||||
filename: "scan.png",
|
||||
});
|
||||
expect(mocks.prepare).toHaveBeenCalledWith(raw, "scan.qoi", {
|
||||
scratchDir: "/tmp/ocr-ingress",
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,333 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
deleteObject: vi.fn(),
|
||||
enqueueToolJob: vi.fn(),
|
||||
extractText: vi.fn(),
|
||||
getAuthUser: vi.fn(),
|
||||
getOcrRuntimeCapability: vi.fn(),
|
||||
prepare: vi.fn(),
|
||||
receiveUpload: vi.fn(),
|
||||
waitForJob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@snapotter/ai", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@snapotter/ai")>();
|
||||
return {
|
||||
...actual,
|
||||
extractText: mocks.extractText,
|
||||
getOcrRuntimeCapability: mocks.getOcrRuntimeCapability,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../../../apps/api/src/modality/input-handler.js", () => ({
|
||||
inputHandlerFor: () => ({ prepare: mocks.prepare }),
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/jobs/enqueue.js", () => ({
|
||||
enqueueToolJob: mocks.enqueueToolJob,
|
||||
waitForJob: mocks.waitForJob,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/object-storage.js", () => ({
|
||||
deleteObject: mocks.deleteObject,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/upload-stream.js", () => ({
|
||||
receiveUpload: mocks.receiveUpload,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/plugins/auth.js", () => ({
|
||||
getAuthUser: mocks.getAuthUser,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/permissions.js", () => ({
|
||||
requireToolAccess: vi.fn(async () => ({ id: "user-1" })),
|
||||
}));
|
||||
|
||||
import { env } from "../../../apps/api/src/config.js";
|
||||
import { runAiToolJob } from "../../../apps/api/src/jobs/ai-handlers.js";
|
||||
import type { ToolJobData } from "../../../apps/api/src/jobs/types.js";
|
||||
import type { ToolProcessCtx } from "../../../apps/api/src/routes/tool-factory.js";
|
||||
import { registerOcr } from "../../../apps/api/src/routes/tools/ocr.js";
|
||||
|
||||
const INPUT = Buffer.from("uploaded");
|
||||
const NORMALIZED = Buffer.from("normalized");
|
||||
|
||||
function job(): ToolJobData {
|
||||
return {
|
||||
jobId: "ocr-job",
|
||||
toolId: "ocr",
|
||||
userId: null,
|
||||
pool: "ai",
|
||||
inputRefs: ["uploads/ocr-job/scan.heic"],
|
||||
filename: "scan.heic",
|
||||
settings: { quality: "fast", language: "en", enhance: true },
|
||||
kind: "ai-tool",
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.getOcrRuntimeCapability.mockReturnValue({
|
||||
available: true,
|
||||
qualities: ["balanced", "best"],
|
||||
providers: ["CPUExecutionProvider"],
|
||||
});
|
||||
mocks.getAuthUser.mockReturnValue({ id: "user-1" });
|
||||
mocks.deleteObject.mockResolvedValue(undefined);
|
||||
mocks.enqueueToolJob.mockResolvedValue(undefined);
|
||||
mocks.waitForJob.mockResolvedValue(null);
|
||||
mocks.receiveUpload.mockResolvedValue({
|
||||
key: "uploads/ocr-job/scan.png",
|
||||
filename: "scan.png",
|
||||
size: 5,
|
||||
});
|
||||
mocks.prepare.mockResolvedValue({ buffer: NORMALIZED, filename: "scan.png" });
|
||||
mocks.extractText.mockResolvedValue({
|
||||
text: "SnapOtter",
|
||||
engine: "tesseract",
|
||||
requestedQuality: "fast",
|
||||
actualQuality: "fast",
|
||||
device: "cpu",
|
||||
provider: "tesseract",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
});
|
||||
});
|
||||
|
||||
describe("OCR route enqueue safety", () => {
|
||||
it("acknowledges long-running OCR without entering the synchronous wait window", async () => {
|
||||
let routeHandler: ((request: unknown, reply: unknown) => Promise<unknown>) | undefined;
|
||||
registerOcr({
|
||||
post: vi.fn((_path, handler) => {
|
||||
routeHandler = handler;
|
||||
}),
|
||||
} as never);
|
||||
|
||||
const request = {
|
||||
headers: {},
|
||||
log: { error: vi.fn(), info: vi.fn() },
|
||||
parts: async function* () {
|
||||
yield { type: "file", filename: "scan.png", file: {} };
|
||||
yield { type: "field", fieldname: "settings", value: '{"quality":"fast"}' };
|
||||
},
|
||||
};
|
||||
const reply = {
|
||||
statusCode: 200,
|
||||
payload: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
send(payload: unknown) {
|
||||
this.payload = payload;
|
||||
return payload;
|
||||
},
|
||||
};
|
||||
|
||||
await expect(routeHandler?.(request, reply)).resolves.toBeDefined();
|
||||
|
||||
expect(reply.statusCode).toBe(202);
|
||||
expect(reply.payload).toMatchObject({
|
||||
jobId: expect.any(String),
|
||||
async: true,
|
||||
});
|
||||
expect(mocks.enqueueToolJob).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ jobId: expect.any(String), toolId: "ocr", pool: "ai" }),
|
||||
);
|
||||
expect(mocks.waitForJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves storage-pressure failures as a truthful 503 response", async () => {
|
||||
mocks.receiveUpload.mockRejectedValueOnce(
|
||||
Object.assign(new Error("Upload storage is below its free-space reserve"), {
|
||||
statusCode: 503,
|
||||
}),
|
||||
);
|
||||
|
||||
let routeHandler: ((request: unknown, reply: unknown) => Promise<unknown>) | undefined;
|
||||
registerOcr({
|
||||
post: vi.fn((_path, handler) => {
|
||||
routeHandler = handler;
|
||||
}),
|
||||
} as never);
|
||||
|
||||
const request = {
|
||||
headers: {},
|
||||
log: { error: vi.fn(), info: vi.fn() },
|
||||
parts: async function* () {
|
||||
yield { type: "file", filename: "scan.png", file: {} };
|
||||
},
|
||||
};
|
||||
const reply = {
|
||||
statusCode: 200,
|
||||
payload: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
send(payload: unknown) {
|
||||
this.payload = payload;
|
||||
return payload;
|
||||
},
|
||||
};
|
||||
|
||||
await expect(routeHandler?.(request, reply)).resolves.toBeDefined();
|
||||
|
||||
expect(reply.statusCode).toBe(503);
|
||||
expect(reply.payload).toMatchObject({
|
||||
error: "Upload storage unavailable",
|
||||
details: "Upload storage is below its free-space reserve",
|
||||
});
|
||||
});
|
||||
|
||||
it("cleans up a bounded upload when enqueueing fails", async () => {
|
||||
const originalLimit = env.MAX_UPLOAD_SIZE_MB;
|
||||
env.MAX_UPLOAD_SIZE_MB = 0;
|
||||
mocks.enqueueToolJob.mockRejectedValueOnce(new Error("Redis unavailable"));
|
||||
|
||||
let routeHandler: ((request: unknown, reply: unknown) => Promise<unknown>) | undefined;
|
||||
registerOcr({
|
||||
post: vi.fn((_path, handler) => {
|
||||
routeHandler = handler;
|
||||
}),
|
||||
} as never);
|
||||
|
||||
const filePart = { type: "file", filename: "scan.png", file: {} };
|
||||
const request = {
|
||||
headers: {},
|
||||
log: { error: vi.fn(), info: vi.fn() },
|
||||
parts: async function* () {
|
||||
yield filePart;
|
||||
yield { type: "field", fieldname: "settings", value: '{"quality":"fast"}' };
|
||||
},
|
||||
};
|
||||
const reply = {
|
||||
statusCode: 200,
|
||||
payload: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
send(payload: unknown) {
|
||||
this.payload = payload;
|
||||
return payload;
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await expect(routeHandler?.(request, reply)).resolves.toBeDefined();
|
||||
} finally {
|
||||
env.MAX_UPLOAD_SIZE_MB = originalLimit;
|
||||
}
|
||||
|
||||
expect(mocks.receiveUpload).toHaveBeenCalledWith(filePart, expect.any(String), {
|
||||
maxBytes: 512 * 1024 * 1024,
|
||||
});
|
||||
expect(mocks.deleteObject).toHaveBeenCalledWith("uploads/ocr-job/scan.png");
|
||||
expect(reply.statusCode).toBe(503);
|
||||
expect(reply.payload).toMatchObject({ error: "Failed to queue OCR" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("OCR AI job handler resource controls", () => {
|
||||
it.each([
|
||||
["balanced", false],
|
||||
["best", true],
|
||||
] as const)("defaults %s enhancement consistently with the UI", async (quality, enhance) => {
|
||||
const data = job();
|
||||
data.settings = { quality, language: "en" };
|
||||
mocks.extractText.mockResolvedValueOnce({
|
||||
text: "SnapOtter",
|
||||
engine: "rapidocr-onnx",
|
||||
requestedQuality: quality,
|
||||
actualQuality: quality,
|
||||
device: "cpu",
|
||||
provider: "CPUExecutionProvider",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
await runAiToolJob(data, INPUT, {
|
||||
scratchDir: "/tmp/ocr-job",
|
||||
signal: new AbortController().signal,
|
||||
report: vi.fn(),
|
||||
});
|
||||
|
||||
expect(mocks.extractText).toHaveBeenCalledWith(
|
||||
NORMALIZED,
|
||||
"/tmp/ocr-job",
|
||||
expect.objectContaining({ quality, enhance }),
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes inside the worker with the OCR pixel cap and job signal", async () => {
|
||||
const controller = new AbortController();
|
||||
const ctx: ToolProcessCtx = {
|
||||
scratchDir: "/tmp/ocr-job",
|
||||
signal: controller.signal,
|
||||
report: vi.fn(),
|
||||
};
|
||||
mocks.extractText.mockImplementationOnce(async (_input, _scratch, _options, report) => {
|
||||
report(0, "starting");
|
||||
report(50, "recognizing");
|
||||
report(100, "complete");
|
||||
return {
|
||||
text: "SnapOtter",
|
||||
engine: "tesseract",
|
||||
requestedQuality: "fast",
|
||||
actualQuality: "fast",
|
||||
device: "cpu",
|
||||
provider: "tesseract",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
};
|
||||
});
|
||||
|
||||
const result = await runAiToolJob(job(), INPUT, ctx);
|
||||
|
||||
expect(mocks.prepare).toHaveBeenCalledWith(INPUT, "scan.heic", {
|
||||
scratchDir: "/tmp/ocr-job",
|
||||
maxDimension: 40_000,
|
||||
maxPixels: 40_000_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect(mocks.extractText).toHaveBeenCalledWith(
|
||||
NORMALIZED,
|
||||
"/tmp/ocr-job",
|
||||
expect.objectContaining({
|
||||
quality: "fast",
|
||||
language: "en",
|
||||
enhance: true,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
filename: "scan_ocr.txt",
|
||||
contentType: "text/plain",
|
||||
resultPayload: { text: "SnapOtter", actualQuality: "fast" },
|
||||
});
|
||||
expect(vi.mocked(ctx.report).mock.calls.map(([percent]) => percent)).toEqual([
|
||||
2, 10, 10, 55, 100,
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not start image preparation for an already-canceled job", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
runAiToolJob(job(), INPUT, {
|
||||
scratchDir: "/tmp/ocr-job",
|
||||
signal: controller.signal,
|
||||
report: vi.fn(),
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
|
||||
expect(mocks.prepare).not.toHaveBeenCalled();
|
||||
expect(mocks.extractText).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
findOcrEncodedInputViolation,
|
||||
OCR_MAX_BATCH_ENCODED_INPUT_BYTES,
|
||||
OCR_MAX_ENCODED_INPUT_BYTES,
|
||||
ocrUploadErrorMessage,
|
||||
ocrUploadErrorStatus,
|
||||
resolveOcrEncodedInputLimit,
|
||||
resolveOcrUploadLimits,
|
||||
} from "../../../apps/api/src/lib/ocr-limits.js";
|
||||
|
||||
describe("OCR ingress limits", () => {
|
||||
it("keeps a hard encoded ceiling when the global upload limit is unlimited or larger", () => {
|
||||
expect(resolveOcrEncodedInputLimit(0)).toBe(OCR_MAX_ENCODED_INPUT_BYTES);
|
||||
expect(resolveOcrEncodedInputLimit(1024)).toBe(OCR_MAX_ENCODED_INPUT_BYTES);
|
||||
});
|
||||
|
||||
it("honors a smaller configured global upload limit", () => {
|
||||
expect(resolveOcrEncodedInputLimit(100)).toBe(100 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("keeps the operator limit per file while reserving the hard ceiling for the aggregate", () => {
|
||||
expect(resolveOcrUploadLimits(100)).toEqual({
|
||||
fileBytes: 100 * 1024 * 1024,
|
||||
aggregateBytes: OCR_MAX_BATCH_ENCODED_INPUT_BYTES,
|
||||
});
|
||||
expect(OCR_MAX_BATCH_ENCODED_INPUT_BYTES).toBe(OCR_MAX_ENCODED_INPUT_BYTES);
|
||||
});
|
||||
|
||||
it("allows multiple individually valid OCR inputs below the independent aggregate ceiling", () => {
|
||||
const mib = 1024 * 1024;
|
||||
expect(findOcrEncodedInputViolation([60 * mib, 60 * mib], 100)).toBeNull();
|
||||
});
|
||||
|
||||
it("distinguishes per-file and aggregate OCR input violations", () => {
|
||||
const mib = 1024 * 1024;
|
||||
expect(findOcrEncodedInputViolation([101 * mib], 100)).toEqual({
|
||||
scope: "file",
|
||||
limitBytes: 100 * mib,
|
||||
});
|
||||
expect(
|
||||
findOcrEncodedInputViolation(
|
||||
Array.from({ length: 9 }, () => 60 * mib),
|
||||
100,
|
||||
),
|
||||
).toEqual({
|
||||
scope: "aggregate",
|
||||
limitBytes: OCR_MAX_BATCH_ENCODED_INPUT_BYTES,
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies both multipart and streaming size failures as payload-too-large", () => {
|
||||
expect(ocrUploadErrorStatus({ statusCode: 413 })).toBe(413);
|
||||
expect(ocrUploadErrorStatus(new Error("Upload exceeds the maximum allowed size"))).toBe(413);
|
||||
expect(ocrUploadErrorStatus(new Error("malformed multipart"))).toBe(400);
|
||||
});
|
||||
|
||||
it("preserves service-unavailable storage failures with a truthful client message", () => {
|
||||
expect(ocrUploadErrorStatus({ statusCode: 503 })).toBe(503);
|
||||
expect(ocrUploadErrorMessage(503)).toBe("Upload storage unavailable");
|
||||
expect(ocrUploadErrorMessage(413)).toBe("Upload exceeds the allowed size");
|
||||
expect(ocrUploadErrorMessage(400)).toBe("Failed to parse multipart request");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const batchSource = readFileSync(resolve(root, "apps/api/src/routes/batch.ts"), "utf8");
|
||||
const pipelineSource = readFileSync(resolve(root, "apps/api/src/routes/pipeline.ts"), "utf8");
|
||||
|
||||
function between(source: string, start: string, end?: string): string {
|
||||
const startIndex = source.indexOf(start);
|
||||
const endIndex = end ? source.indexOf(end, startIndex + start.length) : source.length;
|
||||
expect(startIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(endIndex).toBeGreaterThan(startIndex);
|
||||
return source.slice(startIndex, endIndex);
|
||||
}
|
||||
|
||||
describe("OCR PDF route streaming contract", () => {
|
||||
it("keeps the tool batch OCR-PDF branch path-backed while retaining buffered non-OCR files", () => {
|
||||
expect(batchSource).toMatch(
|
||||
/if \(toolId === "ocr-pdf"\)[\s\S]*spoolMultipartFile[\s\S]*else \{[\s\S]*Buffer\.concat\(chunks\)/,
|
||||
);
|
||||
expect(batchSource).toContain("storeValidatedOcrPdf");
|
||||
});
|
||||
|
||||
it("spools execute-pipeline multipart input and never buffers OCR-PDF validation", () => {
|
||||
const executeRoute = between(
|
||||
pipelineSource,
|
||||
'"/api/v1/pipeline/execute"',
|
||||
'"/api/v1/pipeline/save"',
|
||||
);
|
||||
expect(executeRoute).toContain("spoolMultipartFile");
|
||||
expect(executeRoute).not.toContain("Buffer.concat(chunks)");
|
||||
expect(executeRoute).toContain("storeValidatedOcrPdf");
|
||||
expect(executeRoute).not.toMatch(
|
||||
/inputHandlerFor\([\s\S]{0,300}rejectPasswordProtected: firstToolId === "ocr-pdf"/,
|
||||
);
|
||||
});
|
||||
|
||||
it("spools batch-pipeline multipart inputs and takes the OCR-PDF path branch", () => {
|
||||
const batchRoute = between(pipelineSource, '"/api/v1/pipeline/batch"');
|
||||
expect(batchRoute).toContain("spoolMultipartFile");
|
||||
expect(batchRoute).not.toContain("Buffer.concat(chunks)");
|
||||
expect(batchRoute).toContain("storeValidatedOcrPdf");
|
||||
expect(batchRoute).not.toMatch(/inputHandlerFor\([\s\S]{0,300}resolvedToolId === "ocr-pdf"/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
copyObjectToFile: vi.fn(),
|
||||
deleteObject: vi.fn(),
|
||||
enqueueToolJob: vi.fn(),
|
||||
extractPdfText: vi.fn(),
|
||||
getAuthUser: vi.fn(),
|
||||
getObjectBuffer: vi.fn(),
|
||||
getOcrRuntimeCapability: vi.fn(),
|
||||
receiveUpload: vi.fn(),
|
||||
validatePdfPath: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@snapotter/ai", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@snapotter/ai")>();
|
||||
return {
|
||||
...actual,
|
||||
extractPdfText: mocks.extractPdfText,
|
||||
getOcrRuntimeCapability: mocks.getOcrRuntimeCapability,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../../../apps/api/src/jobs/enqueue.js", () => ({
|
||||
enqueueToolJob: mocks.enqueueToolJob,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/object-storage.js", () => ({
|
||||
copyObjectToFile: mocks.copyObjectToFile,
|
||||
deleteObject: mocks.deleteObject,
|
||||
getObjectBuffer: mocks.getObjectBuffer,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/lib/upload-stream.js", () => ({
|
||||
receiveUpload: mocks.receiveUpload,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/modality/document-input.js", () => ({
|
||||
DocumentInputHandler: class DocumentInputHandler {},
|
||||
validatePdfPath: mocks.validatePdfPath,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/plugins/auth.js", () => ({
|
||||
getAuthUser: mocks.getAuthUser,
|
||||
}));
|
||||
|
||||
vi.mock("../../../apps/api/src/permissions.js", () => ({
|
||||
requireToolAccess: vi.fn(async () => ({ id: "user-1" })),
|
||||
}));
|
||||
|
||||
import { env } from "../../../apps/api/src/config.js";
|
||||
import { type AiPathJobInput, runAiPathToolJob } from "../../../apps/api/src/jobs/ai-handlers.js";
|
||||
import type { ToolJobData } from "../../../apps/api/src/jobs/types.js";
|
||||
import { resolveOcrEncodedInputLimit } from "../../../apps/api/src/lib/ocr-limits.js";
|
||||
import type { ToolProcessCtx } from "../../../apps/api/src/routes/tool-factory.js";
|
||||
import { registerOcrPdf } from "../../../apps/api/src/routes/tools/ocr-pdf.js";
|
||||
|
||||
function job(): ToolJobData {
|
||||
return {
|
||||
jobId: "ocr-pdf-job",
|
||||
toolId: "ocr-pdf",
|
||||
userId: null,
|
||||
pool: "ai",
|
||||
inputRefs: ["uploads/ocr-pdf-job/scan.pdf"],
|
||||
filename: "scan.pdf",
|
||||
settings: { quality: "fast", language: "en", pages: "1" },
|
||||
kind: "ai-tool",
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.copyObjectToFile.mockResolvedValue(42);
|
||||
mocks.deleteObject.mockResolvedValue(undefined);
|
||||
mocks.enqueueToolJob.mockResolvedValue(undefined);
|
||||
mocks.getAuthUser.mockReturnValue({ id: "user-1" });
|
||||
mocks.getOcrRuntimeCapability.mockReturnValue({
|
||||
available: false,
|
||||
qualities: [],
|
||||
providers: [],
|
||||
});
|
||||
mocks.receiveUpload.mockResolvedValue({
|
||||
key: "uploads/ocr-pdf-job/scan.pdf",
|
||||
filename: "scan.pdf",
|
||||
size: 42,
|
||||
});
|
||||
mocks.validatePdfPath.mockResolvedValue(undefined);
|
||||
mocks.extractPdfText.mockResolvedValue({
|
||||
text: "SnapOtter",
|
||||
pages: 1,
|
||||
engine: "tesseract",
|
||||
requestedQuality: "fast",
|
||||
actualQuality: "fast",
|
||||
device: "cpu",
|
||||
provider: "tesseract",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
});
|
||||
});
|
||||
|
||||
describe("OCR PDF path-backed processing", () => {
|
||||
it("validates and processes the worker scratch path without reading the PDF into memory", async () => {
|
||||
const controller = new AbortController();
|
||||
const ctx: ToolProcessCtx = {
|
||||
scratchDir: "/tmp/ocr-pdf-job",
|
||||
signal: controller.signal,
|
||||
report: vi.fn(),
|
||||
};
|
||||
const input: AiPathJobInput = { path: "/tmp/ocr-pdf-job/input.pdf", size: 42 };
|
||||
|
||||
const result = await runAiPathToolJob(job(), input, ctx);
|
||||
|
||||
expect(mocks.validatePdfPath).toHaveBeenCalledWith(input.path, {
|
||||
rejectPasswordProtected: true,
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect(mocks.extractPdfText).toHaveBeenCalledWith(
|
||||
input.path,
|
||||
expect.objectContaining({
|
||||
quality: "fast",
|
||||
language: "en",
|
||||
pages: "1",
|
||||
signal: controller.signal,
|
||||
}),
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(mocks.getObjectBuffer).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
buffer: Buffer.from("SnapOtter"),
|
||||
filename: "scan_ocr.txt",
|
||||
contentType: "text/plain",
|
||||
});
|
||||
});
|
||||
|
||||
it("streams route validation to scratch with a hard cap and removes its scratch directory", async () => {
|
||||
let routeHandler: ((request: unknown, reply: unknown) => Promise<unknown>) | undefined;
|
||||
registerOcrPdf({
|
||||
post: vi.fn((_path, handler) => {
|
||||
routeHandler = handler;
|
||||
}),
|
||||
} as never);
|
||||
|
||||
const request = {
|
||||
headers: {},
|
||||
log: { error: vi.fn() },
|
||||
raw: {
|
||||
aborted: false,
|
||||
once: vi.fn(),
|
||||
removeListener: vi.fn(),
|
||||
},
|
||||
parts: async function* () {
|
||||
yield { type: "file", filename: "scan.pdf", file: {} };
|
||||
yield { type: "field", fieldname: "settings", value: '{"quality":"fast"}' };
|
||||
},
|
||||
};
|
||||
const reply = {
|
||||
statusCode: 200,
|
||||
payload: undefined as unknown,
|
||||
status(code: number) {
|
||||
this.statusCode = code;
|
||||
return this;
|
||||
},
|
||||
send(payload: unknown) {
|
||||
this.payload = payload;
|
||||
return payload;
|
||||
},
|
||||
};
|
||||
|
||||
const originalScratchPath = env.SCRATCH_PATH;
|
||||
const configuredScratchRoot = join(tmpdir(), `snapotter-ocr-pdf-test-${Date.now()}`);
|
||||
env.SCRATCH_PATH = configuredScratchRoot;
|
||||
try {
|
||||
await routeHandler?.(request, reply);
|
||||
} finally {
|
||||
env.SCRATCH_PATH = originalScratchPath;
|
||||
await rm(configuredScratchRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
expect(reply.statusCode).toBe(202);
|
||||
expect(mocks.receiveUpload).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
maxBytes: resolveOcrEncodedInputLimit(env.MAX_UPLOAD_SIZE_MB),
|
||||
signal: expect.any(AbortSignal),
|
||||
}),
|
||||
);
|
||||
expect(mocks.copyObjectToFile).toHaveBeenCalledWith(
|
||||
"uploads/ocr-pdf-job/scan.pdf",
|
||||
expect.stringMatching(/ocr-pdf-validation-[^/]+\/input\.pdf$/),
|
||||
expect.objectContaining({
|
||||
maxBytes: resolveOcrEncodedInputLimit(env.MAX_UPLOAD_SIZE_MB),
|
||||
signal: expect.any(AbortSignal),
|
||||
}),
|
||||
);
|
||||
const validationPath = mocks.copyObjectToFile.mock.calls[0][1];
|
||||
expect(validationPath.startsWith(configuredScratchRoot)).toBe(true);
|
||||
expect(mocks.validatePdfPath).toHaveBeenCalledWith(validationPath, {
|
||||
rejectPasswordProtected: true,
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
expect(mocks.getObjectBuffer).not.toHaveBeenCalled();
|
||||
expect(mocks.enqueueToolJob).toHaveBeenCalledTimes(1);
|
||||
expect(request.raw.removeListener).toHaveBeenCalledWith("aborted", expect.any(Function));
|
||||
expect(existsSync(validationPath.replace(/\/input\.pdf$/, ""))).toBe(false);
|
||||
});
|
||||
});
|
||||
+130
-11
@@ -6,21 +6,38 @@
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const dbMocks = vi.hoisted(() => ({ failure: null as Error | null }));
|
||||
|
||||
// Mock DB
|
||||
vi.mock("../../../apps/api/src/db/index.js", () => ({
|
||||
db: {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: () => ({ get: () => null }),
|
||||
db: (() => {
|
||||
const executor = {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: async () => {
|
||||
if (dbMocks.failure) throw dbMocks.failure;
|
||||
return [];
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
insert: () => ({ values: () => ({ run: vi.fn() }) }),
|
||||
update: () => ({
|
||||
set: () => ({
|
||||
where: () => ({ run: () => ({ changes: 0 }) }),
|
||||
insert: () => ({
|
||||
values: async () => {
|
||||
if (dbMocks.failure) throw dbMocks.failure;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
update: () => ({
|
||||
set: () => ({
|
||||
where: async () => {
|
||||
if (dbMocks.failure) throw dbMocks.failure;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
};
|
||||
return {
|
||||
...executor,
|
||||
transaction: async (callback: (tx: typeof executor) => Promise<void>) => callback(executor),
|
||||
};
|
||||
})(),
|
||||
pool: {},
|
||||
closeDb: async () => {},
|
||||
schema: {
|
||||
@@ -34,8 +51,11 @@ vi.mock("../../../apps/api/src/config.js", () => ({
|
||||
|
||||
import type { JobProgress } from "../../../apps/api/src/routes/progress.js";
|
||||
import {
|
||||
buildPersistedSingleFileProgress,
|
||||
buildSingleFileReplayEvent,
|
||||
updateJobProgress,
|
||||
updateSingleFileProgress,
|
||||
updateSingleFileProgressAtomically,
|
||||
} from "../../../apps/api/src/routes/progress.js";
|
||||
|
||||
describe("updateJobProgress", () => {
|
||||
@@ -107,6 +127,7 @@ describe("updateJobProgress", () => {
|
||||
|
||||
describe("updateSingleFileProgress", () => {
|
||||
beforeEach(() => {
|
||||
dbMocks.failure = null;
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
@@ -157,6 +178,104 @@ describe("updateSingleFileProgress", () => {
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("returns a persistence promise that terminal producers can await", async () => {
|
||||
const persisted = updateSingleFileProgress({
|
||||
jobId: "single-awaitable",
|
||||
phase: "complete",
|
||||
percent: 100,
|
||||
result: { text: "durable OCR result" },
|
||||
});
|
||||
|
||||
expect(persisted).toBeInstanceOf(Promise);
|
||||
await expect(persisted).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("propagates a durable persistence failure to an awaiting terminal producer", async () => {
|
||||
dbMocks.failure = new Error("database unavailable");
|
||||
|
||||
await expect(
|
||||
updateSingleFileProgress({
|
||||
jobId: "single-durable-failure",
|
||||
phase: "complete",
|
||||
percent: 100,
|
||||
result: { text: "must not be reported durable" },
|
||||
}),
|
||||
).rejects.toThrow("database unavailable");
|
||||
});
|
||||
|
||||
it("commits authoritative completion and replay state in one awaited transaction", async () => {
|
||||
const mutateAuthoritative = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
await updateSingleFileProgressAtomically(
|
||||
{
|
||||
jobId: "single-atomic",
|
||||
phase: "complete",
|
||||
percent: 100,
|
||||
result: { text: "transactional result" },
|
||||
},
|
||||
mutateAuthoritative,
|
||||
);
|
||||
|
||||
expect(mutateAuthoritative).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("durable single-file terminal progress", () => {
|
||||
it("persists the terminal result alongside percent and stage", () => {
|
||||
expect(
|
||||
buildPersistedSingleFileProgress({
|
||||
jobId: "single-durable",
|
||||
phase: "complete",
|
||||
percent: 100,
|
||||
stage: "complete",
|
||||
result: { text: "OCR result", downloadUrl: "/result.txt" },
|
||||
}),
|
||||
).toEqual({
|
||||
percent: 100,
|
||||
stage: "complete",
|
||||
result: { text: "OCR result", downloadUrl: "/result.txt" },
|
||||
});
|
||||
});
|
||||
|
||||
it("replays a completed durable result from the database row", () => {
|
||||
expect(
|
||||
buildSingleFileReplayEvent({
|
||||
jobId: "single-replay",
|
||||
status: "completed",
|
||||
progress: {
|
||||
percent: 100,
|
||||
stage: "complete",
|
||||
result: { text: "recovered" },
|
||||
},
|
||||
error: null,
|
||||
}),
|
||||
).toEqual({
|
||||
jobId: "single-replay",
|
||||
type: "single",
|
||||
phase: "complete",
|
||||
percent: 100,
|
||||
stage: "complete",
|
||||
result: { text: "recovered" },
|
||||
});
|
||||
});
|
||||
|
||||
it("turns legacy completed rows without results into an explicit terminal failure", () => {
|
||||
expect(
|
||||
buildSingleFileReplayEvent({
|
||||
jobId: "single-legacy",
|
||||
status: "completed",
|
||||
progress: { percent: 100, stage: "complete" },
|
||||
error: null,
|
||||
}),
|
||||
).toEqual({
|
||||
jobId: "single-legacy",
|
||||
type: "single",
|
||||
phase: "failed",
|
||||
percent: 100,
|
||||
error: "Completed result is no longer available. Run the job again.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("JobProgress type shape", () => {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { withRouteScratch } from "../../../apps/api/src/lib/route-scratch.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(roots.splice(0).map((path) => rm(path, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("withRouteScratch", () => {
|
||||
it("removes the complete scratch root after a successful operation", async () => {
|
||||
let root = "";
|
||||
|
||||
const result = await withRouteScratch("unit-success", async (path) => {
|
||||
root = path;
|
||||
roots.push(path);
|
||||
expect(dirname(path)).toBe(join(tmpdir(), "snapotter-scratch"));
|
||||
expect(basename(path)).toMatch(/^unit-success-/);
|
||||
await mkdir(join(path, "nested"));
|
||||
await writeFile(join(path, "nested", "temporary.bin"), "temporary");
|
||||
return 42;
|
||||
});
|
||||
|
||||
expect(result).toBe(42);
|
||||
expect(existsSync(root)).toBe(false);
|
||||
});
|
||||
|
||||
it("removes the complete scratch root when the operation throws", async () => {
|
||||
let root = "";
|
||||
|
||||
await expect(
|
||||
withRouteScratch("unit-error", async (path) => {
|
||||
root = path;
|
||||
roots.push(path);
|
||||
await writeFile(join(path, "temporary.bin"), "temporary");
|
||||
throw new Error("input rejected");
|
||||
}),
|
||||
).rejects.toThrow("input rejected");
|
||||
|
||||
expect(existsSync(root)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects path components instead of deriving roots from client-controlled IDs", async () => {
|
||||
await expect(withRouteScratch("../../victim", async () => undefined)).rejects.toThrow(
|
||||
"scratch prefix",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,11 @@ vi.mock("@fastify/static", () => ({ default: "fastify-static-plugin" }));
|
||||
const mockMultipartPlugin = vi.fn();
|
||||
vi.mock("@fastify/multipart", () => ({ default: mockMultipartPlugin }));
|
||||
|
||||
const multipartPartsMock = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../../../apps/api/src/lib/multipart-parts.js", () => ({
|
||||
multipartParts: multipartPartsMock,
|
||||
}));
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as Record<string, unknown>;
|
||||
return {
|
||||
@@ -170,4 +175,22 @@ describe("registerUpload", () => {
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards route-specific limits through the request.parts replacement", async () => {
|
||||
let hook: ((request: unknown) => Promise<void>) | undefined;
|
||||
const app = {
|
||||
register: vi.fn().mockResolvedValue(undefined),
|
||||
addHook: vi.fn((_name, handler) => {
|
||||
hook = handler;
|
||||
}),
|
||||
};
|
||||
await registerUpload(app as never);
|
||||
const iterator = {};
|
||||
multipartPartsMock.mockReturnValueOnce(iterator);
|
||||
const request = { isMultipart: () => true, parts: vi.fn() };
|
||||
|
||||
await hook?.(request);
|
||||
expect(request.parts({ limits: { fileSize: 1234, files: 2 } })).toBe(iterator);
|
||||
expect(multipartPartsMock).toHaveBeenCalledWith(request, { fileSize: 1234, files: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user