mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: make OCR portable and reliable across AMD64 and ARM64 (#519)
* fix: make OCR portable and reliable * fix: harden OCR installation portability * fix: pin OCR partials across downloads * fix: make OCR execution reliably asynchronous * fix: harden OCR portability and docs routes * fix: preserve decoder and docs safeguards
This commit is contained in:
SnapOtter
@@ -21,6 +21,8 @@
|
||||
export interface QueuedInstall {
|
||||
bundleId: string;
|
||||
jobId: string;
|
||||
/** Shared destructive-mutation generation observed when this work was submitted. */
|
||||
mutationEpoch: string;
|
||||
}
|
||||
|
||||
let activeInstall: QueuedInstall | null = null;
|
||||
@@ -54,6 +56,10 @@ export function enqueue(entry: QueuedInstall): string {
|
||||
}
|
||||
const existing = queue.find((q) => q.bundleId === entry.bundleId);
|
||||
if (existing) {
|
||||
// A request submitted after reset/uninstall explicitly reauthorizes this
|
||||
// bundle. Preserve the existing job id/FIFO position while refreshing its
|
||||
// epoch so the pump does not cancel genuinely new work with the stale one.
|
||||
existing.mutationEpoch = entry.mutationEpoch;
|
||||
return existing.jobId;
|
||||
}
|
||||
queue.push(entry);
|
||||
|
||||
+988
-177
File diff suppressed because it is too large
Load Diff
+460
-112
@@ -9,6 +9,221 @@ import sharp from "sharp";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface DecodeSafetyOptions {
|
||||
/** Maximum decoded width * height accepted by this operation. */
|
||||
maxPixels?: number;
|
||||
/** Maximum decoded width or height accepted by this operation. */
|
||||
maxDimension?: number;
|
||||
/** Cancels external decoder processes when the owning job is canceled. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function commandOptions(options: DecodeSafetyOptions, timeout: number) {
|
||||
return { timeout, signal: options.signal };
|
||||
}
|
||||
|
||||
function assertDimensionsWithinLimit(
|
||||
width: number | undefined,
|
||||
height: number | undefined,
|
||||
options: DecodeSafetyOptions,
|
||||
): void {
|
||||
if (width === undefined || height === undefined || width <= 0 || height <= 0) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
options.maxDimension !== undefined &&
|
||||
(width > options.maxDimension || height > options.maxDimension)
|
||||
) {
|
||||
throw new Error(
|
||||
`Decoded image exceeds the ${options.maxDimension.toLocaleString("en-US")} pixel dimension safety limit (${width.toLocaleString("en-US")}x${height.toLocaleString("en-US")})`,
|
||||
);
|
||||
}
|
||||
if (options.maxPixels !== undefined && width * height > options.maxPixels) {
|
||||
throw new Error(
|
||||
`Decoded image exceeds the ${options.maxPixels.toLocaleString("en-US")} pixel safety limit (${width}x${height})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertDecodedWithinLimit(
|
||||
buffer: Buffer,
|
||||
options: DecodeSafetyOptions,
|
||||
): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (options.maxPixels !== undefined || options.maxDimension !== undefined) {
|
||||
try {
|
||||
const metadata = await sharp(buffer, {
|
||||
limitInputPixels: options.maxPixels ?? false,
|
||||
}).metadata();
|
||||
assertDimensionsWithinLimit(metadata.width, metadata.height, options);
|
||||
} catch (error) {
|
||||
if (isImageSafetyError(error)) throw error;
|
||||
throw new Error("Decoded image exceeds the configured image safety limits", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
options.signal?.throwIfAborted();
|
||||
return buffer;
|
||||
}
|
||||
|
||||
interface ImageDimensions {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
function readKnownEncodedDimensions(buffer: Buffer, format: string): ImageDimensions | undefined {
|
||||
if (format === "qoi" && buffer.length >= 14) {
|
||||
return { width: buffer.readUInt32BE(4), height: buffer.readUInt32BE(8) };
|
||||
}
|
||||
if (format === "psd" && buffer.length >= 22 && buffer.subarray(0, 4).toString() === "8BPS") {
|
||||
return { width: buffer.readUInt32BE(18), height: buffer.readUInt32BE(14) };
|
||||
}
|
||||
if (format === "tga" && buffer.length >= 18) {
|
||||
return { width: buffer.readUInt16LE(12), height: buffer.readUInt16LE(14) };
|
||||
}
|
||||
if (format === "bmp" && buffer.length >= 26 && buffer.subarray(0, 2).toString() === "BM") {
|
||||
const dibSize = buffer.readUInt32LE(14);
|
||||
if (dibSize === 12) {
|
||||
return { width: buffer.readUInt16LE(18), height: buffer.readUInt16LE(20) };
|
||||
}
|
||||
return { width: Math.abs(buffer.readInt32LE(18)), height: Math.abs(buffer.readInt32LE(22)) };
|
||||
}
|
||||
if (format === "dds" && buffer.length >= 20 && buffer.subarray(0, 4).toString() === "DDS ") {
|
||||
return { width: buffer.readUInt32LE(16), height: buffer.readUInt32LE(12) };
|
||||
}
|
||||
if ((format === "ico" || format === "cur") && buffer.length >= 6) {
|
||||
const count = buffer.readUInt16LE(4);
|
||||
let largest: ImageDimensions | undefined;
|
||||
for (let index = 0; index < count && 6 + index * 16 + 16 <= buffer.length; index++) {
|
||||
const offset = 6 + index * 16;
|
||||
const width = buffer[offset] || 256;
|
||||
const height = buffer[offset + 1] || 256;
|
||||
if (!largest || width * height > largest.width * largest.height) largest = { width, height };
|
||||
}
|
||||
return largest;
|
||||
}
|
||||
if (format === "hdr") {
|
||||
const match = buffer
|
||||
.subarray(0, Math.min(buffer.length, 64 * 1024))
|
||||
.toString("ascii")
|
||||
.match(/[+-]Y\s+(\d+)\s+[+-]X\s+(\d+)/i);
|
||||
if (match) return { width: Number(match[2]), height: Number(match[1]) };
|
||||
}
|
||||
if (format === "ppm" || format === "pgm" || format === "pbm") {
|
||||
const header = buffer.subarray(0, Math.min(buffer.length, 64 * 1024)).toString("ascii");
|
||||
if (/^P7(?:\s|$)/.test(header)) {
|
||||
const width = header.match(/^WIDTH\s+(\d+)/im)?.[1];
|
||||
const height = header.match(/^HEIGHT\s+(\d+)/im)?.[1];
|
||||
if (width && height) return { width: Number(width), height: Number(height) };
|
||||
} else {
|
||||
const tokens = header
|
||||
.replace(/#[^\r\n]*/g, " ")
|
||||
.trim()
|
||||
.split(/\s+/);
|
||||
if (/^P[1-6]$/.test(tokens[0] ?? "") && tokens[1] && tokens[2]) {
|
||||
return { width: Number(tokens[1]), height: Number(tokens[2]) };
|
||||
}
|
||||
}
|
||||
}
|
||||
if (format === "dpx" && buffer.length >= 780) {
|
||||
const magic = buffer.subarray(0, 4).toString("ascii");
|
||||
if (magic === "SDPX") {
|
||||
return { width: buffer.readUInt32BE(772), height: buffer.readUInt32BE(776) };
|
||||
}
|
||||
if (magic === "XPDS") {
|
||||
return { width: buffer.readUInt32LE(772), height: buffer.readUInt32LE(776) };
|
||||
}
|
||||
}
|
||||
if (format === "fits") {
|
||||
const header = buffer.subarray(0, Math.min(buffer.length, 1024 * 1024)).toString("ascii");
|
||||
const width = header.match(/(?:^|\s)NAXIS1\s*=\s*(\d+)/)?.[1];
|
||||
const height = header.match(/(?:^|\s)NAXIS2\s*=\s*(\d+)/)?.[1];
|
||||
if (width && height) return { width: Number(width), height: Number(height) };
|
||||
}
|
||||
if (format === "eps") {
|
||||
// BoundingBox is normally near the prolog, or near the trailer when the
|
||||
// prolog says `(atend)`. Avoid duplicating a potentially large EPS buffer
|
||||
// as a JavaScript string just to inspect those two regions.
|
||||
const sampleSize = 1024 * 1024;
|
||||
const source =
|
||||
buffer.length <= sampleSize * 2
|
||||
? buffer.toString("latin1")
|
||||
: `${buffer.subarray(0, sampleSize).toString("latin1")}\n${buffer
|
||||
.subarray(buffer.length - sampleSize)
|
||||
.toString("latin1")}`;
|
||||
const matches = [
|
||||
...source.matchAll(
|
||||
/^%%(?:HiRes)?BoundingBox:\s*(-?[\d.]+)\s+(-?[\d.]+)\s+(-?[\d.]+)\s+(-?[\d.]+)/gim,
|
||||
),
|
||||
];
|
||||
const match = matches.at(-1);
|
||||
if (match) {
|
||||
// EPS is rasterized by this module at 300 DPI; bounding boxes use points.
|
||||
return {
|
||||
width: Math.ceil(((Number(match[3]) - Number(match[1])) * 300) / 72),
|
||||
height: Math.ceil(((Number(match[4]) - Number(match[2])) * 300) / 72),
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function preflightEncodedDimensions(
|
||||
buffer: Buffer,
|
||||
format: string,
|
||||
ext: string | undefined,
|
||||
options: DecodeSafetyOptions,
|
||||
): Promise<void> {
|
||||
if (options.maxPixels === undefined && options.maxDimension === undefined) return;
|
||||
options.signal?.throwIfAborted();
|
||||
|
||||
const known = readKnownEncodedDimensions(buffer, format);
|
||||
if (
|
||||
known &&
|
||||
Number.isFinite(known.width) &&
|
||||
Number.isFinite(known.height) &&
|
||||
known.width > 0 &&
|
||||
known.height > 0
|
||||
) {
|
||||
assertDimensionsWithinLimit(known.width, known.height, options);
|
||||
return;
|
||||
}
|
||||
|
||||
// ExifTool reads container metadata without rasterizing pixels and supports
|
||||
// the hard-to-parse formats here (camera RAW, EXR, JXL and JPEG 2000). It is
|
||||
// part of every supported container image alongside these decoders.
|
||||
const id = randomUUID();
|
||||
const safeExt = (ext || format || "img").replace(/[^a-z0-9]/gi, "") || "img";
|
||||
const inputPath = join(tmpdir(), `dimensions-${id}.${safeExt}`);
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
const { stdout } = await execFileAsync(
|
||||
"exiftool",
|
||||
["-fast2", "-s3", "-ImageWidth", "-ImageHeight", inputPath],
|
||||
{
|
||||
timeout: 15_000,
|
||||
maxBuffer: 64 * 1024,
|
||||
signal: options.signal,
|
||||
},
|
||||
);
|
||||
const values = String(stdout).trim().split(/\s+/).map(Number);
|
||||
if (values.length >= 2 && values.every(Number.isFinite)) {
|
||||
assertDimensionsWithinLimit(values[0], values[1], options);
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
options.signal?.throwIfAborted();
|
||||
if (isImageSafetyError(error)) throw error;
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Cannot safely decode ${format.toUpperCase()}: encoded dimensions are unavailable for the configured image safety limits`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a buffer to a temp file exclusively (O_CREAT | O_EXCL | O_WRONLY).
|
||||
* Prevents symlink / race-condition attacks on predictable temp paths.
|
||||
@@ -63,53 +278,80 @@ export async function decodeToSharpCompat(
|
||||
buffer: Buffer,
|
||||
format: string,
|
||||
ext?: string,
|
||||
options: DecodeSafetyOptions = {},
|
||||
): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
await preflightEncodedDimensions(buffer, format, ext, options);
|
||||
let decoded: Buffer;
|
||||
switch (format) {
|
||||
case "raw":
|
||||
return decodeRaw(buffer, ext);
|
||||
decoded = await decodeRaw(buffer, ext, options);
|
||||
break;
|
||||
case "ico":
|
||||
return decodeIco(buffer);
|
||||
decoded = await decodeIco(buffer, options);
|
||||
break;
|
||||
case "psd":
|
||||
return decodePsd(buffer);
|
||||
decoded = await decodePsd(buffer, options);
|
||||
break;
|
||||
case "tga":
|
||||
return decodeTga(buffer);
|
||||
decoded = await decodeTga(buffer, options);
|
||||
break;
|
||||
case "exr":
|
||||
return decodeExr(buffer);
|
||||
decoded = await decodeExr(buffer, options);
|
||||
break;
|
||||
case "hdr":
|
||||
return decodeHdr(buffer);
|
||||
decoded = await decodeHdr(buffer, options);
|
||||
break;
|
||||
case "bmp":
|
||||
return decodeBmp(buffer);
|
||||
decoded = await decodeBmp(buffer, options);
|
||||
break;
|
||||
case "jxl":
|
||||
return decodeJxl(buffer);
|
||||
decoded = await decodeJxl(buffer, options);
|
||||
break;
|
||||
case "jp2":
|
||||
return decodeJp2(buffer);
|
||||
decoded = await decodeJp2(buffer, options);
|
||||
break;
|
||||
case "eps":
|
||||
return decodeEps(buffer);
|
||||
decoded = await decodeEps(buffer, options);
|
||||
break;
|
||||
case "dds":
|
||||
return decodeDds(buffer);
|
||||
decoded = await decodeDds(buffer, options);
|
||||
break;
|
||||
case "cur":
|
||||
return decodeIco(buffer); // CUR is structurally identical to ICO
|
||||
decoded = await decodeIco(buffer, options); // CUR is structurally identical to ICO
|
||||
break;
|
||||
case "dpx":
|
||||
return decodeDpx(buffer);
|
||||
decoded = await decodeDpx(buffer, options);
|
||||
break;
|
||||
case "fits":
|
||||
return decodeFits(buffer);
|
||||
decoded = await decodeFits(buffer, options);
|
||||
break;
|
||||
case "qoi":
|
||||
return decodeQoi(buffer);
|
||||
decoded = await decodeQoi(buffer, options);
|
||||
break;
|
||||
case "ppm":
|
||||
case "pgm":
|
||||
case "pbm":
|
||||
return decodeNetpbm(buffer, format);
|
||||
decoded = await decodeNetpbm(buffer, format, options);
|
||||
break;
|
||||
default:
|
||||
return buffer;
|
||||
decoded = buffer;
|
||||
}
|
||||
return assertDecodedWithinLimit(decoded, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Last-resort decode: convert any image to PNG via ImageMagick.
|
||||
* Used when Sharp's bundled decoders fail (e.g. AVIF 2.0 bitstreams).
|
||||
*/
|
||||
export async function decodeAnyFormat(buffer: Buffer, format: string): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
export async function decodeAnyFormat(
|
||||
buffer: Buffer,
|
||||
format: string,
|
||||
options: DecodeSafetyOptions = {},
|
||||
): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
await preflightEncodedDimensions(buffer, format, undefined, options);
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const ext = format || "img";
|
||||
const inputPath = join(tmpdir(), `any-in-${id}.${ext}`);
|
||||
@@ -119,10 +361,10 @@ export async function decodeAnyFormat(buffer: Buffer, format: string): Promise<B
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", `png:${outputPath}`]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
return await assertDecodedWithinLimit(await readFile(outputPath), options);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
await rm(outputPath, { force: true }).catch(() => {});
|
||||
@@ -133,28 +375,73 @@ export async function decodeAnyFormat(buffer: Buffer, format: string): Promise<B
|
||||
|
||||
let cachedMagickCmd: string | null = null;
|
||||
|
||||
async function findMagickCmd(): Promise<string> {
|
||||
async function findMagickCmd(options: DecodeSafetyOptions = {}): Promise<string> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (cachedMagickCmd) return cachedMagickCmd;
|
||||
for (const cmd of ["magick", "convert"]) {
|
||||
try {
|
||||
await execFileAsync(cmd, ["--version"], { timeout: 5_000 });
|
||||
await execFileAsync(cmd, ["--version"], commandOptions(options, 5_000));
|
||||
cachedMagickCmd = cmd;
|
||||
return cmd;
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// try next
|
||||
}
|
||||
}
|
||||
throw new Error("No ImageMagick found. Install imagemagick (provides convert/magick).");
|
||||
}
|
||||
|
||||
function magickArgs(cmd: string, args: string[]): string[] {
|
||||
return cmd === "magick" ? ["convert", ...args] : args;
|
||||
/** Build resource limits with syntax shared by ImageMagick 6 and 7. */
|
||||
export function buildImageMagickResourceLimitArgs(options: DecodeSafetyOptions = {}): string[] {
|
||||
// Defense in depth for the conversion subprocess. ImageMagick's `area`
|
||||
// limit is a cache/spill threshold, not the strict pixel-product check;
|
||||
// preflightEncodedDimensions and assertDecodedWithinLimit provide that.
|
||||
const pixelCacheBytes = options.maxPixels === undefined ? undefined : options.maxPixels * 16;
|
||||
const sideLimit = options.maxDimension ?? options.maxPixels;
|
||||
if (sideLimit === undefined || pixelCacheBytes === undefined) return [];
|
||||
|
||||
// Width and height are pixel counts when unitless. A trailing `P` is not a
|
||||
// pixel unit: ImageMagick 6 treats it as an overflowing SI prefix and
|
||||
// resolves the limit to zero, while ImageMagick 7 clamps it near infinity.
|
||||
return [
|
||||
"-limit",
|
||||
"width",
|
||||
String(sideLimit),
|
||||
"-limit",
|
||||
"height",
|
||||
String(sideLimit),
|
||||
"-limit",
|
||||
"area",
|
||||
`${pixelCacheBytes}B`,
|
||||
"-limit",
|
||||
"memory",
|
||||
`${pixelCacheBytes}B`,
|
||||
"-limit",
|
||||
"map",
|
||||
`${pixelCacheBytes}B`,
|
||||
"-limit",
|
||||
"disk",
|
||||
`${pixelCacheBytes * 2}B`,
|
||||
];
|
||||
}
|
||||
|
||||
function magickArgs(cmd: string, args: string[], options: DecodeSafetyOptions = {}): string[] {
|
||||
const limits = buildImageMagickResourceLimitArgs(options);
|
||||
const convertArgs = [...limits, ...args];
|
||||
return cmd === "magick" ? ["convert", ...convertArgs] : convertArgs;
|
||||
}
|
||||
|
||||
function isImageSafetyError(error: unknown): error is Error {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
/(?:pixel (?:dimension )?safety limit|input image exceeds pixel limit)/i.test(error.message)
|
||||
);
|
||||
}
|
||||
|
||||
// ── ICO decoder ────────────────────────────────────────────────
|
||||
|
||||
async function decodeIco(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeIco(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `ico-in-${id}.ico`);
|
||||
const outputPath = join(tmpdir(), `ico-out-${id}.png`);
|
||||
@@ -162,9 +449,11 @@ async function decodeIco(buffer: Buffer): Promise<Buffer> {
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
// ICO contains multiple sizes; extract the largest by sorting
|
||||
await execFileAsync(cmd, magickArgs(cmd, [`${inputPath}[-1]`, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [`${inputPath}[-1]`, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -184,7 +473,11 @@ async function decodeIco(buffer: Buffer): Promise<Buffer> {
|
||||
// ImageMagick delegate is last because on many distros it is the deprecated
|
||||
// ufraw-batch, which fails outright on newer RAW formats (see issue #289).
|
||||
|
||||
async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
async function decodeRaw(
|
||||
buffer: Buffer,
|
||||
ext: string | undefined,
|
||||
options: DecodeSafetyOptions,
|
||||
): Promise<Buffer> {
|
||||
const id = randomUUID();
|
||||
// Use the original extension so LibRaw / ExifTool / ImageMagick can identify
|
||||
// the RAW variant.
|
||||
@@ -200,13 +493,19 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
|
||||
// Attempt 1: dcraw_emu (direct LibRaw decode to TIFF) -- full resolution.
|
||||
try {
|
||||
await execFileAsync("dcraw_emu", ["-T", "-w", "-o", "1", inputPath], { timeout: 120_000 });
|
||||
await execFileAsync(
|
||||
"dcraw_emu",
|
||||
["-T", "-w", "-o", "1", inputPath],
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
const tiffBuf = await readFile(dcrawOutput);
|
||||
if (tiffBuf.length > 0) {
|
||||
// Sharp handles TIFF natively.
|
||||
return await sharp(tiffBuf).png().toBuffer();
|
||||
return await sharp(tiffBuf, { limitInputPixels: options.maxPixels }).png().toBuffer();
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
options.signal?.throwIfAborted();
|
||||
if (error instanceof Error && /pixel safety limit/i.test(error.message)) throw error;
|
||||
// dcraw_emu not available or unsupported format -- fall through
|
||||
}
|
||||
|
||||
@@ -217,6 +516,7 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
encoding: "buffer",
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
timeout: 30_000,
|
||||
signal: options.signal,
|
||||
} as never);
|
||||
// stdout is a Buffer when encoding is "buffer"
|
||||
const jpegBuf = stdout as unknown as Buffer;
|
||||
@@ -225,6 +525,7 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
return jpegBuf;
|
||||
}
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// ExifTool not available or no embedded JPEG -- fall through
|
||||
}
|
||||
|
||||
@@ -235,6 +536,7 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
encoding: "buffer",
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
timeout: 30_000,
|
||||
signal: options.signal,
|
||||
} as never);
|
||||
const previewBuf = stdout as unknown as Buffer;
|
||||
if (
|
||||
@@ -246,16 +548,21 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
return previewBuf;
|
||||
}
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// fall through
|
||||
}
|
||||
|
||||
// Attempt 4: ImageMagick (last resort -- its RAW delegate may be the
|
||||
// deprecated ufraw-batch, which fails on modern formats).
|
||||
const cmd = await findMagickCmd();
|
||||
const cmd = await findMagickCmd(options);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", "-auto-orient", `png:${outputPath}`]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(
|
||||
cmd,
|
||||
[inputPath, "-colorspace", "sRGB", "-auto-orient", `png:${outputPath}`],
|
||||
options,
|
||||
),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -271,17 +578,19 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise<Buffer> {
|
||||
/**
|
||||
* Decode PSD to PNG. Uses [0] to read only the flattened composite layer.
|
||||
*/
|
||||
async function decodePsd(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodePsd(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `psd-in-${id}.psd`);
|
||||
const outputPath = join(tmpdir(), `psd-out-${id}.png`);
|
||||
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, magickArgs(cmd, [`${inputPath}[0]`, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [`${inputPath}[0]`, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -292,17 +601,19 @@ async function decodePsd(buffer: Buffer): Promise<Buffer> {
|
||||
/**
|
||||
* Decode TGA to PNG.
|
||||
*/
|
||||
async function decodeTga(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeTga(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `tga-in-${id}.tga`);
|
||||
const outputPath = join(tmpdir(), `tga-out-${id}.png`);
|
||||
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, magickArgs(cmd, [inputPath, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -314,7 +625,7 @@ async function decodeTga(buffer: Buffer): Promise<Buffer> {
|
||||
* Decode EXR to PNG. Colorspace conversion from linear to sRGB is needed
|
||||
* because EXR files are typically stored in linear light.
|
||||
*/
|
||||
async function decodeExr(buffer: Buffer): Promise<Buffer> {
|
||||
async function decodeExr(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `exr-in-${id}.exr`);
|
||||
const outputPath = join(tmpdir(), `exr-out-${id}.png`);
|
||||
@@ -324,21 +635,26 @@ async function decodeExr(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ImageMagick needs the OpenEXR delegate which is often missing on macOS
|
||||
try {
|
||||
const cmd = await findMagickCmd();
|
||||
const cmd = await findMagickCmd(options);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", "-depth", "8", `png:${outputPath}`]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(
|
||||
cmd,
|
||||
[inputPath, "-colorspace", "sRGB", "-depth", "8", `png:${outputPath}`],
|
||||
options,
|
||||
),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// ImageMagick failed, try ffmpeg
|
||||
}
|
||||
|
||||
await execFileAsync(
|
||||
"ffmpeg",
|
||||
["-y", "-i", inputPath, "-pix_fmt", "rgba", "-update", "1", outputPath],
|
||||
{ timeout: 120_000 },
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -350,8 +666,8 @@ async function decodeExr(buffer: Buffer): Promise<Buffer> {
|
||||
/**
|
||||
* Decode Radiance HDR to PNG. Same colorspace handling as EXR.
|
||||
*/
|
||||
async function decodeHdr(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeHdr(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `hdr-in-${id}.hdr`);
|
||||
const outputPath = join(tmpdir(), `hdr-out-${id}.png`);
|
||||
@@ -360,8 +676,12 @@ async function decodeHdr(buffer: Buffer): Promise<Buffer> {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", "-depth", "8", `png:${outputPath}`]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(
|
||||
cmd,
|
||||
[inputPath, "-colorspace", "sRGB", "-depth", "8", `png:${outputPath}`],
|
||||
options,
|
||||
),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -370,17 +690,19 @@ async function decodeHdr(buffer: Buffer): Promise<Buffer> {
|
||||
}
|
||||
}
|
||||
|
||||
async function decodeBmp(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeBmp(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `bmp-in-${id}.bmp`);
|
||||
const outputPath = join(tmpdir(), `bmp-out-${id}.png`);
|
||||
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, magickArgs(cmd, [inputPath, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -388,7 +710,7 @@ async function decodeBmp(buffer: Buffer): Promise<Buffer> {
|
||||
}
|
||||
}
|
||||
|
||||
async function decodeJxl(buffer: Buffer): Promise<Buffer> {
|
||||
async function decodeJxl(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `jxl-in-${id}.jxl`);
|
||||
const outputPath = join(tmpdir(), `jxl-out-${id}.png`);
|
||||
@@ -399,16 +721,19 @@ async function decodeJxl(buffer: Buffer): Promise<Buffer> {
|
||||
// Try djxl first (from libjxl-tools) — works even when ImageMagick
|
||||
// lacks a JXL delegate (common on Ubuntu stock packages).
|
||||
try {
|
||||
await execFileAsync("djxl", [inputPath, outputPath], { timeout: 120_000 });
|
||||
await execFileAsync("djxl", [inputPath, outputPath], commandOptions(options, 120_000));
|
||||
return await readFile(outputPath);
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// djxl not available, fall back to ImageMagick
|
||||
}
|
||||
|
||||
const cmd = await findMagickCmd();
|
||||
await execFileAsync(cmd, magickArgs(cmd, [inputPath, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
const cmd = await findMagickCmd(options);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -418,7 +743,7 @@ async function decodeJxl(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── JPEG 2000 decoder (opj_decompress-first, ImageMagick fallback) ──
|
||||
|
||||
async function decodeJp2(buffer: Buffer): Promise<Buffer> {
|
||||
async function decodeJp2(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `jp2-in-${id}.jp2`);
|
||||
const outputPath = join(tmpdir(), `jp2-out-${id}.png`);
|
||||
@@ -427,15 +752,19 @@ async function decodeJp2(buffer: Buffer): Promise<Buffer> {
|
||||
try {
|
||||
await execFileAsync("opj_decompress", ["-i", inputPath, "-o", outputPath], {
|
||||
timeout: 60_000,
|
||||
signal: options.signal,
|
||||
});
|
||||
return await readFile(outputPath);
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// opj_decompress not available, fall back to ImageMagick
|
||||
}
|
||||
const cmd = await findMagickCmd();
|
||||
await execFileAsync(cmd, magickArgs(cmd, [inputPath, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
const cmd = await findMagickCmd(options);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -447,13 +776,13 @@ async function decodeJp2(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
const MAX_EPS_SIZE = 50 * 1024 * 1024;
|
||||
|
||||
async function decodeEps(buffer: Buffer): Promise<Buffer> {
|
||||
async function decodeEps(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
if (buffer.length > MAX_EPS_SIZE) {
|
||||
throw new Error(
|
||||
`EPS file too large (${(buffer.length / 1024 / 1024).toFixed(1)}MB, limit: 50MB)`,
|
||||
);
|
||||
}
|
||||
const cmd = await findMagickCmd();
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `eps-in-${id}.eps`);
|
||||
const outputPath = join(tmpdir(), `eps-out-${id}.png`);
|
||||
@@ -461,17 +790,21 @@ async function decodeEps(buffer: Buffer): Promise<Buffer> {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [
|
||||
"-density",
|
||||
"300",
|
||||
"-define",
|
||||
"gs:MaxBitmap=500000000",
|
||||
inputPath,
|
||||
"-colorspace",
|
||||
"sRGB",
|
||||
`png:${outputPath}`,
|
||||
]),
|
||||
{ timeout: 30_000 },
|
||||
magickArgs(
|
||||
cmd,
|
||||
[
|
||||
"-density",
|
||||
"300",
|
||||
"-define",
|
||||
"gs:MaxBitmap=500000000",
|
||||
inputPath,
|
||||
"-colorspace",
|
||||
"sRGB",
|
||||
`png:${outputPath}`,
|
||||
],
|
||||
options,
|
||||
),
|
||||
commandOptions(options, 30_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -482,16 +815,18 @@ async function decodeEps(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── DDS decoder ──
|
||||
|
||||
async function decodeDds(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeDds(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `dds-in-${id}.dds`);
|
||||
const outputPath = join(tmpdir(), `dds-out-${id}.png`);
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, magickArgs(cmd, [`${inputPath}[0]`, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [`${inputPath}[0]`, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
@@ -501,8 +836,8 @@ async function decodeDds(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── DPX / Cineon decoder ──
|
||||
|
||||
async function decodeDpx(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeDpx(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `dpx-in-${id}.dpx`);
|
||||
const outputPath = join(tmpdir(), `dpx-out-${id}.png`);
|
||||
@@ -510,8 +845,8 @@ async function decodeDpx(buffer: Buffer): Promise<Buffer> {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", `png:${outputPath}`]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(cmd, [inputPath, "-colorspace", "sRGB", `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -522,8 +857,8 @@ async function decodeDpx(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── FITS decoder ──
|
||||
|
||||
async function decodeFits(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd();
|
||||
async function decodeFits(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const inputPath = join(tmpdir(), `fits-in-${id}.fits`);
|
||||
const outputPath = join(tmpdir(), `fits-out-${id}.png`);
|
||||
@@ -531,14 +866,12 @@ async function decodeFits(buffer: Buffer): Promise<Buffer> {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [
|
||||
`${inputPath}[0]`,
|
||||
"-normalize",
|
||||
"-colorspace",
|
||||
"sRGB",
|
||||
`png:${outputPath}`,
|
||||
]),
|
||||
{ timeout: 120_000 },
|
||||
magickArgs(
|
||||
cmd,
|
||||
[`${inputPath}[0]`, "-normalize", "-colorspace", "sRGB", `png:${outputPath}`],
|
||||
options,
|
||||
),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
@@ -549,9 +882,16 @@ async function decodeFits(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── QOI decoder ──
|
||||
|
||||
async function decodeQoi(buffer: Buffer): Promise<Buffer> {
|
||||
async function decodeQoi(buffer: Buffer, options: DecodeSafetyOptions): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (buffer.length < 14 || buffer.subarray(0, 4).toString("ascii") !== "qoif") {
|
||||
throw new Error("Invalid QOI header");
|
||||
}
|
||||
assertDimensionsWithinLimit(buffer.readUInt32BE(4), buffer.readUInt32BE(8), options);
|
||||
const { qoiDecode } = await import("@snapotter/image-engine");
|
||||
options.signal?.throwIfAborted();
|
||||
const { header, pixels } = qoiDecode(new Uint8Array(buffer));
|
||||
assertDimensionsWithinLimit(header.width, header.height, options);
|
||||
return sharp(Buffer.from(pixels), {
|
||||
raw: { width: header.width, height: header.height, channels: 4 },
|
||||
})
|
||||
@@ -561,20 +901,28 @@ async function decodeQoi(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
// ── Netpbm (PPM/PGM/PBM) decoder ──
|
||||
|
||||
async function decodeNetpbm(buffer: Buffer, format: string): Promise<Buffer> {
|
||||
async function decodeNetpbm(
|
||||
buffer: Buffer,
|
||||
format: string,
|
||||
options: DecodeSafetyOptions,
|
||||
): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
try {
|
||||
return await sharp(buffer).png().toBuffer();
|
||||
return await sharp(buffer, { limitInputPixels: options.maxPixels }).png().toBuffer();
|
||||
} catch {
|
||||
const cmd = await findMagickCmd();
|
||||
options.signal?.throwIfAborted();
|
||||
const cmd = await findMagickCmd(options);
|
||||
const id = randomUUID();
|
||||
const ext = format === "pgm" ? "pgm" : format === "pbm" ? "pbm" : "ppm";
|
||||
const inputPath = join(tmpdir(), `netpbm-in-${id}.${ext}`);
|
||||
const outputPath = join(tmpdir(), `netpbm-out-${id}.png`);
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, magickArgs(cmd, [inputPath, `png:${outputPath}`]), {
|
||||
timeout: 120_000,
|
||||
});
|
||||
await execFileAsync(
|
||||
cmd,
|
||||
magickArgs(cmd, [inputPath, `png:${outputPath}`], options),
|
||||
commandOptions(options, 120_000),
|
||||
);
|
||||
return await readFile(outputPath);
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
|
||||
@@ -5,9 +5,59 @@ import { open, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import sharp from "sharp";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface HeicDecodeOptions {
|
||||
maxDimension?: number;
|
||||
maxPixels?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function assertImageLimits(
|
||||
width: number | undefined,
|
||||
height: number | undefined,
|
||||
maxPixels: number | undefined,
|
||||
maxDimension: number | undefined,
|
||||
): void {
|
||||
if (width === undefined || height === undefined || width <= 0 || height <= 0) {
|
||||
return;
|
||||
}
|
||||
if (maxDimension !== undefined && (width > maxDimension || height > maxDimension)) {
|
||||
throw new Error(
|
||||
`Decoded image exceeds the ${maxDimension.toLocaleString("en-US")} pixel dimension safety limit (${width.toLocaleString("en-US")}x${height.toLocaleString("en-US")})`,
|
||||
);
|
||||
}
|
||||
if (maxPixels !== undefined && width * height > maxPixels) {
|
||||
throw new Error(
|
||||
`Decoded image exceeds the ${maxPixels.toLocaleString("en-US")} pixel safety limit (${width}x${height})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function readIspeDimensions(buffer: Buffer): Array<{ width: number; height: number }> {
|
||||
// HEIF stores the display dimensions in an Image Spatial Extents (`ispe`)
|
||||
// full box. Reading it avoids invoking a pixel decoder merely to enforce a
|
||||
// pre-decode allocation bound.
|
||||
let offset = 0;
|
||||
const dimensions: Array<{ width: number; height: number }> = [];
|
||||
while (offset + 20 <= buffer.length) {
|
||||
const index = buffer.indexOf("ispe", offset, "ascii");
|
||||
if (index < 0 || index + 16 > buffer.length) break;
|
||||
const boxStart = index - 4;
|
||||
const boxSize = boxStart >= 0 ? buffer.readUInt32BE(boxStart) : 0;
|
||||
if (boxSize >= 20 && boxStart + boxSize <= buffer.length) {
|
||||
dimensions.push({
|
||||
width: buffer.readUInt32BE(index + 8),
|
||||
height: buffer.readUInt32BE(index + 12),
|
||||
});
|
||||
}
|
||||
offset = index + 4;
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a buffer to a temp file exclusively (O_CREAT | O_EXCL | O_WRONLY).
|
||||
* Prevents symlink / race-condition attacks on predictable temp paths.
|
||||
@@ -27,14 +77,19 @@ async function writeTempExclusive(filePath: string, buffer: Buffer): Promise<voi
|
||||
*/
|
||||
let cachedDecodeCmd: string | null = null;
|
||||
|
||||
async function findDecodeCmd(): Promise<string> {
|
||||
async function findDecodeCmd(options: HeicDecodeOptions = {}): Promise<string> {
|
||||
options.signal?.throwIfAborted();
|
||||
if (cachedDecodeCmd) return cachedDecodeCmd;
|
||||
for (const cmd of ["heif-convert", "heif-dec"]) {
|
||||
try {
|
||||
await execFileAsync(cmd, ["--version"], { timeout: 5_000 });
|
||||
await execFileAsync(cmd, ["--version"], {
|
||||
timeout: 5_000,
|
||||
signal: options.signal,
|
||||
});
|
||||
cachedDecodeCmd = cmd;
|
||||
return cmd;
|
||||
} catch {
|
||||
options.signal?.throwIfAborted();
|
||||
// try next
|
||||
}
|
||||
}
|
||||
@@ -50,8 +105,44 @@ async function findDecodeCmd(): Promise<string> {
|
||||
* to add numeric suffixes (-1, -2, ...) to the output filename. We try the
|
||||
* exact path first, then fall back to the -1 suffixed path.
|
||||
*/
|
||||
export async function decodeHeic(buffer: Buffer): Promise<Buffer> {
|
||||
const cmd = await findDecodeCmd();
|
||||
export async function decodeHeic(buffer: Buffer, options: HeicDecodeOptions = {}): Promise<Buffer> {
|
||||
options.signal?.throwIfAborted();
|
||||
const encodedDimensions = readIspeDimensions(buffer);
|
||||
for (const dimensions of encodedDimensions) {
|
||||
assertImageLimits(dimensions.width, dimensions.height, options.maxPixels, options.maxDimension);
|
||||
}
|
||||
|
||||
if (
|
||||
(options.maxPixels !== undefined || options.maxDimension !== undefined) &&
|
||||
encodedDimensions.length === 0
|
||||
) {
|
||||
let dimensionsVerified = false;
|
||||
try {
|
||||
const metadata = await sharp(buffer, {
|
||||
limitInputPixels: options.maxPixels ?? false,
|
||||
}).metadata();
|
||||
assertImageLimits(metadata.width, metadata.height, options.maxPixels, options.maxDimension);
|
||||
dimensionsVerified =
|
||||
metadata.width !== undefined &&
|
||||
metadata.height !== undefined &&
|
||||
metadata.width > 0 &&
|
||||
metadata.height > 0;
|
||||
} catch (error) {
|
||||
// Sharp often has enough libheif support for metadata but not HEVC pixel
|
||||
// decode. Only turn a proven size-limit failure into a rejection.
|
||||
if (
|
||||
error instanceof Error &&
|
||||
/(?:pixel (?:dimension )?safety limit|input image exceeds pixel limit)/i.test(error.message)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (!dimensionsVerified) {
|
||||
throw new Error("Cannot safely decode HEIF: encoded dimensions are unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
const cmd = await findDecodeCmd(options);
|
||||
// Include the PID so concurrent processes (and test workers) write to
|
||||
// distinct, attributable temp paths in the shared tmpdir.
|
||||
const id = `${process.pid}-${randomUUID()}`;
|
||||
@@ -61,14 +152,41 @@ export async function decodeHeic(buffer: Buffer): Promise<Buffer> {
|
||||
|
||||
try {
|
||||
await writeTempExclusive(inputPath, buffer);
|
||||
await execFileAsync(cmd, [inputPath, outputPath], { timeout: 120_000 });
|
||||
await execFileAsync(cmd, [inputPath, outputPath], {
|
||||
timeout: 120_000,
|
||||
signal: options.signal,
|
||||
});
|
||||
options.signal?.throwIfAborted();
|
||||
|
||||
// Single-image HEIF: exact filename. Multi-image: -1 suffix on first image.
|
||||
let decoded: Buffer;
|
||||
try {
|
||||
return await readFile(outputPath);
|
||||
decoded = await readFile(outputPath);
|
||||
} catch {
|
||||
return await readFile(suffixedPath);
|
||||
decoded = await readFile(suffixedPath);
|
||||
}
|
||||
if (options.maxPixels !== undefined || options.maxDimension !== undefined) {
|
||||
try {
|
||||
const metadata = await sharp(decoded, {
|
||||
limitInputPixels: options.maxPixels ?? false,
|
||||
}).metadata();
|
||||
assertImageLimits(metadata.width, metadata.height, options.maxPixels, options.maxDimension);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
/(?:pixel (?:dimension )?safety limit|input image exceeds pixel limit)/i.test(
|
||||
error.message,
|
||||
)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new Error("Decoded image exceeds the configured image safety limits", {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
options.signal?.throwIfAborted();
|
||||
return decoded;
|
||||
} finally {
|
||||
await rm(inputPath, { force: true }).catch(() => {});
|
||||
await rm(outputPath, { force: true }).catch(() => {});
|
||||
@@ -95,10 +213,14 @@ function isHeifBuffer(buffer: Buffer): boolean {
|
||||
* Ensure a buffer is decodable by Sharp. HEIC/HEIF buffers are decoded to
|
||||
* PNG via the system decoder; all other formats pass through unchanged.
|
||||
*/
|
||||
export async function ensureSharpCompat(buffer: Buffer): Promise<Buffer> {
|
||||
export async function ensureSharpCompat(
|
||||
buffer: Buffer,
|
||||
options: HeicDecodeOptions = {},
|
||||
): Promise<Buffer> {
|
||||
if (isHeifBuffer(buffer)) {
|
||||
return decodeHeic(buffer);
|
||||
return decodeHeic(buffer, options);
|
||||
}
|
||||
options.signal?.throwIfAborted();
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ export interface MultipartFilePart {
|
||||
filename: string;
|
||||
encoding: string;
|
||||
mimetype: string;
|
||||
file: Readable;
|
||||
file: Readable & { truncated?: boolean };
|
||||
}
|
||||
|
||||
export interface MultipartFieldPart {
|
||||
@@ -38,13 +38,18 @@ const DONE = Symbol("multipart-done");
|
||||
* signal (a client abort surfaces as an "error" on the stream and as a
|
||||
* truncated-part error from busboy).
|
||||
*/
|
||||
export async function* multipartParts(request: FastifyRequest): AsyncGenerator<MultipartPart> {
|
||||
export async function* multipartParts(
|
||||
request: FastifyRequest,
|
||||
limits: { fileSize?: number; files?: number } = {},
|
||||
): AsyncGenerator<MultipartPart> {
|
||||
const raw = request.raw;
|
||||
const bb = new Busboy({
|
||||
headers: raw.headers as BusboyHeaders,
|
||||
limits: {
|
||||
fileSize: env.MAX_UPLOAD_SIZE_MB > 0 ? env.MAX_UPLOAD_SIZE_MB * 1024 * 1024 : undefined,
|
||||
files: env.MAX_BATCH_SIZE > 0 ? env.MAX_BATCH_SIZE : undefined,
|
||||
fileSize:
|
||||
limits.fileSize ??
|
||||
(env.MAX_UPLOAD_SIZE_MB > 0 ? env.MAX_UPLOAD_SIZE_MB * 1024 * 1024 : undefined),
|
||||
files: limits.files ?? (env.MAX_BATCH_SIZE > 0 ? env.MAX_BATCH_SIZE : undefined),
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createReadStream, createWriteStream, existsSync } from "node:fs";
|
||||
import { mkdir, readdir, rm, stat, statfs, unlink, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readdir, rename, rm, stat, statfs, unlink, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, normalize, sep } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
@@ -116,34 +117,55 @@ export async function putObject(key: string, data: Buffer): Promise<void> {
|
||||
export async function putObjectStream(
|
||||
key: string,
|
||||
source: Readable,
|
||||
opts: { maxBytes?: number } = {},
|
||||
opts: { maxBytes?: number; signal?: AbortSignal } = {},
|
||||
): Promise<number> {
|
||||
assertValidKey(key);
|
||||
opts.signal?.throwIfAborted();
|
||||
const abortSource = () => {
|
||||
const reason =
|
||||
opts.signal?.reason instanceof Error
|
||||
? opts.signal.reason
|
||||
: Object.assign(new Error("The operation was aborted"), { name: "AbortError" });
|
||||
source.destroy(reason);
|
||||
};
|
||||
opts.signal?.addEventListener("abort", abortSource, { once: true });
|
||||
let written = 0;
|
||||
const counter = async function* (src: AsyncIterable<Buffer>) {
|
||||
for await (const chunk of src) {
|
||||
opts.signal?.throwIfAborted();
|
||||
written += chunk.length;
|
||||
if (opts.maxBytes && written > opts.maxBytes) {
|
||||
throw new Error(`Upload exceeds the maximum allowed size (${opts.maxBytes} bytes)`);
|
||||
if (opts.maxBytes !== undefined && written > opts.maxBytes) {
|
||||
throw objectSizeLimitError(opts.maxBytes);
|
||||
}
|
||||
yield chunk;
|
||||
}
|
||||
};
|
||||
if (isS3Enabled()) {
|
||||
const s3 = await getS3();
|
||||
await s3.putGenericObjectStream(key, counter(source));
|
||||
return written;
|
||||
}
|
||||
await assertLocalCapacity();
|
||||
const p = localPath(key);
|
||||
await mkdir(dirname(p), { recursive: true });
|
||||
try {
|
||||
await pipeline(counter(source), createWriteStream(p));
|
||||
} catch (err) {
|
||||
await unlink(p).catch(() => {});
|
||||
throw err;
|
||||
if (isS3Enabled()) {
|
||||
const s3 = await getS3();
|
||||
try {
|
||||
await s3.putGenericObjectStream(key, counter(source), opts.signal);
|
||||
opts.signal?.throwIfAborted();
|
||||
return written;
|
||||
} catch (error) {
|
||||
await s3.deleteGenericObject(key).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const p = localPath(key);
|
||||
try {
|
||||
await assertLocalCapacity();
|
||||
await mkdir(dirname(p), { recursive: true });
|
||||
await pipeline(counter(source), createWriteStream(p), { signal: opts.signal });
|
||||
opts.signal?.throwIfAborted();
|
||||
} catch (err) {
|
||||
await unlink(p).catch(() => {});
|
||||
throw normalizeOperationalWriteError(err);
|
||||
}
|
||||
return written;
|
||||
} finally {
|
||||
opts.signal?.removeEventListener("abort", abortSource);
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
export async function getObjectStream(
|
||||
@@ -164,6 +186,100 @@ export async function getObjectBuffer(key: string): Promise<Buffer> {
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
export interface CopyObjectToFileOptions {
|
||||
/** Hard ceiling enforced both from object metadata and while streaming. */
|
||||
maxBytes: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface CopyReadableToFileOptions {
|
||||
maxBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function objectSizeLimitError(maxBytes: number): Error & { statusCode: 413 } {
|
||||
return Object.assign(new Error(`Object exceeds the maximum allowed size (${maxBytes} bytes)`), {
|
||||
statusCode: 413 as const,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeOperationalWriteError(error: unknown): unknown {
|
||||
const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined;
|
||||
if (error instanceof Error && code && ["EACCES", "EDQUOT", "ENOSPC", "EROFS"].includes(code)) {
|
||||
return Object.assign(error, { statusCode: 503 });
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy one object into scratch storage without materializing it as a Buffer.
|
||||
*
|
||||
* The metadata check avoids downloading a known-oversized S3 object, while
|
||||
* the streaming counter remains authoritative if the object changes between
|
||||
* the size lookup and read. Partial destinations are always removed.
|
||||
*/
|
||||
export async function copyObjectToFile(
|
||||
key: string,
|
||||
destination: string,
|
||||
opts: CopyObjectToFileOptions,
|
||||
): Promise<number> {
|
||||
if (!Number.isSafeInteger(opts.maxBytes) || opts.maxBytes < 0) {
|
||||
throw new Error("maxBytes must be a non-negative safe integer");
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
|
||||
const declaredSize = await getObjectSize(key);
|
||||
opts.signal?.throwIfAborted();
|
||||
if (declaredSize > opts.maxBytes) throw objectSizeLimitError(opts.maxBytes);
|
||||
|
||||
const source = await getObjectStream(key);
|
||||
return copyReadableToFile(source, destination, opts);
|
||||
}
|
||||
|
||||
/** Atomically spool a readable to scratch with bounded memory and cleanup. */
|
||||
export async function copyReadableToFile(
|
||||
source: Readable,
|
||||
destination: string,
|
||||
opts: CopyReadableToFileOptions = {},
|
||||
): Promise<number> {
|
||||
if (opts.maxBytes !== undefined && (!Number.isSafeInteger(opts.maxBytes) || opts.maxBytes < 0)) {
|
||||
throw new Error("maxBytes must be a non-negative safe integer");
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
const stagingPath = `${destination}.${randomUUID()}.partial`;
|
||||
let written = 0;
|
||||
const countAndLimit = async function* (chunks: AsyncIterable<Buffer | Uint8Array | string>) {
|
||||
for await (const chunk of chunks) {
|
||||
opts.signal?.throwIfAborted();
|
||||
const bytes = typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.byteLength;
|
||||
written += bytes;
|
||||
if (opts.maxBytes !== undefined && written > opts.maxBytes) {
|
||||
throw objectSizeLimitError(opts.maxBytes);
|
||||
}
|
||||
yield chunk;
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
// Create the staging inode before constructing the stream. A write stream
|
||||
// opened with `wx` may fail the pipeline before its asynchronous open has
|
||||
// completed; cleanup can then observe ENOENT and the delayed open can leave
|
||||
// an orphan behind. Opening the already-created file with `r+` cannot
|
||||
// recreate it after cleanup.
|
||||
await writeFile(stagingPath, Buffer.alloc(0), { flag: "wx", mode: 0o600 });
|
||||
await pipeline(countAndLimit(source), createWriteStream(stagingPath, { flags: "r+" }), {
|
||||
signal: opts.signal,
|
||||
});
|
||||
opts.signal?.throwIfAborted();
|
||||
await rename(stagingPath, destination);
|
||||
return written;
|
||||
} catch (error) {
|
||||
await unlink(stagingPath).catch(() => {});
|
||||
throw normalizeOperationalWriteError(error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getObjectSize(key: string): Promise<number> {
|
||||
assertValidKey(key);
|
||||
if (isS3Enabled()) {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import {
|
||||
FAST_KOREAN_UNSUPPORTED_REASON,
|
||||
getOcrRuntimeCapability,
|
||||
type OcrRuntimeCapability,
|
||||
type OcrRuntimeQuality,
|
||||
} from "@snapotter/ai";
|
||||
|
||||
export type OcrIngressQuality = "fast" | OcrRuntimeQuality;
|
||||
|
||||
export const OCR_FAST_KOREAN_GUIDANCE = FAST_KOREAN_UNSUPPORTED_REASON;
|
||||
|
||||
export type OcrIngressResolution =
|
||||
| {
|
||||
ok: true;
|
||||
settings: unknown;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
code: "FEATURE_NOT_INSTALLED" | "FEATURE_INCOMPATIBLE";
|
||||
reason: string;
|
||||
requestedQuality: OcrIngressQuality;
|
||||
guidance?: string;
|
||||
};
|
||||
|
||||
type CapabilityReader = () => OcrRuntimeCapability;
|
||||
|
||||
export interface OcrIngressOptions {
|
||||
/** Original client settings, before a schema applies defaults. */
|
||||
requestedSettings?: unknown;
|
||||
/** Test seam for the filesystem-backed capability reader. */
|
||||
readCapability?: CapabilityReader;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isOcrQuality(value: unknown): value is OcrIngressQuality {
|
||||
return value === "fast" || value === "balanced" || value === "best";
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin an OCR quality tier at API ingress and enforce the optional runtime's
|
||||
* advertised capabilities. Callers must pass settings only after their tool
|
||||
* schema has accepted them; the returned settings are the value persisted on
|
||||
* the job, preventing a queued job from silently selecting another tier.
|
||||
*/
|
||||
export function resolveOcrIngressSettings(
|
||||
toolId: string,
|
||||
settings: unknown,
|
||||
options: OcrIngressOptions = {},
|
||||
): OcrIngressResolution {
|
||||
if (toolId !== "ocr" && toolId !== "ocr-pdf") {
|
||||
return { ok: true, settings };
|
||||
}
|
||||
|
||||
const parsedSettings = isRecord(settings) ? settings : {};
|
||||
const requestedSettings = isRecord(options.requestedSettings)
|
||||
? options.requestedSettings
|
||||
: parsedSettings;
|
||||
const readCapability = options.readCapability ?? getOcrRuntimeCapability;
|
||||
const explicitQuality = isOcrQuality(requestedSettings.quality)
|
||||
? requestedSettings.quality
|
||||
: undefined;
|
||||
const legacyQuality =
|
||||
explicitQuality === undefined
|
||||
? requestedSettings.engine === "tesseract"
|
||||
? "fast"
|
||||
: requestedSettings.engine === "paddleocr"
|
||||
? "balanced"
|
||||
: undefined
|
||||
: undefined;
|
||||
|
||||
let capability: OcrRuntimeCapability | undefined;
|
||||
let quality = explicitQuality ?? legacyQuality;
|
||||
if (!quality) {
|
||||
capability = readCapability();
|
||||
if (parsedSettings.language === "ko") {
|
||||
quality =
|
||||
capability.available && capability.qualities.includes("best")
|
||||
? "best"
|
||||
: capability.available && capability.qualities.includes("balanced")
|
||||
? "balanced"
|
||||
: "best";
|
||||
} else {
|
||||
quality =
|
||||
capability.available && capability.qualities.includes("best")
|
||||
? "best"
|
||||
: capability.available && capability.qualities.includes("balanced")
|
||||
? "balanced"
|
||||
: "fast";
|
||||
}
|
||||
}
|
||||
|
||||
const { engine: _legacyEngine, ...normalizedSettings } = parsedSettings;
|
||||
const settingsWithQuality = { ...normalizedSettings, quality };
|
||||
|
||||
if (quality === "fast") {
|
||||
if (parsedSettings.language === "ko") {
|
||||
return {
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "fast-korean-unsupported",
|
||||
requestedQuality: "fast",
|
||||
guidance: OCR_FAST_KOREAN_GUIDANCE,
|
||||
};
|
||||
}
|
||||
return { ok: true, settings: settingsWithQuality };
|
||||
}
|
||||
|
||||
capability ??= readCapability();
|
||||
if (capability.available) {
|
||||
if (capability.qualities.includes(quality)) {
|
||||
return { ok: true, settings: settingsWithQuality };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
code: "FEATURE_INCOMPATIBLE",
|
||||
reason: "quality-not-supported",
|
||||
requestedQuality: quality,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
code:
|
||||
capability.status === "invalid" || capability.status === "incompatible"
|
||||
? "FEATURE_INCOMPATIBLE"
|
||||
: "FEATURE_NOT_INSTALLED",
|
||||
reason: capability.reason,
|
||||
requestedQuality: quality,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { MAX_OCR_INPUT_DIMENSION, MAX_OCR_INPUT_PIXELS } from "@snapotter/ai";
|
||||
import type { PreparedInput } from "../modality/contract.js";
|
||||
import { inputHandlerFor } from "../modality/input-handler.js";
|
||||
|
||||
/** Safely normalize an image before an OCR batch or pipeline stores it. */
|
||||
export function prepareOcrIngressImage(
|
||||
input: Buffer,
|
||||
filename: string,
|
||||
scratchDir: string,
|
||||
): Promise<PreparedInput> {
|
||||
return inputHandlerFor("image").prepare(input, filename, {
|
||||
scratchDir,
|
||||
maxDimension: MAX_OCR_INPUT_DIMENSION,
|
||||
maxPixels: MAX_OCR_INPUT_PIXELS,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/** Hard ceiling for each encoded image or PDF accepted by every OCR ingress path. */
|
||||
export const OCR_MAX_ENCODED_INPUT_BYTES = 512 * 1024 * 1024;
|
||||
|
||||
/** Independent aggregate ceiling for all encoded objects in one OCR batch request. */
|
||||
export const OCR_MAX_BATCH_ENCODED_INPUT_BYTES = OCR_MAX_ENCODED_INPUT_BYTES;
|
||||
|
||||
export interface OcrUploadLimits {
|
||||
/** Maximum encoded bytes for each individual input object. */
|
||||
fileBytes: number;
|
||||
/** Maximum encoded bytes across all input objects in one request. */
|
||||
aggregateBytes: number;
|
||||
}
|
||||
|
||||
export interface OcrEncodedInputViolation {
|
||||
scope: "file" | "aggregate";
|
||||
limitBytes: number;
|
||||
}
|
||||
|
||||
/** Apply the operator's smaller limit without allowing unlimited/large values to remove OCR's cap. */
|
||||
export function resolveOcrEncodedInputLimit(maxUploadSizeMb: number): number {
|
||||
if (!Number.isFinite(maxUploadSizeMb) || maxUploadSizeMb <= 0) {
|
||||
return OCR_MAX_ENCODED_INPUT_BYTES;
|
||||
}
|
||||
return Math.min(Math.floor(maxUploadSizeMb * 1024 * 1024), OCR_MAX_ENCODED_INPUT_BYTES);
|
||||
}
|
||||
|
||||
/** Keep the operator's configured limit per file; the OCR aggregate has its own hard ceiling. */
|
||||
export function resolveOcrUploadLimits(maxUploadSizeMb: number): OcrUploadLimits {
|
||||
return {
|
||||
fileBytes: resolveOcrEncodedInputLimit(maxUploadSizeMb),
|
||||
aggregateBytes: OCR_MAX_BATCH_ENCODED_INPUT_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
/** Validate buffered ingress sizes when the route could not identify OCR until after multipart. */
|
||||
export function findOcrEncodedInputViolation(
|
||||
inputBytes: readonly number[],
|
||||
maxUploadSizeMb: number,
|
||||
): OcrEncodedInputViolation | null {
|
||||
const limits = resolveOcrUploadLimits(maxUploadSizeMb);
|
||||
let aggregateBytes = 0;
|
||||
for (const bytes of inputBytes) {
|
||||
if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > limits.fileBytes) {
|
||||
return { scope: "file", limitBytes: limits.fileBytes };
|
||||
}
|
||||
aggregateBytes += bytes;
|
||||
if (aggregateBytes > limits.aggregateBytes) {
|
||||
return { scope: "aggregate", limitBytes: limits.aggregateBytes };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Preserve a useful HTTP status across Fastify and object-storage limit errors. */
|
||||
export function ocrUploadErrorStatus(error: unknown): 400 | 413 | 503 {
|
||||
if (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"statusCode" in error &&
|
||||
error.statusCode === 503
|
||||
) {
|
||||
return 503;
|
||||
}
|
||||
if (
|
||||
(typeof error === "object" &&
|
||||
error !== null &&
|
||||
"statusCode" in error &&
|
||||
error.statusCode === 413) ||
|
||||
(error instanceof Error &&
|
||||
/(?:file too large|upload exceeds.*(?:maximum|limit))/i.test(error.message))
|
||||
) {
|
||||
return 413;
|
||||
}
|
||||
return 400;
|
||||
}
|
||||
|
||||
export function ocrUploadErrorMessage(statusCode: 400 | 413 | 503): string {
|
||||
if (statusCode === 503) return "Upload storage unavailable";
|
||||
if (statusCode === 413) return "Upload exceeds the allowed size";
|
||||
return "Failed to parse multipart request";
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { MultipartFile } from "@fastify/multipart";
|
||||
import { validatePdfPath } from "../modality/document-input.js";
|
||||
import { sanitizeFilename } from "./filename.js";
|
||||
import { copyReadableToFile, deleteObject, putObjectStream } from "./object-storage.js";
|
||||
|
||||
export interface SpooledMultipartFile {
|
||||
path: string;
|
||||
filename: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export function configuredUploadLimit(maxUploadSizeMb: number): number | undefined {
|
||||
if (!Number.isFinite(maxUploadSizeMb) || maxUploadSizeMb <= 0) return undefined;
|
||||
return Math.floor(maxUploadSizeMb * 1024 * 1024);
|
||||
}
|
||||
|
||||
/** Stream a multipart file to request scratch without accumulating chunks. */
|
||||
export async function spoolMultipartFile(
|
||||
part: MultipartFile,
|
||||
scratchDir: string,
|
||||
index: number,
|
||||
opts: { maxBytes?: number; signal?: AbortSignal } = {},
|
||||
): Promise<SpooledMultipartFile> {
|
||||
const filename = sanitizeFilename(part.filename || "file");
|
||||
const path = join(scratchDir, `${index}-${filename}`);
|
||||
const size = await copyReadableToFile(part.file, path, opts);
|
||||
return { path, filename, size };
|
||||
}
|
||||
|
||||
/** Validate a spooled PDF by path, then stream the same bytes to its final object ref. */
|
||||
export async function storeValidatedOcrPdf(
|
||||
file: SpooledMultipartFile,
|
||||
key: string,
|
||||
opts: { maxBytes: number; signal?: AbortSignal },
|
||||
): Promise<number> {
|
||||
await validatePdfPath(file.path, {
|
||||
rejectPasswordProtected: true,
|
||||
signal: opts.signal,
|
||||
});
|
||||
try {
|
||||
const written = await putObjectStream(key, createReadStream(file.path), opts);
|
||||
if (written !== file.size) {
|
||||
throw new Error(`OCR PDF scratch file changed size (${file.size} to ${written} bytes)`);
|
||||
}
|
||||
return written;
|
||||
} catch (error) {
|
||||
await deleteObject(key).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { env } from "../config.js";
|
||||
|
||||
const SAFE_SCRATCH_PREFIX = /^[a-z][a-z0-9-]{0,31}$/;
|
||||
|
||||
function routeScratchRoot(): string {
|
||||
return env.SCRATCH_PATH || join(tmpdir(), "snapotter-scratch");
|
||||
}
|
||||
|
||||
/**
|
||||
* Own a request-scoped scratch root for exactly one operation.
|
||||
*
|
||||
* Keeping creation and recursive cleanup in one primitive makes early HTTP
|
||||
* returns and thrown decoder errors follow the same cleanup path.
|
||||
*/
|
||||
export async function withRouteScratch<T>(
|
||||
prefix: string,
|
||||
operation: (path: string) => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!SAFE_SCRATCH_PREFIX.test(prefix)) {
|
||||
throw new Error("Route scratch prefix must be a safe path component");
|
||||
}
|
||||
const root = routeScratchRoot();
|
||||
let path: string;
|
||||
try {
|
||||
await mkdir(root, { recursive: true });
|
||||
path = await mkdtemp(join(root, `${prefix}-`));
|
||||
} catch (error) {
|
||||
const code = error instanceof Error ? (error as NodeJS.ErrnoException).code : undefined;
|
||||
if (error instanceof Error && code && ["EACCES", "EDQUOT", "ENOSPC", "EROFS"].includes(code)) {
|
||||
throw Object.assign(error, { statusCode: 503 });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
return await operation(path);
|
||||
} finally {
|
||||
await rm(path, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,13 @@ export interface ReceivedUpload {
|
||||
export async function receiveUpload(
|
||||
part: MultipartFile,
|
||||
jobId: string,
|
||||
opts: { maxBytes?: number } = {},
|
||||
opts: { maxBytes?: number; signal?: AbortSignal } = {},
|
||||
): Promise<ReceivedUpload> {
|
||||
const filename = sanitizeFilename(part.filename || "upload");
|
||||
const key = `uploads/${jobId}/${filename}`;
|
||||
const size = await putObjectStream(key, part.file, { maxBytes: opts.maxBytes });
|
||||
const size = await putObjectStream(key, part.file, {
|
||||
maxBytes: opts.maxBytes,
|
||||
signal: opts.signal,
|
||||
});
|
||||
return { key, filename, size };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user