diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bd2f6e9..61eb9f27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install system dependencies (HEIC + ExifTool + ImageMagick + exotic format tools) - run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl imagemagick ghostscript libjxl-tools libopenjp2-tools + run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl libraw-bin imagemagick ghostscript libjxl-tools libopenjp2-tools - uses: ./.github/actions/setup - run: pnpm vitest run tests/unit/ --reporter=verbose @@ -72,7 +72,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install system dependencies (image formats + doc-engine qpdf/pandoc/LibreOffice) - run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl imagemagick ghostscript libjxl-tools libopenjp2-tools qpdf pandoc libreoffice-calc libreoffice-impress libreoffice-writer + run: sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl libraw-bin imagemagick ghostscript libjxl-tools libopenjp2-tools qpdf pandoc libreoffice-calc libreoffice-impress libreoffice-writer - name: Install pdfcpu (doc-engine PDF layout binary; matches docker/Dockerfile v0.13.0) run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bda00dc7..78c1fc84 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,7 +9,7 @@ permissions: contents: read env: - SYSTEM_DEPS: libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl imagemagick ghostscript libjxl-tools libopenjp2-tools ffmpeg qpdf + SYSTEM_DEPS: libheif-examples libheif-plugin-x265 libheif-plugin-libde265 libimage-exiftool-perl libraw-bin imagemagick ghostscript libjxl-tools libopenjp2-tools ffmpeg qpdf jobs: e2e-full: diff --git a/apps/api/src/lib/format-decoders.ts b/apps/api/src/lib/format-decoders.ts index 6f64d402..6d875e62 100644 --- a/apps/api/src/lib/format-decoders.ts +++ b/apps/api/src/lib/format-decoders.ts @@ -172,27 +172,46 @@ async function decodeIco(buffer: Buffer): Promise { } } -// ── RAW decoder (ExifTool-first, ImageMagick fallback) ────────── +// ── RAW decoder (LibRaw-first, ExifTool + ImageMagick fallbacks) ── // -// Strategy: Many camera RAW files (CR2, CR3, NEF, ARW, etc.) embed a -// full-size JPEG preview. ExifTool can extract it near-instantly with -// `-b -JpgFromRaw`. This is faster and more reliable than ImageMagick's -// LibRaw delegate, which may not support newer formats like CR3. +// Strategy: decode the full-resolution RAW with dcraw_emu (LibRaw), which is +// actively maintained and handles modern Camera RAW including iPhone ProRAW +// DNG. We prefer this over extracting the embedded JPEG preview so a +// full-resolution RAW never silently comes back as a reduced-size preview. // -// If ExifTool extraction fails (no embedded JPEG, or exiftool not -// installed), we fall back to ImageMagick + LibRaw. +// Fallbacks, in order: the embedded full-size JPEG (ExifTool JpgFromRaw), +// then the embedded preview (ExifTool PreviewImage), then ImageMagick. The +// ImageMagick delegate is last because on many distros it is the deprecated +// ufraw-batch, which fails outright on newer RAW formats (see issue #289). async function decodeRaw(buffer: Buffer, ext?: string): Promise { const id = randomUUID(); - // Use the original extension so ExifTool / ImageMagick can identify the RAW variant. + // Use the original extension so LibRaw / ExifTool / ImageMagick can identify + // the RAW variant. const suffix = ext ? `.${ext.replace(/^\./, "")}` : ".dng"; const inputPath = join(tmpdir(), `raw-in-${id}${suffix}`); const outputPath = join(tmpdir(), `raw-out-${id}.png`); + // dcraw_emu APPENDS the output extension to the full input path + // (raw-in-X.dng -> raw-in-X.dng.tiff); it does NOT replace the extension. + const dcrawOutput = `${inputPath}.tiff`; try { await writeTempExclusive(inputPath, buffer); - // Attempt 1: ExifTool embedded JPEG extraction (fast path) + // Attempt 1: dcraw_emu (direct LibRaw decode to TIFF) -- full resolution. + try { + await execFileAsync("dcraw_emu", ["-T", "-w", "-o", "1", inputPath], { timeout: 120_000 }); + const tiffBuf = await readFile(dcrawOutput); + if (tiffBuf.length > 0) { + // Sharp handles TIFF natively. + return await sharp(tiffBuf).png().toBuffer(); + } + } catch { + // dcraw_emu not available or unsupported format -- fall through + } + + // Attempt 2: ExifTool full-size embedded JPEG (JpgFromRaw). Many formats + // (NEF, RW2, ...) embed a full-resolution JPEG under this tag. try { const { stdout } = await execFileAsync("exiftool", ["-b", "-JpgFromRaw", inputPath], { encoding: "buffer", @@ -201,18 +220,16 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise { } as never); // stdout is a Buffer when encoding is "buffer" const jpegBuf = stdout as unknown as Buffer; - if (jpegBuf && jpegBuf.length > 1000) { - // Verify it starts with JPEG SOI marker - if (jpegBuf[0] === 0xff && jpegBuf[1] === 0xd8) { - return jpegBuf; - } + // length guard + JPEG SOI marker + if (jpegBuf && jpegBuf.length > 1000 && jpegBuf[0] === 0xff && jpegBuf[1] === 0xd8) { + return jpegBuf; } } catch { // ExifTool not available or no embedded JPEG -- fall through } - // Attempt 1b: ExifTool PreviewImage extraction (some formats store - // preview under a different tag than JpgFromRaw). + // Attempt 3: ExifTool PreviewImage (some formats store the embedded image + // under a different tag than JpgFromRaw). try { const { stdout } = await execFileAsync("exiftool", ["-b", "-PreviewImage", inputPath], { encoding: "buffer", @@ -220,32 +237,20 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise { timeout: 30_000, } as never); const previewBuf = stdout as unknown as Buffer; - if (previewBuf && previewBuf.length > 1000) { - if (previewBuf[0] === 0xff && previewBuf[1] === 0xd8) { - return previewBuf; - } + if ( + previewBuf && + previewBuf.length > 1000 && + previewBuf[0] === 0xff && + previewBuf[1] === 0xd8 + ) { + return previewBuf; } } catch { // fall through } - // Attempt 2: dcraw_emu from libraw-bin (direct LibRaw decode to TIFF). - // More reliable than ImageMagick's delegate chain for Camera RAW. - try { - await execFileAsync("dcraw_emu", ["-T", "-w", "-o", "1", inputPath], { timeout: 120_000 }); - // dcraw_emu writes output next to the input with a .tiff extension - const dcrawOutput = inputPath.replace(/\.[^.]+$/, ".tiff"); - const tiffBuf = await readFile(dcrawOutput); - await rm(dcrawOutput, { force: true }).catch(() => {}); - if (tiffBuf.length > 0) { - // Convert TIFF to PNG via Sharp (Sharp handles TIFF natively) - return await sharp(tiffBuf).png().toBuffer(); - } - } catch { - // dcraw_emu not available or unsupported format -- fall through - } - - // Attempt 3: ImageMagick + LibRaw delegate (full decode) + // Attempt 4: ImageMagick (last resort -- its RAW delegate may be the + // deprecated ufraw-batch, which fails on modern formats). const cmd = await findMagickCmd(); await execFileAsync( cmd, @@ -256,6 +261,8 @@ async function decodeRaw(buffer: Buffer, ext?: string): Promise { } finally { await rm(inputPath, { force: true }).catch(() => {}); await rm(outputPath, { force: true }).catch(() => {}); + // Always clean up the dcraw_emu output, even when a later tier won. + await rm(dcrawOutput, { force: true }).catch(() => {}); } } diff --git a/tests/unit/api/format-decoders.test.ts b/tests/unit/api/format-decoders.test.ts index 22f50abf..c7ad8477 100644 --- a/tests/unit/api/format-decoders.test.ts +++ b/tests/unit/api/format-decoders.test.ts @@ -1,14 +1,32 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import sharp from "sharp"; import { describe, expect, it } from "vitest"; import { decodeToSharpCompat, needsCliDecode } from "../../../apps/api/src/lib/format-decoders.js"; import { encodeQoi } from "../../../apps/api/src/lib/format-encoders.js"; import { fixtures, readFixture } from "../../fixtures/index.js"; +const execFileAsync = promisify(execFile); + function isImageMagickError(err: unknown): boolean { if (!(err instanceof Error)) return false; return err.message.includes("No ImageMagick") || err.message.includes("ENOENT"); } +/** + * Whether a CLI tool is on PATH. Distinguishes "not installed" (ENOENT) from + * "ran but errored" (e.g. unknown flag) so tests assert strongly where the + * tool exists and skip the strong assertion where it does not. + */ +async function commandExists(cmd: string): Promise { + try { + await execFileAsync(cmd, ["-version"], { timeout: 5_000 }); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException)?.code !== "ENOENT"; + } +} + const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47]; function isPng(buf: Buffer): boolean { @@ -421,6 +439,45 @@ describe("decodeToSharpCompat - QOI decoder", () => { }); }); +describe("decodeToSharpCompat - RAW (Camera RAW)", () => { + // Regression test for issue #289: DNG decode used to fall through a dead + // dcraw_emu tier (wrong output filename) to ImageMagick's ufraw-batch + // delegate, which fails on modern DNG. Even where it "worked", it returned + // the small embedded preview instead of the full image. The decode chain + // must prefer LibRaw (dcraw_emu) so we get full resolution. + it("decodes DNG to full resolution, not the embedded preview (issue #289)", async () => { + let result: Buffer; + try { + const input = readFixture(fixtures.image.formats("dng")); + result = await decodeToSharpCompat(input, "raw", "dng"); + } catch (err) { + if (isImageMagickError(err)) return; // no RAW decoders available in this env + throw err; + } + const { width, height } = await assertValidImage(result); + // sample.dng: embedded preview is 1024x683; the full image is ~3516x2328. + // With LibRaw available, decode must yield the full image, not the preview. + if (await commandExists("dcraw_emu")) { + expect(width).toBeGreaterThanOrEqual(3000); + expect(height).toBeGreaterThanOrEqual(2000); + } + }); + + const rawFormats = ["dng", "cr2", "nef", "arw", "orf", "rw2"] as const; + for (const ext of rawFormats) { + it(`decodes ${ext.toUpperCase()} to a valid image`, async () => { + try { + const input = readFixture(fixtures.image.formats(ext)); + const result = await decodeToSharpCompat(input, "raw", ext); + await assertValidImage(result); + } catch (err) { + if (isImageMagickError(err)) return; + throw err; + } + }); + } +}); + describe("decodeToSharpCompat - EPS size limit", () => { it("rejects EPS files over 50MB", async () => { const largeBuffer = Buffer.alloc(51 * 1024 * 1024);