diff --git a/apps/api/src/routes/files.ts b/apps/api/src/routes/files.ts index 2001830d..b34760a1 100644 --- a/apps/api/src/routes/files.ts +++ b/apps/api/src/routes/files.ts @@ -227,7 +227,7 @@ export async function fileRoutes(app: FastifyInstance): Promise { }); } -function getContentType(ext: string): string { +export function getContentType(ext: string): string { const map: Record = { jpg: "image/jpeg", jpeg: "image/jpeg", @@ -303,5 +303,9 @@ function getContentType(ext: string): string { heic: "image/heic", heif: "image/heif", }; - return map[ext] ?? "application/octet-stream"; + const type = map[ext] ?? "application/octet-stream"; + // Text payloads (extracted text, markdown, CSV, subtitles) are written UTF-8. + // Without an explicit charset a browser can sniff a legacy encoding and + // mojibake non-Latin scripts like Arabic when the file is viewed inline (#589). + return type.startsWith("text/") ? `${type}; charset=utf-8` : type; } diff --git a/apps/api/src/routes/tools/pdf-to-text.ts b/apps/api/src/routes/tools/pdf-to-text.ts index 48e3722f..ef03509e 100644 --- a/apps/api/src/routes/tools/pdf-to-text.ts +++ b/apps/api/src/routes/tools/pdf-to-text.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { pdfTextPy } from "@snapotter/doc-engine"; import type { FastifyInstance } from "fastify"; import { z } from "zod"; +import { InputValidationError } from "../../modality/contract.js"; import { createToolRoute } from "../tool-factory.js"; const settingsSchema = z.object({}); @@ -23,6 +24,15 @@ export function registerPdfToText(app: FastifyInstance) { const outPath = join(ctx.scratchDir, `${base}.txt`); ctx.report(10, "Extracting text"); const result = await pdfTextPy(inPath, outPath); + // A scanned or image-only PDF has no text layer, so extraction returns an + // empty file. Rather than hand back a silent 0-byte "success", tell the + // user to run OCR instead (#589). + if (!result.hasText) { + throw new InputValidationError( + "This PDF has no extractable text layer. It looks scanned or image-only, so use the PDF OCR tool to read its text.", + 422, + ); + } ctx.report(90, "Done"); return { diff --git a/packages/ai/python/doc_text.py b/packages/ai/python/doc_text.py index a8df44ea..c6639fcb 100644 --- a/packages/ai/python/doc_text.py +++ b/packages/ai/python/doc_text.py @@ -1,4 +1,4 @@ -"""Extract plain text. Args: {"path": in, "out": out-txt-path}. Prints {"chars": N}.""" +"""Extract plain text. Args: {"path": in, "out": out-txt-path}. Prints {"chars": N, "hasText": bool}.""" import json import sys @@ -19,9 +19,14 @@ def main(): parts = [page.get_text() for page in doc] doc.close() text = "\n".join(parts) + # hasText distinguishes a PDF with a real text layer from a scanned or + # image-only PDF, where every page returns "" and only the join newlines + # remain. len(text) alone can't tell them apart, so the caller uses this + # to offer OCR instead of handing back an empty file (#589). + has_text = any(part.strip() for part in parts) with open(out, "w", encoding="utf-8") as fh: fh.write(text) - print(json.dumps({"chars": len(text)})) + print(json.dumps({"chars": len(text), "hasText": has_text})) except Exception as exc: # noqa: BLE001 print(json.dumps({"error": str(exc)})) sys.exit(1) diff --git a/packages/doc-engine/src/python-docs.ts b/packages/doc-engine/src/python-docs.ts index b29e6135..fac42b15 100644 --- a/packages/doc-engine/src/python-docs.ts +++ b/packages/doc-engine/src/python-docs.ts @@ -82,16 +82,24 @@ export async function pdfRedactPy( } /** Extract plain text from a PDF (PyMuPDF get_text). */ -export async function pdfTextPy(inPath: string, outTxtPath: string): Promise<{ chars: number }> { +export async function pdfTextPy( + inPath: string, + outTxtPath: string, +): Promise<{ chars: number; hasText: boolean }> { const stdout = await runDocsScript("doc_text", { path: inPath, out: outTxtPath }); - const parsed = parseDocsJson<{ chars?: number; error?: string }>("doc_text", stdout); + const parsed = parseDocsJson<{ chars?: number; hasText?: boolean; error?: string }>( + "doc_text", + stdout, + ); if (parsed.error) { throw new Error(`doc_text failed: ${parsed.error}`); } if (typeof parsed.chars !== "number") { throw new Error(`doc_text failed: ${stdout.slice(0, 200)}`); } - return { chars: parsed.chars }; + // Fall back to a chars>0 heuristic if an older sidecar omits hasText. + const hasText = parsed.hasText ?? parsed.chars > 0; + return { chars: parsed.chars, hasText }; } /** PDF to DOCX conversion (pdf2docx). Long-running: 5 min timeout. */ diff --git a/tests/helpers/python-gate.ts b/tests/helpers/python-gate.ts index e924b967..6c951241 100644 --- a/tests/helpers/python-gate.ts +++ b/tests/helpers/python-gate.ts @@ -22,7 +22,7 @@ function resolvePython(): string | null { return null; } -const pythonBin = resolvePython(); +export const pythonBin = resolvePython(); /** Check whether a Python module is importable by the resolved interpreter. */ export function pythonWith(mod: string): boolean { diff --git a/tests/integration/tools/document/pdf-to-text.test.ts b/tests/integration/tools/document/pdf-to-text.test.ts index ab43b188..65a06919 100644 --- a/tests/integration/tools/document/pdf-to-text.test.ts +++ b/tests/integration/tools/document/pdf-to-text.test.ts @@ -1,6 +1,10 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { fixtures, readFixture } from "../../../fixtures/index.js"; -import { hasFitz } from "../../../helpers/python-gate.js"; +import { hasFitz, pythonBin } from "../../../helpers/python-gate.js"; import { buildTestApp, createMultipartPayload, @@ -22,10 +26,10 @@ afterAll(async () => { await testApp.cleanup(); }, 10_000); -async function runTool(settings: Record = {}) { +async function runTool(content: Buffer = PDF, filename = "test-3page.pdf") { const { body, contentType } = createMultipartPayload([ - { name: "file", filename: "test-3page.pdf", contentType: "application/pdf", content: PDF }, - { name: "settings", content: JSON.stringify(settings) }, + { name: "file", filename, contentType: "application/pdf", content }, + { name: "settings", content: JSON.stringify({}) }, ]); return testApp.app.inject({ method: "POST", @@ -35,19 +39,45 @@ async function runTool(settings: Record = {}) { }); } +/** Build a one-page PDF whose only content is a rendered image, so it has no + * text layer (the shape of a scanned document). Uses the same PyMuPDF that + * gates this suite. */ +function makeImageOnlyPdf(): Buffer { + const dir = mkdtempSync(join(tmpdir(), "pdf-scan-")); + const out = join(dir, "scanned.pdf"); + const script = [ + "import sys, fitz", + "d = fitz.open(); p = d.new_page()", + "tmp = fitz.open(); tp = tmp.new_page(); tp.insert_text((50, 50), 'SCANNED PAGE')", + "pix = tp.get_pixmap(dpi=100); tmp.close()", + "p.insert_image(p.rect, pixmap=pix)", + "d.save(sys.argv[-1]); d.close()", + ].join("\n"); + const res = spawnSync(pythonBin as string, ["-c", script, out], { encoding: "utf8" }); + if (res.status !== 0) throw new Error(`could not build image-only PDF: ${res.stderr}`); + return readFileSync(out); +} + describe.skipIf(!hasFitz)("pdf-to-text (requires PyMuPDF)", () => { - it("extracts text and returns a .txt file", async () => { + it("extracts text and serves the .txt as UTF-8", async () => { const res = await runTool(); expect(res.statusCode).toBe(200); const envelope = JSON.parse(res.body); expect(envelope.downloadUrl).toBeDefined(); - const dl = await testApp.app.inject({ - method: "GET", - url: envelope.downloadUrl, - }); + const dl = await testApp.app.inject({ method: "GET", url: envelope.downloadUrl }); expect(dl.statusCode).toBe(200); - // The text output should have content (the 3-page fixture has text). + // The 3-page fixture has a text layer, so the output has content. expect(dl.rawPayload.length).toBeGreaterThan(0); + // Charset must be explicit so non-Latin scripts don't mojibake inline (#589). + expect(dl.headers["content-type"]).toContain("charset=utf-8"); + }, 60_000); + + it("tells the user to run OCR when the PDF has no text layer", async () => { + const res = await runTool(makeImageOnlyPdf(), "scanned.pdf"); + expect(res.statusCode).toBe(422); + const body = JSON.parse(res.body); + expect(body.details).toMatch(/text layer/i); + expect(body.details).toMatch(/OCR/); }, 60_000); }); diff --git a/tests/unit/api/content-type.test.ts b/tests/unit/api/content-type.test.ts new file mode 100644 index 00000000..1d74c7f2 --- /dev/null +++ b/tests/unit/api/content-type.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { getContentType } from "../../../apps/api/src/routes/files.js"; + +describe("getContentType charset", () => { + it("adds charset=utf-8 to text types so non-Latin scripts render inline", () => { + // Without a charset a browser can sniff a legacy encoding and mojibake + // UTF-8 Arabic when viewing an extracted .txt inline (#589). + expect(getContentType("txt")).toBe("text/plain; charset=utf-8"); + expect(getContentType("md")).toBe("text/markdown; charset=utf-8"); + expect(getContentType("csv")).toBe("text/csv; charset=utf-8"); + expect(getContentType("vtt")).toBe("text/vtt; charset=utf-8"); + }); + + it("leaves binary and application types unchanged", () => { + expect(getContentType("pdf")).toBe("application/pdf"); + expect(getContentType("png")).toBe("image/png"); + expect(getContentType("json")).toBe("application/json"); + expect(getContentType("unknown-ext")).toBe("application/octet-stream"); + }); +}); diff --git a/tests/unit/api/document-route-processors.test.ts b/tests/unit/api/document-route-processors.test.ts index 6c0050a1..aebb5893 100644 --- a/tests/unit/api/document-route-processors.test.ts +++ b/tests/unit/api/document-route-processors.test.ts @@ -31,7 +31,7 @@ vi.mock("../../../apps/api/src/config.js", () => ({ vi.mock("@snapotter/doc-engine", () => ({ htmlToPdfPy: vi.fn(), pdfFlattenPy: vi.fn(), - pdfTextPy: vi.fn(async () => ({ chars: 12 })), + pdfTextPy: vi.fn(async () => ({ chars: 12, hasText: true })), qpdfAvailable: vi.fn(() => false), qpdfCheck: vi.fn(), qpdfPageCount: vi.fn(), @@ -105,6 +105,19 @@ describe("document route processors", () => { }); }); + it("rejects a PDF with no text layer and points the user to OCR", async () => { + // A scanned/image-only PDF yields hasText=false; the route must reject with + // an OCR handoff instead of returning a silent empty .txt (#589). + vi.mocked(pdfTextPy).mockResolvedValueOnce({ chars: 0, hasText: false }); + registerPdfToText(createMockApp()); + const config = getToolConfig("pdf-to-text"); + + await withScratch(async (scratchDir) => { + const ctx = createCtx(scratchDir, "scanned.pdf"); + await expect(config?.processV2?.(ctx)).rejects.toThrow(/text layer/i); + }); + }); + it("flattens PDFs through pdfFlattenPy", async () => { registerFlattenPdf(createMockApp()); const config = getToolConfig("flatten-pdf");