fix(pdf): flag scanned PDFs in pdf-to-text and serve text as UTF-8 (#603)

When a PDF has no text layer (scanned or image-only), pdf-to-text now
returns a 422 that points at the OCR tool instead of a silent empty file,
and text downloads carry charset=utf-8 so UTF-8 Arabic renders correctly
when the .txt is viewed inline.

Fixes #589

apps/api/src/routes/files.ts

@@ -227,7 +227,7 @@ export async function fileRoutes(app: FastifyInstance): Promise<void> {
   });
 }
 
-function getContentType(ext: string): string {
+export function getContentType(ext: string): string {
   const map: Record<string, string> = {
     jpg: "image/jpeg",
     jpeg: "image/jpeg",
@@ -303,5 +303,9 @@ function getContentType(ext: string): string {
     heic: "image/heic",
     heif: "image/heif",
   };
-  return map[ext] ?? "application/octet-stream";
+  const type = map[ext] ?? "application/octet-stream";
+  // Text payloads (extracted text, markdown, CSV, subtitles) are written UTF-8.
+  // Without an explicit charset a browser can sniff a legacy encoding and
+  // mojibake non-Latin scripts like Arabic when the file is viewed inline (#589).
+  return type.startsWith("text/") ? `${type}; charset=utf-8` : type;
 }

apps/api/src/routes/tools/pdf-to-text.ts

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { pdfTextPy } from "@snapotter/doc-engine";
 import type { FastifyInstance } from "fastify";
 import { z } from "zod";
+import { InputValidationError } from "../../modality/contract.js";
 import { createToolRoute } from "../tool-factory.js";
 
 const settingsSchema = z.object({});
@@ -23,6 +24,15 @@ export function registerPdfToText(app: FastifyInstance) {
       const outPath = join(ctx.scratchDir, `${base}.txt`);
       ctx.report(10, "Extracting text");
       const result = await pdfTextPy(inPath, outPath);
+      // A scanned or image-only PDF has no text layer, so extraction returns an
+      // empty file. Rather than hand back a silent 0-byte "success", tell the
+      // user to run OCR instead (#589).
+      if (!result.hasText) {
+        throw new InputValidationError(
+          "This PDF has no extractable text layer. It looks scanned or image-only, so use the PDF OCR tool to read its text.",
+          422,
+        );
+      }
       ctx.report(90, "Done");
 
       return {

packages/ai/python/doc_text.py

@@ -1,4 +1,4 @@
-"""Extract plain text. Args: {"path": in, "out": out-txt-path}. Prints {"chars": N}."""
+"""Extract plain text. Args: {"path": in, "out": out-txt-path}. Prints {"chars": N, "hasText": bool}."""
 import json
 import sys
 
@@ -19,9 +19,14 @@ def main():
         parts = [page.get_text() for page in doc]
         doc.close()
         text = "\n".join(parts)
+        # hasText distinguishes a PDF with a real text layer from a scanned or
+        # image-only PDF, where every page returns "" and only the join newlines
+        # remain. len(text) alone can't tell them apart, so the caller uses this
+        # to offer OCR instead of handing back an empty file (#589).
+        has_text = any(part.strip() for part in parts)
         with open(out, "w", encoding="utf-8") as fh:
             fh.write(text)
-        print(json.dumps({"chars": len(text)}))
+        print(json.dumps({"chars": len(text), "hasText": has_text}))
     except Exception as exc:  # noqa: BLE001
         print(json.dumps({"error": str(exc)}))
         sys.exit(1)

packages/doc-engine/src/python-docs.ts

@@ -82,16 +82,24 @@ export async function pdfRedactPy(
 }
 
 /** Extract plain text from a PDF (PyMuPDF get_text). */
-export async function pdfTextPy(inPath: string, outTxtPath: string): Promise<{ chars: number }> {
+export async function pdfTextPy(
+  inPath: string,
+  outTxtPath: string,
+): Promise<{ chars: number; hasText: boolean }> {
   const stdout = await runDocsScript("doc_text", { path: inPath, out: outTxtPath });
-  const parsed = parseDocsJson<{ chars?: number; error?: string }>("doc_text", stdout);
+  const parsed = parseDocsJson<{ chars?: number; hasText?: boolean; error?: string }>(
+    "doc_text",
+    stdout,
+  );
   if (parsed.error) {
     throw new Error(`doc_text failed: ${parsed.error}`);
   }
   if (typeof parsed.chars !== "number") {
     throw new Error(`doc_text failed: ${stdout.slice(0, 200)}`);
   }
-  return { chars: parsed.chars };
+  // Fall back to a chars>0 heuristic if an older sidecar omits hasText.
+  const hasText = parsed.hasText ?? parsed.chars > 0;
+  return { chars: parsed.chars, hasText };
 }
 
 /** PDF to DOCX conversion (pdf2docx). Long-running: 5 min timeout. */

tests/helpers/python-gate.ts

@@ -22,7 +22,7 @@ function resolvePython(): string | null {
   return null;
 }
 
-const pythonBin = resolvePython();
+export const pythonBin = resolvePython();
 
 /** Check whether a Python module is importable by the resolved interpreter. */
 export function pythonWith(mod: string): boolean {

tests/integration/tools/document/pdf-to-text.test.ts

@@ -1,6 +1,10 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { fixtures, readFixture } from "../../../fixtures/index.js";
-import { hasFitz } from "../../../helpers/python-gate.js";
+import { hasFitz, pythonBin } from "../../../helpers/python-gate.js";
 import {
   buildTestApp,
   createMultipartPayload,
@@ -22,10 +26,10 @@ afterAll(async () => {
   await testApp.cleanup();
 }, 10_000);
 
-async function runTool(settings: Record<string, unknown> = {}) {
+async function runTool(content: Buffer = PDF, filename = "test-3page.pdf") {
   const { body, contentType } = createMultipartPayload([
-    { name: "file", filename: "test-3page.pdf", contentType: "application/pdf", content: PDF },
+    { name: "file", filename, contentType: "application/pdf", content },
-    { name: "settings", content: JSON.stringify(settings) },
+    { name: "settings", content: JSON.stringify({}) },
   ]);
   return testApp.app.inject({
     method: "POST",
@@ -35,19 +39,45 @@ async function runTool(settings: Record<string, unknown> = {}) {
   });
 }
 
+/** Build a one-page PDF whose only content is a rendered image, so it has no
+ *  text layer (the shape of a scanned document). Uses the same PyMuPDF that
+ *  gates this suite. */
+function makeImageOnlyPdf(): Buffer {
+  const dir = mkdtempSync(join(tmpdir(), "pdf-scan-"));
+  const out = join(dir, "scanned.pdf");
+  const script = [
+    "import sys, fitz",
+    "d = fitz.open(); p = d.new_page()",
+    "tmp = fitz.open(); tp = tmp.new_page(); tp.insert_text((50, 50), 'SCANNED PAGE')",
+    "pix = tp.get_pixmap(dpi=100); tmp.close()",
+    "p.insert_image(p.rect, pixmap=pix)",
+    "d.save(sys.argv[-1]); d.close()",
+  ].join("\n");
+  const res = spawnSync(pythonBin as string, ["-c", script, out], { encoding: "utf8" });
+  if (res.status !== 0) throw new Error(`could not build image-only PDF: ${res.stderr}`);
+  return readFileSync(out);
+}
+
 describe.skipIf(!hasFitz)("pdf-to-text (requires PyMuPDF)", () => {
-  it("extracts text and returns a .txt file", async () => {
+  it("extracts text and serves the .txt as UTF-8", async () => {
     const res = await runTool();
     expect(res.statusCode).toBe(200);
     const envelope = JSON.parse(res.body);
     expect(envelope.downloadUrl).toBeDefined();
 
-    const dl = await testApp.app.inject({
+    const dl = await testApp.app.inject({ method: "GET", url: envelope.downloadUrl });
-      method: "GET",
-      url: envelope.downloadUrl,
-    });
     expect(dl.statusCode).toBe(200);
-    // The text output should have content (the 3-page fixture has text).
+    // The 3-page fixture has a text layer, so the output has content.
     expect(dl.rawPayload.length).toBeGreaterThan(0);
+    // Charset must be explicit so non-Latin scripts don't mojibake inline (#589).
+    expect(dl.headers["content-type"]).toContain("charset=utf-8");
+  }, 60_000);
+
+  it("tells the user to run OCR when the PDF has no text layer", async () => {
+    const res = await runTool(makeImageOnlyPdf(), "scanned.pdf");
+    expect(res.statusCode).toBe(422);
+    const body = JSON.parse(res.body);
+    expect(body.details).toMatch(/text layer/i);
+    expect(body.details).toMatch(/OCR/);
   }, 60_000);
 });

tests/unit/api/content-type.test.ts

@@ -0,0 +1,20 @@
+import { describe, expect, it } from "vitest";
+import { getContentType } from "../../../apps/api/src/routes/files.js";
+
+describe("getContentType charset", () => {
+  it("adds charset=utf-8 to text types so non-Latin scripts render inline", () => {
+    // Without a charset a browser can sniff a legacy encoding and mojibake
+    // UTF-8 Arabic when viewing an extracted .txt inline (#589).
+    expect(getContentType("txt")).toBe("text/plain; charset=utf-8");
+    expect(getContentType("md")).toBe("text/markdown; charset=utf-8");
+    expect(getContentType("csv")).toBe("text/csv; charset=utf-8");
+    expect(getContentType("vtt")).toBe("text/vtt; charset=utf-8");
+  });
+
+  it("leaves binary and application types unchanged", () => {
+    expect(getContentType("pdf")).toBe("application/pdf");
+    expect(getContentType("png")).toBe("image/png");
+    expect(getContentType("json")).toBe("application/json");
+    expect(getContentType("unknown-ext")).toBe("application/octet-stream");
+  });
+});

tests/unit/api/document-route-processors.test.ts

@@ -31,7 +31,7 @@ vi.mock("../../../apps/api/src/config.js", () => ({
 vi.mock("@snapotter/doc-engine", () => ({
   htmlToPdfPy: vi.fn(),
   pdfFlattenPy: vi.fn(),
-  pdfTextPy: vi.fn(async () => ({ chars: 12 })),
+  pdfTextPy: vi.fn(async () => ({ chars: 12, hasText: true })),
   qpdfAvailable: vi.fn(() => false),
   qpdfCheck: vi.fn(),
   qpdfPageCount: vi.fn(),
@@ -105,6 +105,19 @@ describe("document route processors", () => {
     });
   });
 
+  it("rejects a PDF with no text layer and points the user to OCR", async () => {
+    // A scanned/image-only PDF yields hasText=false; the route must reject with
+    // an OCR handoff instead of returning a silent empty .txt (#589).
+    vi.mocked(pdfTextPy).mockResolvedValueOnce({ chars: 0, hasText: false });
+    registerPdfToText(createMockApp());
+    const config = getToolConfig("pdf-to-text");
+
+    await withScratch(async (scratchDir) => {
+      const ctx = createCtx(scratchDir, "scanned.pdf");
+      await expect(config?.processV2?.(ctx)).rejects.toThrow(/text layer/i);
+    });
+  });
+
   it("flattens PDFs through pdfFlattenPy", async () => {
     registerFlattenPdf(createMockApp());
     const config = getToolConfig("flatten-pdf");