diff --git a/.gitattributes b/.gitattributes index 8a21a527..f7b90fe3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,6 +19,7 @@ Dockerfile* text eol=lf *.jpeg binary *.gif binary *.ico binary +*.pdf binary *.woff binary *.woff2 binary *.ttf binary diff --git a/packages/ai/python/doc_to_word.py b/packages/ai/python/doc_to_word.py index 104c95db..b9667192 100644 --- a/packages/ai/python/doc_to_word.py +++ b/packages/ai/python/doc_to_word.py @@ -13,13 +13,17 @@ def main(): sys.exit(1) try: from pdf2docx import Converter + from pdf2docx_layout import install_pdf2docx_layout_fixes except ImportError: print(json.dumps({"error": "pdf2docx not installed"})) sys.exit(1) try: + install_pdf2docx_layout_fixes() cv = Converter(path) - cv.convert(out) - cv.close() + try: + cv.convert(out, max_border_width=2.0) + finally: + cv.close() print(json.dumps({"ok": True})) except Exception as exc: # noqa: BLE001 print(json.dumps({"error": str(exc)})) diff --git a/packages/ai/python/pdf2docx_layout.py b/packages/ai/python/pdf2docx_layout.py new file mode 100644 index 00000000..ab775275 --- /dev/null +++ b/packages/ai/python/pdf2docx_layout.py @@ -0,0 +1,85 @@ +"""Targeted layout compatibility fixes for pdf2docx 0.5.13.""" + +from docx.enum.table import WD_ROW_HEIGHT +from docx.shared import Pt +from pdf2docx.common import constants +from pdf2docx.common.share import rgb_value +from pdf2docx.layout.Blocks import Blocks +from pdf2docx.table.Row import Row +from pdf2docx.table.TableBlock import TableBlock + + +_original_collect_stream_lines = Blocks.collect_stream_lines +_original_table_make_docx = TableBlock.make_docx +_original_row_make_docx = Row.make_docx +_installed = False + + +def _collect_stream_lines( + self, potential_shadings, line_separate_threshold, **kwargs +): + groups = _original_collect_stream_lines( + self, potential_shadings, line_separate_threshold, **kwargs + ) + fills = [ + shape + for shape in potential_shadings + if not shape.is_determined and shape.color != rgb_value((1, 1, 1)) + ] + merged = [] + active_fill_ids = set() + + for group in groups: + covered_fill_ids = { + id(fill) + for fill in fills + if any( + fill.contains(line, threshold=constants.FACTOR_MOST) + for line in group + ) + } + if ( + merged + and covered_fill_ids + and active_fill_ids.intersection(covered_fill_ids) + ): + merged[-1].extend(group) + active_fill_ids.update(covered_fill_ids) + else: + merged.append(group) + active_fill_ids = covered_fill_ids + + return merged + + +def _make_table_docx(self, table): + _original_table_make_docx(self, table) + + if not self.is_stream_table_block or self.num_cols != 1: + return + + cells = [cell for row in self for cell in row if cell] + if not cells or any(cell.bg_color is None for cell in cells): + return + + table.columns[0].width = Pt(max(cell.bbox.width for cell in cells)) + + +def _make_row_docx(self, table, idx_row): + _original_row_make_docx(self, table, idx_row) + + if any(cell and cell.bg_color is not None for cell in self): + table.rows[idx_row].height_rule = WD_ROW_HEIGHT.AT_LEAST + + +def install_pdf2docx_layout_fixes() -> None: + """Install the pdf2docx layout compatibility wrappers exactly once.""" + global _installed + + if _installed: + return + + Blocks.collect_stream_lines = _collect_stream_lines + TableBlock.make_docx = _make_table_docx + Row.make_docx = _make_row_docx + _installed = True diff --git a/tests/fixtures/document/edge/colored-block.pdf b/tests/fixtures/document/edge/colored-block.pdf new file mode 100644 index 00000000..4884a7ab --- /dev/null +++ b/tests/fixtures/document/edge/colored-block.pdf @@ -0,0 +1,48 @@ +%PDF-1.7 +% + +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj + +2 0 obj +<< /Type /Pages /Kids [5 0 R] /Count 1 >> +endobj + +3 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj + +4 0 obj +<< /Length 120 >> +stream +0.12 0.52 0.72 rg +72 520 240 65 re f +BT +/F1 14 Tf +1 1 1 rg +84 562 Td +(BLOCK LINE ONE) Tj +0 -24 Td +(BLOCK LINE TWO) Tj +ET +endstream +endobj + +5 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 3 0 R >> >> >> +endobj + +xref +0 6 +0000000000 65535 f +0000000016 00000 n +0000000066 00000 n +0000000124 00000 n +0000000195 00000 n +0000000367 00000 n +trailer +<< /Size 6 /Root 1 0 R >> +startxref +494 +%%EOF diff --git a/tests/fixtures/gen-synthetic-content.mjs b/tests/fixtures/gen-synthetic-content.mjs index 7a39f355..62ef2b84 100644 --- a/tests/fixtures/gen-synthetic-content.mjs +++ b/tests/fixtures/gen-synthetic-content.mjs @@ -23,6 +23,7 @@ import { createRequire } from "node:module"; const __dirname = dirname(fileURLToPath(import.meta.url)); const IMAGE_VALID = join(__dirname, "image/valid"); const DOC_VALID = join(__dirname, "document/valid"); +const DOC_EDGE = join(__dirname, "document/edge"); const require = createRequire(join(__dirname, "../../apps/api/package.json")); const sharp = require("sharp"); const QRCode = require("qrcode"); @@ -111,7 +112,10 @@ function barcodeSvg(text, width, height) { // ── Minimal PDF generator ────────────────────────────────── -function generatePdf(pageCount) { +function generatePdf( + pageCount, + contentForPage = (page) => `BT /F1 24 Tf 72 700 Td (Page ${page}) Tj ET`, +) { const objects = []; let nextObj = 1; const addObj = (content) => { @@ -126,7 +130,7 @@ function generatePdf(pageCount) { const pageIds = []; for (let i = 1; i <= pageCount; i++) { - const stream = `BT /F1 24 Tf 72 700 Td (Page ${i}) Tj ET`; + const stream = contentForPage(i); const sId = addObj(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`); const pId = addObj( `<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] ` + @@ -157,6 +161,19 @@ function generatePdf(pageCount) { return Buffer.from(pdf, "binary"); } +const COLORED_BLOCK_STREAM = [ + "0.12 0.52 0.72 rg", + "72 520 240 65 re f", + "BT", + "/F1 14 Tf", + "1 1 1 rg", + "84 562 Td", + "(BLOCK LINE ONE) Tj", + "0 -24 Td", + "(BLOCK LINE TWO) Tj", + "ET", +].join("\n"); + // ── SVG logo (project-owned, replaces ConvertICO brand) ─── function projectSvg() { @@ -310,6 +327,11 @@ async function main() { console.log("PDFs:"); writeIfMissing(join(DOC_VALID, "alt-2page.pdf"), generatePdf(2), "alt-2page.pdf"); writeIfMissing(join(DOC_VALID, "multipage-6.pdf"), generatePdf(6), "multipage-6.pdf"); + writeIfMissing( + join(DOC_EDGE, "colored-block.pdf"), + generatePdf(1, () => COLORED_BLOCK_STREAM), + "colored-block.pdf", + ); // ── SVG logo (B: replaces ConvertICO brand) ── console.log("SVG logo (replacing ConvertICO brand):"); diff --git a/tests/fixtures/index.ts b/tests/fixtures/index.ts index 57d3e488..00b6f6f4 100644 --- a/tests/fixtures/index.ts +++ b/tests/fixtures/index.ts @@ -140,6 +140,7 @@ export const fixtures = { pdf3: p("document/valid/test-3page.pdf"), pdfScanned: p("document/valid/ocr-scanned.pdf"), encrypted: p("document/valid/encrypted.pdf"), + coloredBlock: p("document/edge/colored-block.pdf"), tiny: (ext: string) => p(`document/formats/tiny.${ext}`), remoteImgHtml: p("document/edge/remote-img.html"), hostile: { diff --git a/tests/integration/tools/document/pdf-to-word.test.ts b/tests/integration/tools/document/pdf-to-word.test.ts index d92c3a24..50360553 100644 --- a/tests/integration/tools/document/pdf-to-word.test.ts +++ b/tests/integration/tools/document/pdf-to-word.test.ts @@ -3,6 +3,8 @@ // the Task 13 Docker compose smoke is the real proof. Uses the 202+poll // pattern because pdf-to-word has executionHint "long". +import AdmZip from "adm-zip"; +import { JSDOM } from "jsdom"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { fixtures, readFixture } from "../../../fixtures/index.js"; import { pythonWith } from "../../../helpers/python-gate.js"; @@ -14,6 +16,7 @@ import { } from "../../test-server.js"; const PDF = readFixture(fixtures.document.pdf3); +const COLORED_BLOCK_PDF = readFixture(fixtures.document.coloredBlock); const hasPdf2docx = pythonWith("pdf2docx"); let testApp: TestApp; @@ -28,9 +31,9 @@ afterAll(async () => { await testApp.cleanup(); }, 10_000); -async function runTool(settings: Record = {}) { +async function runTool(pdf: Buffer, filename: string, settings: Record = {}) { const { body, contentType } = createMultipartPayload([ - { name: "file", filename: "test-3page.pdf", contentType: "application/pdf", content: PDF }, + { name: "file", filename, contentType: "application/pdf", content: pdf }, { name: "settings", content: JSON.stringify(settings) }, ]); return testApp.app.inject({ @@ -41,28 +44,108 @@ async function runTool(settings: Record = {}) { }); } +async function downloadCompletedDocx( + pdf: Buffer, + filename: string, + settings: Record = {}, +): Promise { + const res = await runTool(pdf, filename, settings); + expect(res.statusCode).toBe(202); + const { jobId } = JSON.parse(res.body); + // Poll the durable row until terminal (the long hint skips the sync window). + const { db, schema } = await import("../../../../apps/api/src/db/index.js"); + const { eq } = await import("drizzle-orm"); + let row: { status: string; outputRefs: unknown } | undefined; + for (let i = 0; i < 120; i++) { + [row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)); + if (row && ["completed", "failed", "canceled"].includes(row.status)) break; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + expect(row?.status).toBe("completed"); + const outName = (row?.outputRefs as string[])[0].split("/").pop() as string; + const download = await testApp.app.inject({ + method: "GET", + url: `/api/v1/download/${jobId}/${encodeURIComponent(outName)}`, + }); + expect(download.statusCode).toBe(200); + return download.rawPayload; +} + +function ancestorByTagName(element: Element, tagName: string): Element | null { + let current: Element | null = element; + while (current) { + if (current.tagName === tagName) return current; + current = current.parentElement; + } + return null; +} + describe.skipIf(!hasPdf2docx)("pdf-to-word (requires pdf2docx)", () => { it("returns 202 (long hint) and the job completes with a docx", async () => { - const res = await runTool(); - expect(res.statusCode).toBe(202); - const { jobId } = JSON.parse(res.body); - // Poll the durable row until terminal (the long hint skips the sync window). - const { db, schema } = await import("../../../../apps/api/src/db/index.js"); - const { eq } = await import("drizzle-orm"); - let row: { status: string; outputRefs: unknown } | undefined; - for (let i = 0; i < 120; i++) { - [row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)); - if (row && ["completed", "failed", "canceled"].includes(row.status)) break; - await new Promise((r) => setTimeout(r, 500)); - } - expect(row?.status).toBe("completed"); - const outName = (row?.outputRefs as string[])[0].split("/").pop() as string; - const dl = await testApp.app.inject({ - method: "GET", - url: `/api/v1/download/${jobId}/${encodeURIComponent(outName)}`, - }); - expect(dl.statusCode).toBe(200); + const docx = await downloadCompletedDocx(PDF, "test-3page.pdf"); // DOCX files are ZIP archives; PK magic bytes. - expect(dl.rawPayload.subarray(0, 2).toString()).toBe("PK"); + expect(docx.subarray(0, 2).toString()).toBe("PK"); + }, 90_000); + + it("preserves a multi-line colored block as one table with flexible row height", async () => { + const docx = await downloadCompletedDocx(COLORED_BLOCK_PDF, "colored-block.pdf"); + const zip = new AdmZip(docx); + const documentEntry = zip.getEntry("word/document.xml"); + if (!documentEntry) throw new Error("DOCX is missing word/document.xml"); + + const document = new JSDOM(documentEntry.getData().toString("utf8"), { + contentType: "text/xml", + // Keep failed DOM equality diagnostics away from opaque-origin storage. + url: "http://localhost/", + }).window.document; + const textNodes = Array.from(document.getElementsByTagName("w:t")); + const lineOneText = textNodes.find((node) => node.textContent?.trim() === "BLOCK LINE ONE"); + const lineTwoText = textNodes.find((node) => node.textContent?.trim() === "BLOCK LINE TWO"); + expect(lineOneText).toBeDefined(); + expect(lineTwoText).toBeDefined(); + if (!lineOneText || !lineTwoText) throw new Error("DOCX is missing colored-block text"); + + const lineOneCell = ancestorByTagName(lineOneText, "w:tc"); + const lineTwoCell = ancestorByTagName(lineTwoText, "w:tc"); + const lineOneRow = ancestorByTagName(lineOneText, "w:tr"); + const lineTwoRow = ancestorByTagName(lineTwoText, "w:tr"); + const lineOneTable = ancestorByTagName(lineOneText, "w:tbl"); + const lineTwoTable = ancestorByTagName(lineTwoText, "w:tbl"); + expect(lineOneCell).not.toBeNull(); + expect(lineTwoCell).not.toBeNull(); + expect(lineOneRow).not.toBeNull(); + expect(lineTwoRow).not.toBeNull(); + expect(lineOneTable).not.toBeNull(); + expect(lineTwoTable).not.toBeNull(); + + expect(lineOneTable).toBe(lineTwoTable); + expect(lineOneTable?.getElementsByTagName("w:tblpPr")).toHaveLength(0); + const gridColumns = Array.from(lineOneTable?.getElementsByTagName("w:gridCol") ?? []); + expect(gridColumns).toHaveLength(1); + const gridColumn = gridColumns[0]; + expect(Number(gridColumn?.getAttribute("w:w"))).toBeCloseTo(4_800, -1); + + const targetCells = new Set( + [lineOneCell, lineTwoCell].filter((cell): cell is Element => cell !== null), + ); + for (const cell of targetCells) { + const shading = cell.getElementsByTagName("w:shd")[0]; + const width = cell.getElementsByTagName("w:tcW")[0]; + expect(shading?.getAttribute("w:fill")?.toLowerCase()).toBe("1e84b7"); + expect(Number(width?.getAttribute("w:w"))).toBeCloseTo(4_800, -1); + } + + const targetRows = new Set( + [lineOneRow, lineTwoRow].filter((row): row is Element => row !== null), + ); + let totalRowHeight = 0; + for (const row of targetRows) { + const height = row.getElementsByTagName("w:trHeight")[0]; + const heightRule = height?.getAttribute("w:hRule"); + expect(heightRule).not.toBe("exact"); + if (heightRule) expect(heightRule).toBe("atLeast"); + totalRowHeight += Number(height?.getAttribute("w:val")); + } + expect(Math.abs(totalRowHeight - 1_300)).toBeLessThanOrEqual(40); }, 90_000); });