mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: preserve colored blocks in PDF-to-Word (#500)
This commit is contained in:
SnapOtter
@@ -19,6 +19,7 @@ Dockerfile* text eol=lf
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.ico binary
|
||||
*.pdf binary
|
||||
*.woff binary
|
||||
*.woff2 binary
|
||||
*.ttf binary
|
||||
|
||||
@@ -13,13 +13,17 @@ def main():
|
||||
sys.exit(1)
|
||||
try:
|
||||
from pdf2docx import Converter
|
||||
from pdf2docx_layout import install_pdf2docx_layout_fixes
|
||||
except ImportError:
|
||||
print(json.dumps({"error": "pdf2docx not installed"}))
|
||||
sys.exit(1)
|
||||
try:
|
||||
install_pdf2docx_layout_fixes()
|
||||
cv = Converter(path)
|
||||
cv.convert(out)
|
||||
cv.close()
|
||||
try:
|
||||
cv.convert(out, max_border_width=2.0)
|
||||
finally:
|
||||
cv.close()
|
||||
print(json.dumps({"ok": True}))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(json.dumps({"error": str(exc)}))
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Targeted layout compatibility fixes for pdf2docx 0.5.13."""
|
||||
|
||||
from docx.enum.table import WD_ROW_HEIGHT
|
||||
from docx.shared import Pt
|
||||
from pdf2docx.common import constants
|
||||
from pdf2docx.common.share import rgb_value
|
||||
from pdf2docx.layout.Blocks import Blocks
|
||||
from pdf2docx.table.Row import Row
|
||||
from pdf2docx.table.TableBlock import TableBlock
|
||||
|
||||
|
||||
_original_collect_stream_lines = Blocks.collect_stream_lines
|
||||
_original_table_make_docx = TableBlock.make_docx
|
||||
_original_row_make_docx = Row.make_docx
|
||||
_installed = False
|
||||
|
||||
|
||||
def _collect_stream_lines(
|
||||
self, potential_shadings, line_separate_threshold, **kwargs
|
||||
):
|
||||
groups = _original_collect_stream_lines(
|
||||
self, potential_shadings, line_separate_threshold, **kwargs
|
||||
)
|
||||
fills = [
|
||||
shape
|
||||
for shape in potential_shadings
|
||||
if not shape.is_determined and shape.color != rgb_value((1, 1, 1))
|
||||
]
|
||||
merged = []
|
||||
active_fill_ids = set()
|
||||
|
||||
for group in groups:
|
||||
covered_fill_ids = {
|
||||
id(fill)
|
||||
for fill in fills
|
||||
if any(
|
||||
fill.contains(line, threshold=constants.FACTOR_MOST)
|
||||
for line in group
|
||||
)
|
||||
}
|
||||
if (
|
||||
merged
|
||||
and covered_fill_ids
|
||||
and active_fill_ids.intersection(covered_fill_ids)
|
||||
):
|
||||
merged[-1].extend(group)
|
||||
active_fill_ids.update(covered_fill_ids)
|
||||
else:
|
||||
merged.append(group)
|
||||
active_fill_ids = covered_fill_ids
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
def _make_table_docx(self, table):
|
||||
_original_table_make_docx(self, table)
|
||||
|
||||
if not self.is_stream_table_block or self.num_cols != 1:
|
||||
return
|
||||
|
||||
cells = [cell for row in self for cell in row if cell]
|
||||
if not cells or any(cell.bg_color is None for cell in cells):
|
||||
return
|
||||
|
||||
table.columns[0].width = Pt(max(cell.bbox.width for cell in cells))
|
||||
|
||||
|
||||
def _make_row_docx(self, table, idx_row):
|
||||
_original_row_make_docx(self, table, idx_row)
|
||||
|
||||
if any(cell and cell.bg_color is not None for cell in self):
|
||||
table.rows[idx_row].height_rule = WD_ROW_HEIGHT.AT_LEAST
|
||||
|
||||
|
||||
def install_pdf2docx_layout_fixes() -> None:
|
||||
"""Install the pdf2docx layout compatibility wrappers exactly once."""
|
||||
global _installed
|
||||
|
||||
if _installed:
|
||||
return
|
||||
|
||||
Blocks.collect_stream_lines = _collect_stream_lines
|
||||
TableBlock.make_docx = _make_table_docx
|
||||
Row.make_docx = _make_row_docx
|
||||
_installed = True
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
%PDF-1.7
|
||||
%����
|
||||
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [5 0 R] /Count 1 >>
|
||||
endobj
|
||||
|
||||
3 0 obj
|
||||
<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
|
||||
endobj
|
||||
|
||||
4 0 obj
|
||||
<< /Length 120 >>
|
||||
stream
|
||||
0.12 0.52 0.72 rg
|
||||
72 520 240 65 re f
|
||||
BT
|
||||
/F1 14 Tf
|
||||
1 1 1 rg
|
||||
84 562 Td
|
||||
(BLOCK LINE ONE) Tj
|
||||
0 -24 Td
|
||||
(BLOCK LINE TWO) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
|
||||
5 0 obj
|
||||
<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 3 0 R >> >> >>
|
||||
endobj
|
||||
|
||||
xref
|
||||
0 6
|
||||
0000000000 65535 f
|
||||
0000000016 00000 n
|
||||
0000000066 00000 n
|
||||
0000000124 00000 n
|
||||
0000000195 00000 n
|
||||
0000000367 00000 n
|
||||
trailer
|
||||
<< /Size 6 /Root 1 0 R >>
|
||||
startxref
|
||||
494
|
||||
%%EOF
|
||||
+24
-2
@@ -23,6 +23,7 @@ import { createRequire } from "node:module";
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const IMAGE_VALID = join(__dirname, "image/valid");
|
||||
const DOC_VALID = join(__dirname, "document/valid");
|
||||
const DOC_EDGE = join(__dirname, "document/edge");
|
||||
const require = createRequire(join(__dirname, "../../apps/api/package.json"));
|
||||
const sharp = require("sharp");
|
||||
const QRCode = require("qrcode");
|
||||
@@ -111,7 +112,10 @@ function barcodeSvg(text, width, height) {
|
||||
|
||||
// ── Minimal PDF generator ──────────────────────────────────
|
||||
|
||||
function generatePdf(pageCount) {
|
||||
function generatePdf(
|
||||
pageCount,
|
||||
contentForPage = (page) => `BT /F1 24 Tf 72 700 Td (Page ${page}) Tj ET`,
|
||||
) {
|
||||
const objects = [];
|
||||
let nextObj = 1;
|
||||
const addObj = (content) => {
|
||||
@@ -126,7 +130,7 @@ function generatePdf(pageCount) {
|
||||
|
||||
const pageIds = [];
|
||||
for (let i = 1; i <= pageCount; i++) {
|
||||
const stream = `BT /F1 24 Tf 72 700 Td (Page ${i}) Tj ET`;
|
||||
const stream = contentForPage(i);
|
||||
const sId = addObj(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
|
||||
const pId = addObj(
|
||||
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] ` +
|
||||
@@ -157,6 +161,19 @@ function generatePdf(pageCount) {
|
||||
return Buffer.from(pdf, "binary");
|
||||
}
|
||||
|
||||
const COLORED_BLOCK_STREAM = [
|
||||
"0.12 0.52 0.72 rg",
|
||||
"72 520 240 65 re f",
|
||||
"BT",
|
||||
"/F1 14 Tf",
|
||||
"1 1 1 rg",
|
||||
"84 562 Td",
|
||||
"(BLOCK LINE ONE) Tj",
|
||||
"0 -24 Td",
|
||||
"(BLOCK LINE TWO) Tj",
|
||||
"ET",
|
||||
].join("\n");
|
||||
|
||||
// ── SVG logo (project-owned, replaces ConvertICO brand) ───
|
||||
|
||||
function projectSvg() {
|
||||
@@ -310,6 +327,11 @@ async function main() {
|
||||
console.log("PDFs:");
|
||||
writeIfMissing(join(DOC_VALID, "alt-2page.pdf"), generatePdf(2), "alt-2page.pdf");
|
||||
writeIfMissing(join(DOC_VALID, "multipage-6.pdf"), generatePdf(6), "multipage-6.pdf");
|
||||
writeIfMissing(
|
||||
join(DOC_EDGE, "colored-block.pdf"),
|
||||
generatePdf(1, () => COLORED_BLOCK_STREAM),
|
||||
"colored-block.pdf",
|
||||
);
|
||||
|
||||
// ── SVG logo (B: replaces ConvertICO brand) ──
|
||||
console.log("SVG logo (replacing ConvertICO brand):");
|
||||
|
||||
Vendored
+1
@@ -140,6 +140,7 @@ export const fixtures = {
|
||||
pdf3: p("document/valid/test-3page.pdf"),
|
||||
pdfScanned: p("document/valid/ocr-scanned.pdf"),
|
||||
encrypted: p("document/valid/encrypted.pdf"),
|
||||
coloredBlock: p("document/edge/colored-block.pdf"),
|
||||
tiny: (ext: string) => p(`document/formats/tiny.${ext}`),
|
||||
remoteImgHtml: p("document/edge/remote-img.html"),
|
||||
hostile: {
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
// the Task 13 Docker compose smoke is the real proof. Uses the 202+poll
|
||||
// pattern because pdf-to-word has executionHint "long".
|
||||
|
||||
import AdmZip from "adm-zip";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { fixtures, readFixture } from "../../../fixtures/index.js";
|
||||
import { pythonWith } from "../../../helpers/python-gate.js";
|
||||
@@ -14,6 +16,7 @@ import {
|
||||
} from "../../test-server.js";
|
||||
|
||||
const PDF = readFixture(fixtures.document.pdf3);
|
||||
const COLORED_BLOCK_PDF = readFixture(fixtures.document.coloredBlock);
|
||||
const hasPdf2docx = pythonWith("pdf2docx");
|
||||
|
||||
let testApp: TestApp;
|
||||
@@ -28,9 +31,9 @@ afterAll(async () => {
|
||||
await testApp.cleanup();
|
||||
}, 10_000);
|
||||
|
||||
async function runTool(settings: Record<string, unknown> = {}) {
|
||||
async function runTool(pdf: Buffer, filename: string, settings: Record<string, unknown> = {}) {
|
||||
const { body, contentType } = createMultipartPayload([
|
||||
{ name: "file", filename: "test-3page.pdf", contentType: "application/pdf", content: PDF },
|
||||
{ name: "file", filename, contentType: "application/pdf", content: pdf },
|
||||
{ name: "settings", content: JSON.stringify(settings) },
|
||||
]);
|
||||
return testApp.app.inject({
|
||||
@@ -41,28 +44,108 @@ async function runTool(settings: Record<string, unknown> = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
async function downloadCompletedDocx(
|
||||
pdf: Buffer,
|
||||
filename: string,
|
||||
settings: Record<string, unknown> = {},
|
||||
): Promise<Buffer> {
|
||||
const res = await runTool(pdf, filename, settings);
|
||||
expect(res.statusCode).toBe(202);
|
||||
const { jobId } = JSON.parse(res.body);
|
||||
// Poll the durable row until terminal (the long hint skips the sync window).
|
||||
const { db, schema } = await import("../../../../apps/api/src/db/index.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
let row: { status: string; outputRefs: unknown } | undefined;
|
||||
for (let i = 0; i < 120; i++) {
|
||||
[row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId));
|
||||
if (row && ["completed", "failed", "canceled"].includes(row.status)) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
expect(row?.status).toBe("completed");
|
||||
const outName = (row?.outputRefs as string[])[0].split("/").pop() as string;
|
||||
const download = await testApp.app.inject({
|
||||
method: "GET",
|
||||
url: `/api/v1/download/${jobId}/${encodeURIComponent(outName)}`,
|
||||
});
|
||||
expect(download.statusCode).toBe(200);
|
||||
return download.rawPayload;
|
||||
}
|
||||
|
||||
function ancestorByTagName(element: Element, tagName: string): Element | null {
|
||||
let current: Element | null = element;
|
||||
while (current) {
|
||||
if (current.tagName === tagName) return current;
|
||||
current = current.parentElement;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPdf2docx)("pdf-to-word (requires pdf2docx)", () => {
|
||||
it("returns 202 (long hint) and the job completes with a docx", async () => {
|
||||
const res = await runTool();
|
||||
expect(res.statusCode).toBe(202);
|
||||
const { jobId } = JSON.parse(res.body);
|
||||
// Poll the durable row until terminal (the long hint skips the sync window).
|
||||
const { db, schema } = await import("../../../../apps/api/src/db/index.js");
|
||||
const { eq } = await import("drizzle-orm");
|
||||
let row: { status: string; outputRefs: unknown } | undefined;
|
||||
for (let i = 0; i < 120; i++) {
|
||||
[row] = await db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId));
|
||||
if (row && ["completed", "failed", "canceled"].includes(row.status)) break;
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
expect(row?.status).toBe("completed");
|
||||
const outName = (row?.outputRefs as string[])[0].split("/").pop() as string;
|
||||
const dl = await testApp.app.inject({
|
||||
method: "GET",
|
||||
url: `/api/v1/download/${jobId}/${encodeURIComponent(outName)}`,
|
||||
});
|
||||
expect(dl.statusCode).toBe(200);
|
||||
const docx = await downloadCompletedDocx(PDF, "test-3page.pdf");
|
||||
// DOCX files are ZIP archives; PK magic bytes.
|
||||
expect(dl.rawPayload.subarray(0, 2).toString()).toBe("PK");
|
||||
expect(docx.subarray(0, 2).toString()).toBe("PK");
|
||||
}, 90_000);
|
||||
|
||||
it("preserves a multi-line colored block as one table with flexible row height", async () => {
|
||||
const docx = await downloadCompletedDocx(COLORED_BLOCK_PDF, "colored-block.pdf");
|
||||
const zip = new AdmZip(docx);
|
||||
const documentEntry = zip.getEntry("word/document.xml");
|
||||
if (!documentEntry) throw new Error("DOCX is missing word/document.xml");
|
||||
|
||||
const document = new JSDOM(documentEntry.getData().toString("utf8"), {
|
||||
contentType: "text/xml",
|
||||
// Keep failed DOM equality diagnostics away from opaque-origin storage.
|
||||
url: "http://localhost/",
|
||||
}).window.document;
|
||||
const textNodes = Array.from(document.getElementsByTagName("w:t"));
|
||||
const lineOneText = textNodes.find((node) => node.textContent?.trim() === "BLOCK LINE ONE");
|
||||
const lineTwoText = textNodes.find((node) => node.textContent?.trim() === "BLOCK LINE TWO");
|
||||
expect(lineOneText).toBeDefined();
|
||||
expect(lineTwoText).toBeDefined();
|
||||
if (!lineOneText || !lineTwoText) throw new Error("DOCX is missing colored-block text");
|
||||
|
||||
const lineOneCell = ancestorByTagName(lineOneText, "w:tc");
|
||||
const lineTwoCell = ancestorByTagName(lineTwoText, "w:tc");
|
||||
const lineOneRow = ancestorByTagName(lineOneText, "w:tr");
|
||||
const lineTwoRow = ancestorByTagName(lineTwoText, "w:tr");
|
||||
const lineOneTable = ancestorByTagName(lineOneText, "w:tbl");
|
||||
const lineTwoTable = ancestorByTagName(lineTwoText, "w:tbl");
|
||||
expect(lineOneCell).not.toBeNull();
|
||||
expect(lineTwoCell).not.toBeNull();
|
||||
expect(lineOneRow).not.toBeNull();
|
||||
expect(lineTwoRow).not.toBeNull();
|
||||
expect(lineOneTable).not.toBeNull();
|
||||
expect(lineTwoTable).not.toBeNull();
|
||||
|
||||
expect(lineOneTable).toBe(lineTwoTable);
|
||||
expect(lineOneTable?.getElementsByTagName("w:tblpPr")).toHaveLength(0);
|
||||
const gridColumns = Array.from(lineOneTable?.getElementsByTagName("w:gridCol") ?? []);
|
||||
expect(gridColumns).toHaveLength(1);
|
||||
const gridColumn = gridColumns[0];
|
||||
expect(Number(gridColumn?.getAttribute("w:w"))).toBeCloseTo(4_800, -1);
|
||||
|
||||
const targetCells = new Set(
|
||||
[lineOneCell, lineTwoCell].filter((cell): cell is Element => cell !== null),
|
||||
);
|
||||
for (const cell of targetCells) {
|
||||
const shading = cell.getElementsByTagName("w:shd")[0];
|
||||
const width = cell.getElementsByTagName("w:tcW")[0];
|
||||
expect(shading?.getAttribute("w:fill")?.toLowerCase()).toBe("1e84b7");
|
||||
expect(Number(width?.getAttribute("w:w"))).toBeCloseTo(4_800, -1);
|
||||
}
|
||||
|
||||
const targetRows = new Set(
|
||||
[lineOneRow, lineTwoRow].filter((row): row is Element => row !== null),
|
||||
);
|
||||
let totalRowHeight = 0;
|
||||
for (const row of targetRows) {
|
||||
const height = row.getElementsByTagName("w:trHeight")[0];
|
||||
const heightRule = height?.getAttribute("w:hRule");
|
||||
expect(heightRule).not.toBe("exact");
|
||||
if (heightRule) expect(heightRule).toBe("atLeast");
|
||||
totalRowHeight += Number(height?.getAttribute("w:val"));
|
||||
}
|
||||
expect(Math.abs(totalRowHeight - 1_300)).toBeLessThanOrEqual(40);
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user