From 29fafd07227fa953e30779a4451017206ed02d14 Mon Sep 17 00:00:00 2001 From: Siddharth Kumar Sah Date: Sun, 12 Apr 2026 23:46:39 +0800 Subject: [PATCH] fix(ocr): fix PaddleOCR crashes, add multi-image and auto-detect language - Pin PaddlePaddle to 3.0.0 on ARM64 to fix segfault in PIR inference engine (3.1+ crashes on aarch64 Debian Bookworm) - Fix text extraction for PaddleOCR 3.4.x result format (rec_texts) - Add Node.js-level fallback chain (best -> balanced -> fast) when Python subprocess crashes - Add multi-image OCR: processes all uploaded files sequentially with per-file progress and filename headers in combined output - Convert input images to PNG via Sharp before OCR so HEIC, AVIF, WebP, TIFF all work transparently - Implement real auto-detect language using Tesseract multi-lang script detection (analyzes Unicode ranges for Hangul, CJK, Kana, Latin) - Default enhance to off (hurts clean digital images) --- apps/api/src/routes/tools/ocr.ts | 78 ++++++-- .../web/src/components/tools/ocr-settings.tsx | 181 +++++++++++------- docker/Dockerfile | 2 +- packages/ai/python/ocr.py | 109 ++++++++--- packages/ai/python/requirements.txt | 2 +- packages/ai/src/ocr.ts | 7 +- 6 files changed, 259 insertions(+), 120 deletions(-) diff --git a/apps/api/src/routes/tools/ocr.ts b/apps/api/src/routes/tools/ocr.ts index 01997090..57999870 100644 --- a/apps/api/src/routes/tools/ocr.ts +++ b/apps/api/src/routes/tools/ocr.ts @@ -103,30 +103,66 @@ export function registerOcr(app: FastifyInstance) { } : undefined; - const result = await extractText( - fileBuffer, - workspacePath, - { - quality, - language: settings.language, - enhance: settings.enhance, - }, - onProgress, - ); + // Fallback chain: best -> balanced -> fast + // PaddleOCR can crash with segfault on some platforms, so we retry + // with a lower quality tier at the Node.js level. + const fallbackChain: Array<"fast" | "balanced" | "best"> = + quality === "best" + ? ["best", "balanced", "fast"] + : quality === "balanced" + ? ["balanced", "fast"] + : ["fast"]; - if (clientJobId) { - updateSingleFileProgress({ - jobId: clientJobId, - phase: "complete", - percent: 100, - }); + let lastError: unknown; + for (const tier of fallbackChain) { + try { + const result = await extractText( + fileBuffer, + workspacePath, + { + quality: tier, + language: settings.language, + enhance: settings.enhance, + }, + onProgress, + ); + + if (clientJobId) { + updateSingleFileProgress({ + jobId: clientJobId, + phase: "complete", + percent: 100, + }); + } + + return reply.send({ + jobId, + filename, + text: result.text, + }); + } catch (err) { + lastError = err; + const msg = err instanceof Error ? err.message : String(err); + // If the Python process crashed (segfault, dispatcher exit), try next tier + if ( + msg.includes("exited unexpectedly") || + msg.includes("exited with code") || + msg.includes("Segmentation fault") + ) { + request.log.warn( + { toolId: "ocr", quality: tier, err }, + `OCR ${tier} crashed, falling back`, + ); + if (onProgress) onProgress(15, "Retrying..."); + continue; + } + // Non-crash errors (validation, timeout) should not retry + throw err; + } } - return reply.send({ - jobId, - filename, - text: result.text, - }); + // All tiers failed + throw lastError; } catch (err) { request.log.error({ err, toolId: "ocr" }, "OCR failed"); return reply.status(422).send({ diff --git a/apps/web/src/components/tools/ocr-settings.tsx b/apps/web/src/components/tools/ocr-settings.tsx index 25d9ba15..a6da74a3 100644 --- a/apps/web/src/components/tools/ocr-settings.tsx +++ b/apps/web/src/components/tools/ocr-settings.tsx @@ -25,8 +25,8 @@ const LANGUAGES = [ ]; const ENHANCE_DEFAULTS: Record = { - fast: true, - balanced: true, + fast: false, + balanced: false, best: false, }; @@ -38,12 +38,73 @@ function SectionLabel({ children }: { children: React.ReactNode }) { ); } +/** Send one file to the OCR API and return the extracted text. */ +function ocrOneFile( + file: File, + settings: { quality: string; language: string; enhance: boolean }, + callbacks: { + onUploadProgress: (pct: number) => void; + onProcessingProgress: (pct: number, stage: string) => void; + }, +): Promise { + return new Promise((resolve, reject) => { + const clientJobId = generateId(); + + const es = new EventSource(`/api/v1/jobs/${clientJobId}/progress`); + es.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + if (data.type === "single" && typeof data.percent === "number") { + callbacks.onProcessingProgress(data.percent, data.stage); + } + } catch {} + }; + es.onerror = () => es.close(); + + const formData = new FormData(); + formData.append("file", file); + formData.append("settings", JSON.stringify(settings)); + formData.append("clientJobId", clientJobId); + + const xhr = new XMLHttpRequest(); + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) callbacks.onUploadProgress((e.loaded / e.total) * 100); + }; + xhr.onload = () => { + es.close(); + if (xhr.status >= 200 && xhr.status < 300) { + try { + resolve(JSON.parse(xhr.responseText).text ?? ""); + } catch { + reject(new Error("Invalid response")); + } + } else { + try { + const body = JSON.parse(xhr.responseText); + reject(new Error(body.error || body.details || `Failed: ${xhr.status}`)); + } catch { + reject(new Error(`Processing failed: ${xhr.status}`)); + } + } + }; + xhr.onerror = () => { + es.close(); + reject(new Error("Network error")); + }; + xhr.open("POST", "/api/v1/tools/ocr"); + for (const [key, value] of formatHeaders()) { + xhr.setRequestHeader(key, value); + } + xhr.send(formData); + }); +} + export function OcrSettings() { const { files, processing, error, setProcessing, setError } = useFileStore(); const [quality, setQuality] = useState("balanced"); const [language, setLanguage] = useState("auto"); - const [enhance, setEnhance] = useState(true); + const [enhance, setEnhance] = useState(false); const [enhanceManuallySet, setEnhanceManuallySet] = useState(false); const [langOpen, setLangOpen] = useState(false); @@ -57,10 +118,7 @@ export function OcrSettings() { const handleQualityChange = (q: OcrQuality) => { setQuality(q); - // Update enhance default unless user has manually toggled it - if (!enhanceManuallySet) { - setEnhance(ENHANCE_DEFAULTS[q]); - } + if (!enhanceManuallySet) setEnhance(ENHANCE_DEFAULTS[q]); }; const handleEnhanceToggle = (checked: boolean) => { @@ -84,70 +142,50 @@ export function OcrSettings() { setElapsed(Math.floor((Date.now() - startTime) / 1000)); }, 1000); - const clientJobId = generateId(); + const settings = { quality, language, enhance }; + const total = files.length; + const results: string[] = []; + const errors: string[] = []; + + for (let i = 0; i < total; i++) { + const file = files[i]; + const prefix = total > 1 ? `[${i + 1}/${total}] ` : ""; + // Each file gets an equal share of the 0-100 progress bar + const fileBase = (i / total) * 100; + const fileShare = 100 / total; - const es = new EventSource(`/api/v1/jobs/${clientJobId}/progress`); - es.onmessage = (event) => { try { - const data = JSON.parse(event.data); - if (data.type === "single" && typeof data.percent === "number") { - setProgressPhase("processing"); - setProgressPercent(15 + (data.percent / 100) * 85); - setProgressStage(data.stage); - } - } catch {} - }; - es.onerror = () => es.close(); - - const formData = new FormData(); - formData.append("file", files[0]); - formData.append("settings", JSON.stringify({ quality, language, enhance })); - formData.append("clientJobId", clientJobId); - - const xhr = new XMLHttpRequest(); - xhr.upload.onprogress = (e) => { - if (e.lengthComputable) { - setProgressPercent((e.loaded / e.total) * 15); + const text = await ocrOneFile(file, settings, { + onUploadProgress: (pct) => { + setProgressPhase("uploading"); + setProgressPercent(fileBase + (pct / 100) * fileShare * 0.15); + setProgressStage(`${prefix}Uploading...`); + }, + onProcessingProgress: (pct, stage) => { + setProgressPhase("processing"); + setProgressPercent(fileBase + fileShare * 0.15 + (pct / 100) * fileShare * 0.85); + setProgressStage(`${prefix}${stage}`); + }, + }); + results.push(total > 1 ? `--- ${file.name} ---\n${text || "(no text detected)"}` : text); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + errors.push(`${file.name}: ${msg}`); + results.push(total > 1 ? `--- ${file.name} ---\n(error: ${msg})` : ""); } - }; - xhr.upload.onload = () => { - setProgressPhase("processing"); - setProgressPercent(15); - setProgressStage("Starting..."); - }; - xhr.onload = () => { - if (elapsedRef.current) clearInterval(elapsedRef.current); - es.close(); - if (xhr.status >= 200 && xhr.status < 300) { - try { - const data = JSON.parse(xhr.responseText); - setText(data.text ?? ""); - } catch { - setError("Invalid response"); - } - } else { - try { - const body = JSON.parse(xhr.responseText); - setError(body.error || body.details || `Failed: ${xhr.status}`); - } catch { - setError(`Processing failed: ${xhr.status}`); - } - } - setProcessing(false); - setProgressPhase("idle"); - }; - xhr.onerror = () => { - if (elapsedRef.current) clearInterval(elapsedRef.current); - es.close(); - setError("Network error"); - setProcessing(false); - setProgressPhase("idle"); - }; - xhr.open("POST", "/api/v1/tools/ocr"); - formatHeaders().forEach((value, key) => { - xhr.setRequestHeader(key, value); - }); - xhr.send(formData); + } + + if (elapsedRef.current) clearInterval(elapsedRef.current); + + if (errors.length === total) { + setError(errors.join("; ")); + } else if (errors.length > 0) { + setError(`${errors.length} of ${total} files failed`); + } + + setText(results.join("\n\n")); + setProcessing(false); + setProgressPhase("idle"); }; const handleCopy = async () => { @@ -165,7 +203,8 @@ export function OcrSettings() { const blob = new Blob([text], { type: "text/plain" }); const url = URL.createObjectURL(blob); const a = document.createElement("a"); - const baseName = files[0]?.name?.replace(/\.[^.]+$/, "") ?? "extracted"; + const baseName = + files.length === 1 ? (files[0]?.name?.replace(/\.[^.]+$/, "") ?? "extracted") : "ocr_results"; a.href = url; a.download = `${baseName}_ocr.txt`; document.body.appendChild(a); @@ -264,7 +303,7 @@ export function OcrSettings() { disabled={!hasFile || processing} className="w-full py-2.5 rounded-lg bg-primary text-primary-foreground font-medium disabled:opacity-50 disabled:cursor-not-allowed flex items-center justify-center gap-2" > - Extract Text + {files.length > 1 ? `Extract Text (${files.length} files)` : "Extract Text"} )} @@ -300,7 +339,7 @@ export function OcrSettings() { data-testid="ocr-result-text" value={text} onChange={(e) => setText(e.target.value)} - rows={8} + rows={Math.min(16, Math.max(8, text.split("\n").length + 2))} className="w-full px-2 py-1.5 rounded border border-border bg-muted text-xs text-foreground font-mono resize-y" />

{text.length} characters

diff --git a/docker/Dockerfile b/docker/Dockerfile index 91522f89..9fdbb4b7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -127,7 +127,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ ; else \ /opt/venv/bin/pip install "rembg[cpu]==2.0.62" && \ /opt/venv/bin/pip install realesrgan==0.3.0 && \ - /opt/venv/bin/pip install paddlepaddle>=3.2.1 "paddleocr[doc-parser]>=3.4.0,<3.5.0" \ + /opt/venv/bin/pip install paddlepaddle==3.0.0 "paddleocr[doc-parser]>=3.4.0,<3.5.0" \ ; fi # mediapipe 0.10.21 only has amd64 wheels; arm64 maxes out at 0.10.18 diff --git a/packages/ai/python/ocr.py b/packages/ai/python/ocr.py index 45ac8c47..3e7e4df5 100644 --- a/packages/ai/python/ocr.py +++ b/packages/ai/python/ocr.py @@ -24,21 +24,53 @@ PADDLE_LANG_MAP = { def auto_detect_language(input_path): - """Return the default language for OCR. + """Detect the predominant script in the image using Tesseract multi-lang. - Currently defaults to "en" which works well across engines. - PaddleOCR PP-OCRv5 and VL handle multi-script input natively - regardless of the language parameter, so the default is sufficient - for most use cases. Users can override via the language dropdown. + Runs a quick Tesseract pass with all installed language packs, + then analyzes the Unicode character ranges in the output to + determine which PaddleOCR language model to use. """ - return "en" + import subprocess + + try: + result = subprocess.run( + ["tesseract", input_path, "stdout", "-l", "eng+kor+chi_sim+jpn"], + capture_output=True, text=True, timeout=30, + ) + text = result.stdout.strip() + if not text: + return "en" + + hangul = sum(1 for c in text if "\uAC00" <= c <= "\uD7AF" or "\u1100" <= c <= "\u11FF") + cjk = sum(1 for c in text if "\u4E00" <= c <= "\u9FFF") + hiragana = sum(1 for c in text if "\u3040" <= c <= "\u309F") + katakana = sum(1 for c in text if "\u30A0" <= c <= "\u30FF") + latin = sum(1 for c in text if c.isascii() and c.isalpha()) + + total = hangul + cjk + hiragana + katakana + latin + if total == 0: + return "en" + + if hangul / total > 0.3: + return "ko" + if (hiragana + katakana) / total > 0.2: + return "ja" + if cjk / total > 0.3: + return "zh" + return "en" + except Exception: + return "en" -def run_tesseract(input_path, language): +def run_tesseract(input_path, language, is_auto=False): """Run Tesseract OCR (Fast tier).""" import subprocess - tess_lang = TESSERACT_LANG_MAP.get(language, "eng") + # When auto-detected, use all installed language packs for best coverage + if is_auto: + tess_lang = "eng+kor+chi_sim+jpn+deu+fra+spa" + else: + tess_lang = TESSERACT_LANG_MAP.get(language, "eng") emit_progress(30, "Scanning") result = subprocess.run( @@ -54,6 +86,28 @@ def run_tesseract(input_path, language): return text +def _extract_ocr_texts(results): + """Extract text from PaddleOCR 3.x result objects. + + Handles multiple result formats across PaddleOCR versions: + - 3.4.x: OCRResult with .json["res"]["rec_texts"] + - Earlier: result objects with .res dict containing "text" list + """ + text_parts = [] + for res in results: + # PaddleOCR 3.4.x format: OCRResult with .json dict + if hasattr(res, "json") and isinstance(res.json, dict): + inner = res.json.get("res", {}) + rec_texts = inner.get("rec_texts", []) + if rec_texts: + text_parts.extend(rec_texts) + continue + # Older format: .res dict with "text" list + if hasattr(res, "res") and isinstance(res.res, dict): + text_parts.extend(res.res.get("text", [])) + return "\n".join(text_parts) + + def run_paddleocr_v5(input_path, language): """Run PaddleOCR PP-OCRv5 server models (Balanced tier).""" os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True" @@ -62,30 +116,28 @@ def run_paddleocr_v5(input_path, language): os.dup2(2, 1) try: + import logging from paddleocr import PaddleOCR from gpu import gpu_available + # Suppress PaddleOCR internal logging (replaces removed show_log param) + for name in ("ppocr", "paddleocr", "paddle"): + logging.getLogger(name).setLevel(logging.ERROR) + paddle_lang = PADDLE_LANG_MAP.get(language, "en") + device = "gpu:0" if gpu_available() else "cpu" emit_progress(20, "Loading") ocr = PaddleOCR( lang=paddle_lang, - use_gpu=gpu_available(), - show_log=False, + device=device, ocr_version="PP-OCRv5", ) emit_progress(30, "Scanning") - result = ocr.ocr(input_path) + results = ocr.predict(input=input_path) emit_progress(70, "Extracting text") - text = "\n".join( - [ - line[1][0] - for res in result - if res - for line in res - if line and line[1] - ] - ) + + text = _extract_ocr_texts(results) finally: os.dup2(stdout_fd, 1) os.close(stdout_fd) @@ -98,6 +150,7 @@ def run_paddleocr_vl(input_path): The VLM is lazy-loaded on first call and stays resident in the dispatcher process for subsequent requests. + Requires PaddlePaddle >= 3.2 for fused_rms_norm_ext. """ global _paddleocr_vl_instance os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True" @@ -127,6 +180,11 @@ def run_paddleocr_vl(input_path): text_parts.append(content) elif hasattr(res, "rec_text"): text_parts.append(res.rec_text) + # Also try the json-based extraction as fallback + elif hasattr(res, "json") and isinstance(res.json, dict): + inner = res.json.get("res", {}) + rec_texts = inner.get("rec_texts", []) + text_parts.extend(rec_texts) text = "\n".join(text_parts) finally: @@ -166,14 +224,15 @@ def main(): preprocessed_path = None # Language auto-detection - if language == "auto": + was_auto = language == "auto" + if was_auto: emit_progress(10, "Detecting language") language = auto_detect_language(input_path) # Route to engine based on quality tier if quality == "fast": try: - text = run_tesseract(input_path, language) + text = run_tesseract(input_path, language, is_auto=was_auto) except FileNotFoundError: print(json.dumps({"success": False, "error": "Tesseract is not installed"})) sys.exit(1) @@ -187,7 +246,7 @@ def main(): except Exception: emit_progress(25, "Falling back") try: - text = run_tesseract(input_path, language) + text = run_tesseract(input_path, language, is_auto=was_auto) except FileNotFoundError: print(json.dumps({"success": False, "error": "OCR engines unavailable"})) sys.exit(1) @@ -200,13 +259,13 @@ def main(): try: text = run_paddleocr_v5(input_path, language) except Exception: - text = run_tesseract(input_path, language) + text = run_tesseract(input_path, language, is_auto=was_auto) except Exception: emit_progress(20, "Falling back") try: text = run_paddleocr_v5(input_path, language) except Exception: - text = run_tesseract(input_path, language) + text = run_tesseract(input_path, language, is_auto=was_auto) else: print(json.dumps({"success": False, "error": f"Unknown quality: {quality}"})) diff --git a/packages/ai/python/requirements.txt b/packages/ai/python/requirements.txt index 59c02290..d449c378 100644 --- a/packages/ai/python/requirements.txt +++ b/packages/ai/python/requirements.txt @@ -1,7 +1,7 @@ rembg[cpu]==2.0.62 realesrgan==0.3.0 paddleocr[doc-parser]>=3.4.0,<3.5.0 -paddlepaddle>=3.2.1 +paddlepaddle>=3.0.0,<3.1.0 mediapipe==0.10.21 onnxruntime==1.20.1 numpy==1.26.4 diff --git a/packages/ai/src/ocr.ts b/packages/ai/src/ocr.ts index be9c938a..faddf25f 100644 --- a/packages/ai/src/ocr.ts +++ b/packages/ai/src/ocr.ts @@ -1,5 +1,6 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import sharp from "sharp"; import { type ProgressCallback, runPythonWithProgress } from "./bridge.js"; export type OcrQuality = "fast" | "balanced" | "best"; @@ -24,7 +25,11 @@ export async function extractText( ): Promise { const inputPath = join(outputDir, "input_ocr.png"); - await writeFile(inputPath, inputBuffer); + // Convert any input format (HEIC, AVIF, WebP, TIFF, etc.) to PNG + // so Tesseract and PaddleOCR can read it reliably. + const pngBuffer = await sharp(inputBuffer).png().toBuffer(); + await writeFile(inputPath, pngBuffer); + const { stdout } = await runPythonWithProgress("ocr.py", [inputPath, JSON.stringify(options)], { onProgress, timeout: 600_000, // 10 min timeout for VLM on CPU