mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: make OCR portable and reliable across AMD64 and ARM64 (#519)
* fix: make OCR portable and reliable * fix: harden OCR installation portability * fix: pin OCR partials across downloads * fix: make OCR execution reliably asynchronous * fix: harden OCR portability and docs routes * fix: preserve decoder and docs safeguards
This commit is contained in:
SnapOtter
@@ -66,8 +66,6 @@ ALLOWED_SCRIPTS = {
|
||||
"inpaint",
|
||||
"install_feature",
|
||||
"noise_removal",
|
||||
"ocr",
|
||||
"ocr_pdf",
|
||||
"ocr_preprocess",
|
||||
"outpaint",
|
||||
"red_eye_removal",
|
||||
@@ -122,8 +120,6 @@ TOOL_BUNDLE_MAP = {
|
||||
"enhance_faces": "upscale-enhance",
|
||||
"noise_removal": "upscale-enhance",
|
||||
"restore": "photo-restoration",
|
||||
"ocr": "ocr",
|
||||
"ocr_pdf": "ocr",
|
||||
"transcribe": "transcription",
|
||||
}
|
||||
|
||||
|
||||
+11
-43
@@ -36,10 +36,10 @@ def gpu_available():
|
||||
"""Return True if a usable CUDA GPU is present at runtime.
|
||||
|
||||
This is the general "can any framework use a GPU" check (torch, then ONNX
|
||||
Runtime, then paddle). Tools bound to a single framework should instead call
|
||||
Runtime). Tools bound to a single framework should instead call
|
||||
the matching per-framework helper (torch_gpu_available,
|
||||
ctranslate2_gpu_available) so a GPU that only paddle or ONNX can use is not
|
||||
mistaken for a torch GPU.
|
||||
ctranslate2_gpu_available) so a GPU that only ONNX can use is not mistaken
|
||||
for a torch GPU.
|
||||
"""
|
||||
if _override_disables_gpu():
|
||||
return False
|
||||
@@ -55,16 +55,14 @@ def gpu_available():
|
||||
if onnx_available:
|
||||
return True
|
||||
|
||||
# A GPU is physically present but neither torch nor ONNX Runtime can use it.
|
||||
# The OCR bundle ships paddlepaddle-gpu, which still can, so probe paddle in
|
||||
# an isolated subprocess. This runs only now that nvidia-smi confirms a GPU,
|
||||
# and never in-process, because a GPU-less paddle import segfaults.
|
||||
# A GPU is physically present but neither supported framework can use it.
|
||||
# OCR is now an isolated portable CPU ONNX runtime, so Paddle must never be
|
||||
# imported as a fallback probe here (the old GPU wheel could crash while
|
||||
# resolving libcuda on otherwise valid CPU-only hosts).
|
||||
gpu_name = _nvidia_smi_gpu_name()
|
||||
if gpu_name:
|
||||
if _try_paddle_cuda_subprocess():
|
||||
return True
|
||||
print(f"[gpu] nvidia-smi found GPU ({gpu_name}) but neither torch, ONNX "
|
||||
"Runtime, nor paddle can use it; reinstall AI features for GPU support",
|
||||
print(f"[gpu] nvidia-smi found GPU ({gpu_name}) but neither torch nor ONNX "
|
||||
"Runtime can use it; reinstall the relevant AI feature for GPU support",
|
||||
file=sys.stderr, flush=True)
|
||||
return False
|
||||
|
||||
@@ -129,42 +127,12 @@ def _try_onnx_cuda():
|
||||
return False
|
||||
|
||||
|
||||
def _try_paddle_cuda_subprocess():
|
||||
"""Check GPU via paddle in an isolated subprocess.
|
||||
|
||||
The OCR bundle ships paddlepaddle-gpu with no torch or ONNX Runtime, so paddle
|
||||
is the only framework that can see the GPU on an OCR-only host. Importing
|
||||
paddlepaddle-gpu in-process segfaults on a GPU-less machine, so this runs in a
|
||||
throwaway subprocess and callers must confirm a GPU is present (via nvidia-smi)
|
||||
before invoking it. Returns True only when paddle has a CUDA build and a
|
||||
visible GPU. The result is signalled through the exit code so paddle's own
|
||||
import chatter on stdout cannot corrupt the reading.
|
||||
"""
|
||||
probe = (
|
||||
"import paddle, sys; "
|
||||
"sys.exit(0 if (paddle.is_compiled_with_cuda() "
|
||||
"and paddle.device.cuda.device_count() > 0) else 1)"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", probe],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return False
|
||||
if result.returncode == 0:
|
||||
print("[gpu] CUDA available via paddle (paddlepaddle-gpu)",
|
||||
file=sys.stderr, flush=True)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def torch_gpu_available():
|
||||
"""True iff torch itself can use CUDA (honors the SNAPOTTER_GPU override).
|
||||
|
||||
Torch-based tools (upscale, denoise, face enhancement, restore) must gate on
|
||||
this rather than gpu_available(), which can report True based on paddle or
|
||||
ONNX Runtime while torch is a CPU-only build. Routing those tools to CUDA on a
|
||||
this rather than gpu_available(), which can report True based on ONNX Runtime
|
||||
while torch is a CPU-only build. Routing those tools to CUDA on a
|
||||
device torch cannot use would crash them.
|
||||
"""
|
||||
if _override_disables_gpu():
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,435 +0,0 @@
|
||||
"""Text extraction from images using Tesseract, PaddleOCR PP-OCRv5, or PaddleOCR-VL 1.5."""
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
|
||||
# Prevent PaddlePaddle C++ runtime from probing for CUDA on CPU-only systems.
|
||||
# Without these, paddlepaddle-gpu can segfault during import on machines without
|
||||
# a GPU, because the C++ layer attempts GPU initialization before Python-level
|
||||
# device routing takes effect. Must run before any PaddleOCR import.
|
||||
from gpu import gpu_available
|
||||
if not gpu_available():
|
||||
if not os.environ.get("FLAGS_use_cuda"):
|
||||
os.environ["FLAGS_use_cuda"] = "0"
|
||||
if not os.environ.get("FLAGS_use_cudnn"):
|
||||
os.environ["FLAGS_use_cudnn"] = "0"
|
||||
|
||||
# Lazy-loaded VLM instance (stays resident in dispatcher process)
|
||||
_paddleocr_vl_instance = None
|
||||
|
||||
|
||||
def emit_progress(percent, stage):
|
||||
"""Emit structured progress to stderr for bridge.ts to capture."""
|
||||
print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
# OCR quality tiers backed by PaddleOCR. PaddleOCR ships as the GPU build
|
||||
# (paddlepaddle-gpu) in the amd64 bundle; its native libs dlopen libcuda.so.1 at
|
||||
# import and segfault on a CPU-only host, so these tiers need a usable GPU.
|
||||
PADDLE_QUALITY_TIERS = ("balanced", "best")
|
||||
|
||||
|
||||
def effective_quality(requested):
|
||||
"""Return the OCR quality tier that can actually run on this host.
|
||||
|
||||
On a CPU-only host the PaddleOCR tiers (balanced/best) cannot load, so they
|
||||
transparently fall back to "fast" (Tesseract), which runs on CPU. This keeps a
|
||||
GPU-less host from importing paddlepaddle-gpu, whose import segfaults and wedges
|
||||
the shared AI dispatcher.
|
||||
"""
|
||||
if requested in PADDLE_QUALITY_TIERS and not gpu_available():
|
||||
return "fast"
|
||||
return requested
|
||||
|
||||
|
||||
TESSERACT_LANG_MAP = {
|
||||
"en": "eng", "de": "deu", "fr": "fra", "es": "spa",
|
||||
"zh": "chi_sim", "ja": "jpn", "ko": "kor",
|
||||
}
|
||||
|
||||
PADDLE_LANG_MAP = {
|
||||
"en": "en", "de": "latin", "fr": "latin", "es": "latin",
|
||||
"zh": "ch", "ja": "japan", "ko": "korean",
|
||||
}
|
||||
|
||||
# Bundled PaddleOCR models (shipped by the OCR feature bundle into MODELS_PATH).
|
||||
# Pinning the constructor at these dirs keeps OCR fully offline / air-gapped and
|
||||
# skips slow HuggingFace model resolution on first use. PP-OCRv5 server rec
|
||||
# covers Chinese+English; latin covers en/de/fr/es; plus a dedicated Korean rec.
|
||||
PADDLE_DET_MODEL = "PP-OCRv5_server_det"
|
||||
PADDLE_TEXTLINE_MODEL = "PP-LCNet_x1_0_textline_ori"
|
||||
PADDLE_REC_MODEL = {
|
||||
"ch": "PP-OCRv5_server_rec",
|
||||
"en": "latin_PP-OCRv5_mobile_rec",
|
||||
"latin": "latin_PP-OCRv5_mobile_rec",
|
||||
"korean": "korean_PP-OCRv5_mobile_rec",
|
||||
}
|
||||
|
||||
|
||||
def _bundled_paddle_kwargs(paddle_lang):
|
||||
"""Build PaddleOCR kwargs that use the bundled models in MODELS_PATH.
|
||||
|
||||
Only pins a component when its model is actually present on disk, so a
|
||||
partial bundle (or an unbundled language such as Japanese) falls back to
|
||||
PaddleOCR's default resolution for that component. The doc-orientation and
|
||||
doc-unwarping models are not bundled and not needed for plain OCR, so they
|
||||
are disabled to avoid a runtime HuggingFace download.
|
||||
"""
|
||||
models_dir = os.environ.get("MODELS_PATH", "/data/ai/models")
|
||||
|
||||
def model_dir(name):
|
||||
if not name:
|
||||
return None
|
||||
path = os.path.join(models_dir, name)
|
||||
return path if os.path.isdir(path) else None
|
||||
|
||||
kwargs = {"use_doc_orientation_classify": False, "use_doc_unwarping": False}
|
||||
|
||||
det = model_dir(PADDLE_DET_MODEL)
|
||||
if det:
|
||||
kwargs["text_detection_model_name"] = PADDLE_DET_MODEL
|
||||
kwargs["text_detection_model_dir"] = det
|
||||
|
||||
rec_name = PADDLE_REC_MODEL.get(paddle_lang)
|
||||
rec = model_dir(rec_name)
|
||||
if rec:
|
||||
kwargs["text_recognition_model_name"] = rec_name
|
||||
kwargs["text_recognition_model_dir"] = rec
|
||||
|
||||
textline = model_dir(PADDLE_TEXTLINE_MODEL)
|
||||
if textline:
|
||||
kwargs["textline_orientation_model_name"] = PADDLE_TEXTLINE_MODEL
|
||||
kwargs["textline_orientation_model_dir"] = textline
|
||||
kwargs["use_textline_orientation"] = True
|
||||
else:
|
||||
kwargs["use_textline_orientation"] = False
|
||||
|
||||
return kwargs
|
||||
|
||||
|
||||
def auto_detect_language(input_path):
|
||||
"""Detect the predominant script in the image using Tesseract multi-lang.
|
||||
|
||||
Runs a quick Tesseract pass with all installed language packs,
|
||||
then analyzes the Unicode character ranges in the output to
|
||||
determine which PaddleOCR language model to use.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["tesseract", input_path, "stdout", "-l", "eng+kor+chi_sim+jpn"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
text = result.stdout.strip()
|
||||
if not text:
|
||||
return "en"
|
||||
|
||||
hangul = sum(1 for c in text if "\uAC00" <= c <= "\uD7AF" or "\u1100" <= c <= "\u11FF")
|
||||
cjk = sum(1 for c in text if "\u4E00" <= c <= "\u9FFF")
|
||||
hiragana = sum(1 for c in text if "\u3040" <= c <= "\u309F")
|
||||
katakana = sum(1 for c in text if "\u30A0" <= c <= "\u30FF")
|
||||
latin = sum(1 for c in text if c.isascii() and c.isalpha())
|
||||
|
||||
total = hangul + cjk + hiragana + katakana + latin
|
||||
if total == 0:
|
||||
return "en"
|
||||
|
||||
if hangul / total > 0.3:
|
||||
return "ko"
|
||||
if (hiragana + katakana) / total > 0.2:
|
||||
return "ja"
|
||||
if cjk / total > 0.3:
|
||||
return "zh"
|
||||
return "en"
|
||||
except Exception:
|
||||
return "en"
|
||||
|
||||
|
||||
def run_tesseract(input_path, language, is_auto=False):
|
||||
"""Run Tesseract OCR (Fast tier)."""
|
||||
import subprocess
|
||||
|
||||
# When auto-detected, use all installed language packs for best coverage
|
||||
if is_auto:
|
||||
tess_lang = "eng+kor+chi_sim+jpn+deu+fra+spa"
|
||||
else:
|
||||
tess_lang = TESSERACT_LANG_MAP.get(language, "eng")
|
||||
|
||||
emit_progress(30, "Scanning")
|
||||
result = subprocess.run(
|
||||
["tesseract", input_path, "stdout", "-l", tess_lang],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
emit_progress(70, "Extracting text")
|
||||
text = result.stdout.strip()
|
||||
if result.returncode != 0 and not text:
|
||||
raise RuntimeError(result.stderr.strip() or "Tesseract failed")
|
||||
return text
|
||||
|
||||
|
||||
def _extract_ocr_texts(results):
|
||||
"""Extract text from PaddleOCR 3.x result objects.
|
||||
|
||||
Handles multiple result formats across PaddleOCR versions:
|
||||
- 3.4.x: OCRResult with .json["res"]["rec_texts"]
|
||||
- Earlier: result objects with .res dict containing "text" list
|
||||
"""
|
||||
text_parts = []
|
||||
for res in results:
|
||||
# PaddleOCR 3.4.x format: OCRResult with .json dict
|
||||
if hasattr(res, "json") and isinstance(res.json, dict):
|
||||
inner = res.json.get("res", {})
|
||||
rec_texts = inner.get("rec_texts", [])
|
||||
if rec_texts:
|
||||
text_parts.extend(rec_texts)
|
||||
continue
|
||||
# Older format: .res dict with "text" list
|
||||
if hasattr(res, "res") and isinstance(res.res, dict):
|
||||
text_parts.extend(res.res.get("text", []))
|
||||
return "\n".join(text_parts)
|
||||
|
||||
|
||||
def run_paddleocr_v5(input_path, language):
|
||||
"""Run PaddleOCR PP-OCRv5 server models (Balanced tier)."""
|
||||
# GPU-only: paddlepaddle-gpu segfaults at import on a CPU-only host (libcuda
|
||||
# absent). Refuse before importing so the caller falls back to Tesseract.
|
||||
if not gpu_available():
|
||||
raise ImportError(
|
||||
"PaddleOCR (paddlepaddle-gpu) requires a GPU; the amd64 bundle ships the "
|
||||
"GPU build, which cannot load on a CPU-only host. Use quality=fast (Tesseract)."
|
||||
)
|
||||
os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True"
|
||||
|
||||
stdout_fd = os.dup(1)
|
||||
os.dup2(2, 1)
|
||||
|
||||
try:
|
||||
import logging
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
# Suppress PaddleOCR internal logging (replaces removed show_log param)
|
||||
for name in ("ppocr", "paddleocr", "paddle"):
|
||||
logging.getLogger(name).setLevel(logging.ERROR)
|
||||
|
||||
paddle_lang = PADDLE_LANG_MAP.get(language, "en")
|
||||
device = "gpu:0" if gpu_available() else "cpu"
|
||||
|
||||
emit_progress(20, "Loading")
|
||||
mk = _bundled_paddle_kwargs(paddle_lang)
|
||||
# When a bundled recognizer is pinned, the model selects the script, so
|
||||
# we omit lang (this is the proven fully-offline path). Lang-based
|
||||
# resolution for a language without a bundled recognizer (e.g. ja) and
|
||||
# a missing detection model both make PaddleOCR fetch models over the
|
||||
# network, which strict offline mode blocks with a clear error.
|
||||
if "text_recognition_model_dir" not in mk:
|
||||
from offline_guard import ensure_download_allowed
|
||||
ensure_download_allowed(f"PaddleOCR recognition model for language '{language}'")
|
||||
mk["lang"] = paddle_lang
|
||||
if "text_detection_model_dir" not in mk:
|
||||
from offline_guard import ensure_download_allowed
|
||||
ensure_download_allowed(f"PaddleOCR text detection model ({PADDLE_DET_MODEL})")
|
||||
ocr = PaddleOCR(
|
||||
device=device,
|
||||
ocr_version="PP-OCRv5",
|
||||
enable_mkldnn=False,
|
||||
**mk,
|
||||
)
|
||||
emit_progress(30, "Scanning")
|
||||
results = ocr.predict(input=input_path)
|
||||
emit_progress(70, "Extracting text")
|
||||
|
||||
text = _extract_ocr_texts(results)
|
||||
finally:
|
||||
os.dup2(stdout_fd, 1)
|
||||
os.close(stdout_fd)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def run_paddleocr_vl(input_path):
|
||||
"""Run PaddleOCR-VL 1.5 vision-language model (Best tier).
|
||||
|
||||
The VLM is lazy-loaded on first call and stays resident in the
|
||||
dispatcher process for subsequent requests.
|
||||
Requires PaddlePaddle >= 3.2 for fused_rms_norm_ext.
|
||||
"""
|
||||
global _paddleocr_vl_instance
|
||||
# GPU-only: see run_paddleocr_v5. Refuse before importing paddle on CPU.
|
||||
if not gpu_available():
|
||||
raise ImportError(
|
||||
"PaddleOCR-VL (paddlepaddle-gpu) requires a GPU; the amd64 bundle ships the "
|
||||
"GPU build, which cannot load on a CPU-only host. Use quality=balanced or fast."
|
||||
)
|
||||
os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True"
|
||||
|
||||
stdout_fd = os.dup(1)
|
||||
os.dup2(2, 1)
|
||||
|
||||
try:
|
||||
if _paddleocr_vl_instance is None:
|
||||
emit_progress(15, "Loading model")
|
||||
from paddleocr import PaddleOCRVL
|
||||
|
||||
device = "gpu" if gpu_available() else "cpu"
|
||||
_paddleocr_vl_instance = PaddleOCRVL(device=device)
|
||||
|
||||
emit_progress(30, "Scanning")
|
||||
output = _paddleocr_vl_instance.predict(input_path)
|
||||
emit_progress(70, "Extracting text")
|
||||
|
||||
text_parts = []
|
||||
for res in output:
|
||||
# PaddleOCR-VL 1.5+: markdown_texts holds the extracted text
|
||||
if hasattr(res, "markdown") and isinstance(res.markdown, dict):
|
||||
md_text = res.markdown.get("markdown_texts", "")
|
||||
if md_text:
|
||||
text_parts.append(md_text)
|
||||
continue
|
||||
if hasattr(res, "parsing_res_list"):
|
||||
for block in res.parsing_res_list:
|
||||
content = block.get("block_content", "")
|
||||
if content:
|
||||
text_parts.append(content)
|
||||
elif hasattr(res, "rec_text"):
|
||||
text_parts.append(res.rec_text)
|
||||
# Also try the json-based extraction as fallback
|
||||
elif hasattr(res, "json") and isinstance(res.json, dict):
|
||||
inner = res.json.get("res", {})
|
||||
rec_texts = inner.get("rec_texts", [])
|
||||
text_parts.extend(rec_texts)
|
||||
|
||||
text = "\n".join(text_parts)
|
||||
finally:
|
||||
os.dup2(stdout_fd, 1)
|
||||
os.close(stdout_fd)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def main():
|
||||
input_path = sys.argv[1]
|
||||
settings = json.loads(sys.argv[2]) if len(sys.argv) > 2 else {}
|
||||
|
||||
quality = settings.get("quality", None)
|
||||
language = settings.get("language", "auto")
|
||||
enhance = settings.get("enhance", True)
|
||||
|
||||
# Backward compat: old "engine" param maps to quality
|
||||
if quality is None:
|
||||
engine = settings.get("engine", "tesseract")
|
||||
quality = "fast" if engine == "tesseract" else "balanced"
|
||||
|
||||
# On a CPU-only host, downgrade GPU-only tiers (PaddleOCR) to Tesseract so we
|
||||
# never import paddlepaddle-gpu, whose import segfaults and wedges the dispatcher.
|
||||
downgraded = effective_quality(quality)
|
||||
if downgraded != quality:
|
||||
print(
|
||||
json.dumps({"info": f"{quality} OCR needs a GPU; using {downgraded} (Tesseract) on CPU"}),
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
quality = downgraded
|
||||
|
||||
preprocessed_path = None
|
||||
try:
|
||||
emit_progress(5, "Preparing")
|
||||
|
||||
# Preprocessing (if enabled)
|
||||
if enhance:
|
||||
emit_progress(8, "Enhancing image")
|
||||
try:
|
||||
from ocr_preprocess import preprocess
|
||||
preprocessed_path = input_path + "_enhanced.png"
|
||||
preprocess(input_path, preprocessed_path)
|
||||
input_path = preprocessed_path
|
||||
except Exception as e:
|
||||
print(json.dumps({"warning": f"Enhancement skipped: {e}"}), file=sys.stderr, flush=True)
|
||||
preprocessed_path = None
|
||||
|
||||
# Language auto-detection
|
||||
was_auto = language == "auto"
|
||||
if was_auto:
|
||||
emit_progress(10, "Detecting language")
|
||||
language = auto_detect_language(input_path)
|
||||
|
||||
engine_used = quality
|
||||
|
||||
# Route to engine based on quality tier
|
||||
if quality == "fast":
|
||||
try:
|
||||
text = run_tesseract(input_path, language, is_auto=was_auto)
|
||||
engine_used = "tesseract"
|
||||
except FileNotFoundError:
|
||||
print(json.dumps({"success": False, "error": "Tesseract is not installed"}))
|
||||
sys.exit(1)
|
||||
|
||||
elif quality == "balanced":
|
||||
try:
|
||||
text = run_paddleocr_v5(input_path, language)
|
||||
engine_used = "paddleocr-v5"
|
||||
except ImportError as e:
|
||||
print(json.dumps({
|
||||
"success": False,
|
||||
"error": (
|
||||
f"PaddleOCR is not installed: {e}. "
|
||||
"Install the OCR feature or use quality=fast for Tesseract."
|
||||
),
|
||||
}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({
|
||||
"success": False,
|
||||
"error": (
|
||||
f"PaddleOCR PP-OCRv5 failed: {type(e).__name__}: {e}. "
|
||||
"Install the OCR feature or use quality=fast for Tesseract."
|
||||
),
|
||||
}))
|
||||
sys.exit(1)
|
||||
|
||||
elif quality == "best":
|
||||
try:
|
||||
text = run_paddleocr_vl(input_path)
|
||||
engine_used = "paddleocr-vl"
|
||||
except ImportError as e:
|
||||
print(json.dumps({
|
||||
"success": False,
|
||||
"error": (
|
||||
f"PaddleOCR-VL is not available: {e}. "
|
||||
"Install the OCR feature or use quality=balanced for PP-OCRv5."
|
||||
),
|
||||
}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({
|
||||
"success": False,
|
||||
"error": (
|
||||
f"PaddleOCR-VL failed: {type(e).__name__}: {e}. "
|
||||
"Install the OCR feature or use quality=balanced for PP-OCRv5."
|
||||
),
|
||||
}))
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
print(json.dumps({"success": False, "error": f"Unknown quality: {quality}"}))
|
||||
sys.exit(1)
|
||||
|
||||
emit_progress(95, "Done")
|
||||
print(json.dumps({"success": True, "text": text, "engine": engine_used}))
|
||||
|
||||
except Exception as e:
|
||||
print(json.dumps({"success": False, "error": str(e)}))
|
||||
sys.exit(1)
|
||||
finally:
|
||||
# Clean up preprocessed temp file
|
||||
if preprocessed_path:
|
||||
try:
|
||||
os.remove(preprocessed_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,196 +0,0 @@
|
||||
"""OCR text extraction from PDF documents.
|
||||
|
||||
Rasterizes each page via PyMuPDF (fitz) and runs the same OCR engine
|
||||
pipeline as ocr.py (tesseract / PaddleOCR PP-OCRv5 / PaddleOCR-VL).
|
||||
|
||||
Contract: argv[1] = pdf path, argv[2] = options JSON {"quality","language","pages"}.
|
||||
Output: {"success": true, "engine": ..., "pages": n, "text": ...} to stdout.
|
||||
Errors: {"error": ...} to stdout, exit 1.
|
||||
|
||||
Engine functions are imported from sibling ocr.py (the dispatcher adds the
|
||||
scripts directory to sys.path, and `import ocr` loads ocr.py as a module
|
||||
without triggering its __main__ guard).
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
|
||||
MAX_PAGES = 50
|
||||
|
||||
|
||||
def emit_progress(percent, stage):
|
||||
"""Emit structured progress to stderr for bridge.ts to capture."""
|
||||
print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def parse_page_spec(spec, total_pages):
|
||||
"""Parse a page spec like 'all', '1-3,5', '2,4-6' into a sorted list of 0-based indices.
|
||||
|
||||
Returns (pages, error) where error is a string if the spec is invalid.
|
||||
"""
|
||||
if spec.strip().lower() == "all":
|
||||
return list(range(total_pages)), None
|
||||
|
||||
pages = set()
|
||||
parts = spec.split(",")
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
if "-" in part:
|
||||
bounds = part.split("-", 1)
|
||||
try:
|
||||
start = int(bounds[0].strip())
|
||||
end = int(bounds[1].strip())
|
||||
except ValueError:
|
||||
return None, f"Invalid page range: {part}"
|
||||
if start < 1 or end < 1:
|
||||
return None, f"Invalid page range: {part} (pages start at 1)"
|
||||
if start > end:
|
||||
return None, f"Invalid page range: {part} (start > end)"
|
||||
if end > total_pages:
|
||||
return None, f"Invalid page range: {part} (document has {total_pages} pages)"
|
||||
for i in range(start, end + 1):
|
||||
pages.add(i - 1) # 0-based
|
||||
else:
|
||||
try:
|
||||
num = int(part)
|
||||
except ValueError:
|
||||
return None, f"Invalid page number: {part}"
|
||||
if num < 1:
|
||||
return None, f"Invalid page number: {num} (pages start at 1)"
|
||||
if num > total_pages:
|
||||
return None, f"Invalid page number: {num} (document has {total_pages} pages)"
|
||||
pages.add(num - 1) # 0-based
|
||||
|
||||
if not pages:
|
||||
return None, "No pages specified"
|
||||
|
||||
return sorted(pages), None
|
||||
|
||||
|
||||
def main():
|
||||
input_path = sys.argv[1]
|
||||
settings = json.loads(sys.argv[2]) if len(sys.argv) > 2 else {}
|
||||
|
||||
quality = settings.get("quality", "balanced")
|
||||
language = settings.get("language", "auto")
|
||||
pages_spec = settings.get("pages", "all")
|
||||
|
||||
try:
|
||||
emit_progress(2, "Opening PDF")
|
||||
|
||||
# Lazy import: fitz (PyMuPDF) is in the base venv but keep it lazy
|
||||
# so py_compile succeeds without it.
|
||||
import fitz
|
||||
|
||||
doc = fitz.open(input_path)
|
||||
total_pages = len(doc)
|
||||
|
||||
if total_pages == 0:
|
||||
print(json.dumps({"error": "PDF has no pages"}))
|
||||
sys.exit(1)
|
||||
|
||||
# Parse page specification
|
||||
page_indices, parse_error = parse_page_spec(pages_spec, total_pages)
|
||||
if parse_error:
|
||||
print(json.dumps({"error": parse_error}))
|
||||
sys.exit(1)
|
||||
|
||||
if len(page_indices) > MAX_PAGES:
|
||||
print(json.dumps({"error": f"Too many pages for OCR (max {MAX_PAGES})"}))
|
||||
sys.exit(1)
|
||||
|
||||
emit_progress(5, "Preparing")
|
||||
|
||||
# Import engine functions from sibling ocr.py.
|
||||
# The dispatcher's scripts directory is on sys.path, so this import
|
||||
# loads ocr.py as a module (its __main__ guard prevents re-execution).
|
||||
import ocr as ocr_module
|
||||
|
||||
# Language auto-detection on the first page's raster
|
||||
was_auto = language == "auto"
|
||||
|
||||
# Create a temp directory for scratch PNGs next to the input
|
||||
scratch_dir = tempfile.mkdtemp(
|
||||
prefix="ocr_pdf_",
|
||||
dir=os.path.dirname(input_path) or tempfile.gettempdir(),
|
||||
)
|
||||
|
||||
page_texts = []
|
||||
engine_used = quality
|
||||
num_pages = len(page_indices)
|
||||
|
||||
for i, page_idx in enumerate(page_indices):
|
||||
page_num = page_idx + 1 # 1-based for display
|
||||
pct = 5 + int((i / num_pages) * 85)
|
||||
emit_progress(pct, f"Page {page_num}/{total_pages}")
|
||||
|
||||
# Rasterize page to PNG at 200 DPI
|
||||
page = doc[page_idx]
|
||||
pix = page.get_pixmap(dpi=200)
|
||||
png_path = os.path.join(scratch_dir, f"page_{page_idx}.png")
|
||||
pix.save(png_path)
|
||||
|
||||
# Auto-detect language on the first page only
|
||||
current_lang = language
|
||||
if was_auto and i == 0:
|
||||
current_lang = ocr_module.auto_detect_language(png_path)
|
||||
language = current_lang # reuse for remaining pages
|
||||
elif was_auto:
|
||||
current_lang = language # already detected
|
||||
|
||||
# Always route PDF OCR through tesseract. PaddleOCR segfaults
|
||||
# on arm64 CPU when processing rasterised PDF pages (SIGSEGV in
|
||||
# the doc-orientation / structural-analysis stage). Tesseract is
|
||||
# reliable for page-level images and is already installed in the
|
||||
# container with multi-language packs. The image-OCR tool still
|
||||
# offers PaddleOCR tiers for single images where it is stable.
|
||||
if quality not in ("fast", "balanced", "best"):
|
||||
print(json.dumps({"error": f"Unknown quality: {quality}"}))
|
||||
sys.exit(1)
|
||||
|
||||
text = ocr_module.run_tesseract(png_path, current_lang, is_auto=was_auto)
|
||||
engine_used = "tesseract"
|
||||
|
||||
page_texts.append((page_num, text))
|
||||
|
||||
# Clean up scratch PNG immediately
|
||||
try:
|
||||
os.remove(png_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Clean up scratch directory
|
||||
try:
|
||||
os.rmdir(scratch_dir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
doc.close()
|
||||
|
||||
# Join page texts with page headers
|
||||
parts = []
|
||||
for page_num, text in page_texts:
|
||||
parts.append(f"\n\n--- Page {page_num} ---\n\n{text}")
|
||||
|
||||
# Strip leading newlines from the first page
|
||||
full_text = "".join(parts).strip()
|
||||
|
||||
emit_progress(95, "Done")
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"engine": engine_used,
|
||||
"pages": num_pages,
|
||||
"text": full_text,
|
||||
}))
|
||||
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,263 @@
|
||||
"""Persistent JSON-line entrypoint for the isolated OCR runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Callable, Mapping, TextIO
|
||||
|
||||
try:
|
||||
# Direct execution inside the immutable runtime generation.
|
||||
from ocr_runtime import OcrRuntime
|
||||
except ModuleNotFoundError: # pragma: no cover - exercised by package imports in tooling
|
||||
from .ocr_runtime import OcrRuntime
|
||||
|
||||
|
||||
PROTOCOL_VERSION = 1
|
||||
MAX_REQUEST_BYTES = 64 * 1024
|
||||
MAX_ERROR_CHARS = 400
|
||||
|
||||
|
||||
def configure_offline_environment() -> None:
|
||||
"""Set offline switches before importing any optional model libraries."""
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
||||
os.environ["SNAPOTTER_OCR_OFFLINE"] = "1"
|
||||
os.environ["NO_PROXY"] = "*"
|
||||
os.environ["no_proxy"] = "*"
|
||||
|
||||
|
||||
def _request_id(request: Any) -> str:
|
||||
if isinstance(request, Mapping):
|
||||
value = request.get("requestId")
|
||||
if isinstance(value, str) and 0 < len(value) <= 128:
|
||||
return value
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _failure(request_id: str, code: str, message: str) -> dict[str, Any]:
|
||||
bounded = " ".join(str(message).split())[:MAX_ERROR_CHARS] or "OCR runtime request failed"
|
||||
return {
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"requestId": request_id,
|
||||
"ok": False,
|
||||
"error": {"code": code, "message": bounded},
|
||||
}
|
||||
|
||||
|
||||
def _parse_settings(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
settings = json.loads(value)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError("OCR settings are not valid JSON") from error
|
||||
if not isinstance(settings, dict):
|
||||
raise ValueError("OCR settings must be a JSON object")
|
||||
return settings
|
||||
|
||||
|
||||
def _parse_pages(value: str) -> tuple[tuple[int, Path], ...]:
|
||||
try:
|
||||
raw_pages = json.loads(value)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError("Rasterized pages are not valid JSON") from error
|
||||
if not isinstance(raw_pages, list) or not raw_pages:
|
||||
raise ValueError("Rasterized pages must be a non-empty JSON list")
|
||||
pages = []
|
||||
for index, item in enumerate(raw_pages):
|
||||
if isinstance(item, str):
|
||||
pages.append((index + 1, Path(item)))
|
||||
continue
|
||||
if (
|
||||
isinstance(item, dict)
|
||||
and isinstance(item.get("page"), int)
|
||||
and not isinstance(item.get("page"), bool)
|
||||
and isinstance(item.get("path"), str)
|
||||
):
|
||||
pages.append((item["page"], Path(item["path"])))
|
||||
continue
|
||||
raise ValueError("Each rasterized page must be a path or {page, path} object")
|
||||
return tuple(pages)
|
||||
|
||||
|
||||
def _validate_request(request: Any) -> tuple[str, str, list[str]]:
|
||||
if not isinstance(request, dict):
|
||||
raise ValueError("OCR runtime request must be an object")
|
||||
request_id = _request_id(request)
|
||||
if request_id == "unknown":
|
||||
raise ValueError("OCR runtime requestId is missing or invalid")
|
||||
protocol_version = request.get("protocolVersion")
|
||||
if type(protocol_version) is not int or protocol_version != PROTOCOL_VERSION:
|
||||
raise ValueError(f"Unsupported OCR runtime protocol version; expected {PROTOCOL_VERSION}")
|
||||
script = request.get("script")
|
||||
if script not in ("ocr", "ocr_pdf", "smoke"):
|
||||
raise ValueError("OCR runtime script must be ocr, ocr_pdf, or smoke")
|
||||
args = request.get("args")
|
||||
if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args):
|
||||
raise ValueError("OCR runtime args must be a list of strings")
|
||||
if script == "smoke" and args:
|
||||
raise ValueError("OCR runtime smoke requires no arguments")
|
||||
if script != "smoke" and len(args) != 2:
|
||||
raise ValueError("OCR runtime scripts require exactly two arguments")
|
||||
return request_id, script, args
|
||||
|
||||
|
||||
def _prepare_request(request: Any) -> tuple[str, str, tuple[Any, ...]]:
|
||||
"""Validate and decode a frame without constructing model sessions."""
|
||||
request_id, script, args = _validate_request(request)
|
||||
if script == "smoke":
|
||||
return request_id, script, ()
|
||||
settings = _parse_settings(args[1])
|
||||
if script == "ocr":
|
||||
return request_id, script, (Path(args[0]), settings)
|
||||
return request_id, script, (_parse_pages(args[0]), settings)
|
||||
|
||||
|
||||
def _process_prepared_request(
|
||||
prepared: tuple[str, str, tuple[Any, ...]], runtime: OcrRuntime
|
||||
) -> dict[str, Any]:
|
||||
request_id, script, arguments = prepared
|
||||
try:
|
||||
if script == "smoke":
|
||||
result = runtime.smoke()
|
||||
elif script == "ocr":
|
||||
result = runtime.recognize_image(*arguments)
|
||||
else:
|
||||
result = runtime.recognize_pages(*arguments)
|
||||
return {
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"requestId": request_id,
|
||||
"ok": True,
|
||||
"result": result,
|
||||
}
|
||||
except ValueError as error:
|
||||
return _failure(request_id, "invalid-request", str(error))
|
||||
except FileNotFoundError as error:
|
||||
return _failure(request_id, "file-not-found", str(error))
|
||||
except Exception as error:
|
||||
return _failure(request_id, "ocr-runtime-failed", str(error))
|
||||
|
||||
|
||||
def process_request(
|
||||
request: Any,
|
||||
*,
|
||||
runtime: OcrRuntime | None = None,
|
||||
runtime_factory: Callable[[], OcrRuntime] = OcrRuntime,
|
||||
) -> dict[str, Any]:
|
||||
request_id = _request_id(request)
|
||||
try:
|
||||
prepared = _prepare_request(request)
|
||||
request_id = prepared[0]
|
||||
active_runtime = runtime or runtime_factory()
|
||||
return _process_prepared_request(prepared, active_runtime)
|
||||
except ValueError as error:
|
||||
return _failure(request_id, "invalid-request", str(error))
|
||||
except FileNotFoundError as error:
|
||||
return _failure(request_id, "file-not-found", str(error))
|
||||
except Exception as error:
|
||||
return _failure(request_id, "ocr-runtime-failed", str(error))
|
||||
|
||||
|
||||
def main(
|
||||
argv: list[str] | None = None,
|
||||
*,
|
||||
runtime_factory: Callable[[], OcrRuntime] = OcrRuntime,
|
||||
stdin: BinaryIO | None = None,
|
||||
stdout: TextIO | None = None,
|
||||
stderr: TextIO | None = None,
|
||||
) -> int:
|
||||
configure_offline_environment()
|
||||
arguments = list(sys.argv[1:] if argv is None else argv)
|
||||
input_stream = sys.stdin.buffer if stdin is None else stdin
|
||||
output_stream = sys.stdout if stdout is None else stdout
|
||||
error_stream = sys.stderr if stderr is None else stderr
|
||||
|
||||
if arguments == ["--smoke"]:
|
||||
try:
|
||||
smoke_result = runtime_factory().smoke()
|
||||
except Exception as error:
|
||||
error_stream.write(f"OCR runtime smoke failed: {error}\n")
|
||||
error_stream.flush()
|
||||
return 1
|
||||
output_stream.write(
|
||||
json.dumps(
|
||||
{"smoke": True, **smoke_result},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
output_stream.flush()
|
||||
return 0
|
||||
if arguments:
|
||||
error_stream.write("Usage: ocr_runner.py [--smoke]\n")
|
||||
error_stream.flush()
|
||||
return 2
|
||||
|
||||
runtime: OcrRuntime | None = None
|
||||
handled_requests = 0
|
||||
|
||||
while True:
|
||||
raw = input_stream.readline(MAX_REQUEST_BYTES + 1)
|
||||
if not raw:
|
||||
if handled_requests == 0:
|
||||
response = _failure(
|
||||
"unknown", "invalid-request", "OCR runtime request is empty"
|
||||
)
|
||||
output_stream.write(
|
||||
json.dumps(response, ensure_ascii=False, separators=(",", ":"))
|
||||
+ "\n"
|
||||
)
|
||||
output_stream.flush()
|
||||
return 1
|
||||
return 0
|
||||
|
||||
exit_after_response = False
|
||||
if len(raw) > MAX_REQUEST_BYTES:
|
||||
response = _failure(
|
||||
"unknown", "invalid-request", "OCR runtime request is too large"
|
||||
)
|
||||
# readline() leaves the rest of an oversized unterminated frame in
|
||||
# the stream. Exit after the bounded error instead of interpreting
|
||||
# the tail as another request and losing frame synchronization.
|
||||
exit_after_response = not raw.endswith(b"\n")
|
||||
else:
|
||||
try:
|
||||
request = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
response = _failure(
|
||||
"unknown", "invalid-request", f"Malformed JSON request: {error}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
prepared = _prepare_request(request)
|
||||
except ValueError as error:
|
||||
response = _failure(
|
||||
_request_id(request), "invalid-request", str(error)
|
||||
)
|
||||
else:
|
||||
if runtime is None:
|
||||
try:
|
||||
runtime = runtime_factory()
|
||||
except Exception as error:
|
||||
response = _failure(
|
||||
_request_id(request), "ocr-runtime-failed", str(error)
|
||||
)
|
||||
exit_after_response = True
|
||||
else:
|
||||
response = _process_prepared_request(prepared, runtime)
|
||||
else:
|
||||
response = _process_prepared_request(prepared, runtime)
|
||||
|
||||
output_stream.write(
|
||||
json.dumps(response, ensure_ascii=False, separators=(",", ":")) + "\n"
|
||||
)
|
||||
output_stream.flush()
|
||||
handled_requests += 1
|
||||
if exit_after_response:
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,8 +7,6 @@
|
||||
# allowlisted model names (remove_bg.py ALLOWED_MODELS), never user paths.
|
||||
rembg==2.0.69
|
||||
realesrgan==0.3.0
|
||||
paddleocr==2.9.1
|
||||
paddlepaddle-gpu==3.0.0
|
||||
mediapipe>=0.10.21
|
||||
onnxruntime-gpu==1.20.1
|
||||
numpy==1.26.4
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
# allowlisted model names (remove_bg.py ALLOWED_MODELS), never user paths.
|
||||
rembg[cpu]==2.0.69
|
||||
realesrgan==0.3.0
|
||||
paddleocr[doc-parser]>=3.4.0,<3.5.0
|
||||
paddlepaddle>=3.0.0,<3.1.0
|
||||
mediapipe>=0.10.21
|
||||
onnxruntime==1.20.1
|
||||
numpy==1.26.4
|
||||
|
||||
@@ -1,15 +1,5 @@
|
||||
"""gpu_available() must recognize a GPU that only paddle can use.
|
||||
|
||||
The OCR feature bundle ships paddlepaddle-gpu but no torch or ONNX Runtime. On an
|
||||
OCR-only GPU host the torch and ONNX probes both come up empty, so before this
|
||||
fix gpu_available() fell through to a plain nvidia-smi check that returned False
|
||||
by design, and OCR silently downgraded to Tesseract. gpu_available() now probes
|
||||
paddle too, in an isolated subprocess, but only after nvidia-smi confirms a GPU
|
||||
is physically present (importing paddlepaddle-gpu in-process segfaults on a
|
||||
GPU-less host and would wedge the shared AI dispatcher).
|
||||
"""
|
||||
"""Framework-specific CUDA probes must never depend on obsolete Paddle OCR."""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
|
||||
@@ -17,32 +7,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
import gpu # noqa: E402
|
||||
|
||||
|
||||
# --- The isolated paddle probe --------------------------------------------
|
||||
|
||||
def test_paddle_probe_true_when_subprocess_reports_cuda(monkeypatch):
|
||||
def fake_run(cmd, **kwargs):
|
||||
return subprocess.CompletedProcess(cmd, returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gpu.subprocess, "run", fake_run)
|
||||
assert gpu._try_paddle_cuda_subprocess() is True
|
||||
|
||||
|
||||
def test_paddle_probe_false_when_subprocess_reports_no_cuda(monkeypatch):
|
||||
def fake_run(cmd, **kwargs):
|
||||
return subprocess.CompletedProcess(cmd, returncode=1, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gpu.subprocess, "run", fake_run)
|
||||
assert gpu._try_paddle_cuda_subprocess() is False
|
||||
|
||||
|
||||
def test_paddle_probe_false_on_timeout(monkeypatch):
|
||||
def fake_run(cmd, **kwargs):
|
||||
raise subprocess.TimeoutExpired(cmd, 30)
|
||||
|
||||
monkeypatch.setattr(gpu.subprocess, "run", fake_run)
|
||||
assert gpu._try_paddle_cuda_subprocess() is False
|
||||
|
||||
|
||||
# --- gpu_available() orchestration -----------------------------------------
|
||||
|
||||
def _patch_probes(monkeypatch, torch, onnx, smi):
|
||||
@@ -54,39 +18,19 @@ def _patch_probes(monkeypatch, torch, onnx, smi):
|
||||
gpu.gpu_available.cache_clear()
|
||||
|
||||
|
||||
def test_gpu_available_true_when_only_paddle_sees_gpu(monkeypatch):
|
||||
# OCR-only GPU box: torch and ONNX absent, GPU present, paddle can use it.
|
||||
def test_gpu_available_does_not_treat_hardware_presence_as_framework_support(monkeypatch):
|
||||
_patch_probes(monkeypatch, torch=False, onnx=False, smi="NVIDIA GeForce RTX 4070")
|
||||
monkeypatch.setattr(gpu, "_try_paddle_cuda_subprocess", lambda: True)
|
||||
assert gpu.gpu_available() is True
|
||||
|
||||
|
||||
def test_gpu_available_false_when_paddle_cannot_use_gpu(monkeypatch):
|
||||
# GPU present but paddle is a CPU build or cannot init CUDA: stay conservative.
|
||||
_patch_probes(monkeypatch, torch=False, onnx=False, smi="NVIDIA GeForce RTX 4070")
|
||||
monkeypatch.setattr(gpu, "_try_paddle_cuda_subprocess", lambda: False)
|
||||
assert gpu.gpu_available() is False
|
||||
|
||||
|
||||
def test_gpu_available_never_probes_paddle_without_a_gpu(monkeypatch):
|
||||
# Safety invariant: on a GPU-less host a paddlepaddle-gpu import segfaults, so
|
||||
# the probe must never run when nvidia-smi finds no GPU.
|
||||
_patch_probes(monkeypatch, torch=False, onnx=False, smi=None)
|
||||
called = {"paddle": False}
|
||||
|
||||
def spy():
|
||||
called["paddle"] = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(gpu, "_try_paddle_cuda_subprocess", spy)
|
||||
assert gpu.gpu_available() is False
|
||||
assert called["paddle"] is False
|
||||
def test_gpu_module_has_no_paddle_probe():
|
||||
assert not hasattr(gpu, "_try_paddle_cuda_subprocess")
|
||||
|
||||
|
||||
# --- Per-framework detection (torch, ctranslate2) --------------------------
|
||||
#
|
||||
# Torch and CTranslate2 tools must gate on their OWN framework, not the general
|
||||
# gpu_available(), which can report True based on paddle or ONNX Runtime while
|
||||
# gpu_available(), which can report True based on ONNX Runtime while
|
||||
# torch is a CPU-only build. Consuming the shared boolean would make those tools
|
||||
# route to CUDA on a device their framework cannot use.
|
||||
|
||||
@@ -103,7 +47,7 @@ def test_torch_gpu_available_false_when_override_disables_gpu(monkeypatch):
|
||||
|
||||
|
||||
def test_torch_gpu_available_false_when_torch_is_cpu_only(monkeypatch):
|
||||
# The crux: gpu_available() may be True via paddle or ONNX on a GPU box, but a
|
||||
# The crux: gpu_available() may be True via ONNX on a GPU box, but a
|
||||
# CPU-only torch build must report no GPU so torch tools do not touch CUDA.
|
||||
monkeypatch.delenv("SNAPOTTER_GPU", raising=False)
|
||||
monkeypatch.setattr(gpu, "_try_torch_cuda", lambda: False)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,44 +0,0 @@
|
||||
"""OCR must degrade gracefully on CPU-only hosts.
|
||||
|
||||
The amd64 AI bundle ships paddlepaddle-gpu, whose native libraries dlopen
|
||||
libcuda.so.1 at import time and segfault on a host without a GPU (libcuda is the
|
||||
driver lib, injected only by nvidia-container-toolkit on GPU hosts). That segfault
|
||||
crashes the shared long-lived AI dispatcher and wedges all AI. So on a CPU-only
|
||||
host the PaddleOCR tiers (balanced/best) must transparently fall back to Tesseract
|
||||
and must never reach the paddle import.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
import ocr # noqa: E402
|
||||
|
||||
|
||||
def test_effective_quality_downgrades_paddle_tiers_on_cpu(monkeypatch):
|
||||
monkeypatch.setattr(ocr, "gpu_available", lambda: False)
|
||||
assert ocr.effective_quality("balanced") == "fast"
|
||||
assert ocr.effective_quality("best") == "fast"
|
||||
assert ocr.effective_quality("fast") == "fast"
|
||||
|
||||
|
||||
def test_effective_quality_preserves_paddle_tiers_on_gpu(monkeypatch):
|
||||
monkeypatch.setattr(ocr, "gpu_available", lambda: True)
|
||||
assert ocr.effective_quality("balanced") == "balanced"
|
||||
assert ocr.effective_quality("best") == "best"
|
||||
assert ocr.effective_quality("fast") == "fast"
|
||||
|
||||
|
||||
def test_run_paddleocr_v5_refuses_on_cpu_before_import(monkeypatch):
|
||||
# Must raise a GPU-specific error (the guard), NOT attempt the paddle import
|
||||
# that would segfault on a CPU-only host.
|
||||
monkeypatch.setattr(ocr, "gpu_available", lambda: False)
|
||||
with pytest.raises(ImportError, match="GPU"):
|
||||
ocr.run_paddleocr_v5("/nonexistent.png", "en")
|
||||
|
||||
|
||||
def test_run_paddleocr_vl_refuses_on_cpu_before_import(monkeypatch):
|
||||
monkeypatch.setattr(ocr, "gpu_available", lambda: False)
|
||||
with pytest.raises(ImportError, match="GPU"):
|
||||
ocr.run_paddleocr_vl("/nonexistent.png")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
"""Accurate OCR must never re-enter the mutable legacy Python dispatcher."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from packages.ai.python import dispatcher
|
||||
|
||||
|
||||
def test_legacy_dispatcher_does_not_allow_ocr_scripts():
|
||||
assert "ocr" not in dispatcher.ALLOWED_SCRIPTS
|
||||
assert "ocr_pdf" not in dispatcher.ALLOWED_SCRIPTS
|
||||
|
||||
|
||||
def test_shared_requirements_do_not_install_paddle():
|
||||
python_root = Path(__file__).resolve().parents[1]
|
||||
for name in ("requirements.txt", "requirements-gpu.txt"):
|
||||
requirements = (python_root / name).read_text(encoding="utf-8").lower()
|
||||
assert "paddleocr" not in requirements
|
||||
assert "paddlepaddle" not in requirements
|
||||
@@ -289,9 +289,8 @@ export class PythonDispatcher {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Diagnostic notices (e.g. ocr.py's GPU-to-tesseract downgrade
|
||||
// notice) - forward so they reach docker logs instead of being
|
||||
// silently dropped for matching neither shape above.
|
||||
// Diagnostic notices are forwarded so they reach docker logs
|
||||
// instead of being silently dropped for matching neither shape.
|
||||
if (typeof parsed.info === "string") {
|
||||
console.log(`[python] ${parsed.info}`);
|
||||
continue;
|
||||
|
||||
@@ -23,8 +23,6 @@ export const SCRIPT_BUNDLE_MAP: Record<string, string> = {
|
||||
enhance_faces: "upscale-enhance",
|
||||
noise_removal: "upscale-enhance",
|
||||
restore: "photo-restoration",
|
||||
ocr: "ocr",
|
||||
ocr_pdf: "ocr",
|
||||
transcribe: "transcription",
|
||||
};
|
||||
|
||||
|
||||
@@ -29,13 +29,104 @@ export { detectFaceLandmarks } from "./face-landmarks.js";
|
||||
export { missingBundleForScript, SCRIPT_BUNDLE_MAP } from "./feature-gate.js";
|
||||
export { inpaint } from "./inpainting.js";
|
||||
export { noiseRemoval } from "./noise-removal.js";
|
||||
export type { PdfOcrOptions, PdfOcrResult } from "./ocr.js";
|
||||
export { extractPdfText, extractText } from "./ocr.js";
|
||||
export type {
|
||||
OcrExecutionMetadata,
|
||||
OcrOptions,
|
||||
OcrQuality,
|
||||
OcrResult,
|
||||
PdfOcrOptions,
|
||||
PdfOcrResult,
|
||||
} from "./ocr.js";
|
||||
export {
|
||||
extractPdfText,
|
||||
extractText,
|
||||
FAST_KOREAN_UNSUPPORTED_REASON,
|
||||
MAX_OCR_INPUT_DIMENSION,
|
||||
MAX_OCR_INPUT_PIXELS,
|
||||
} from "./ocr.js";
|
||||
export type {
|
||||
OcrRuntimeRunOptions,
|
||||
OcrRuntimeRunResult,
|
||||
OcrRuntimeScript,
|
||||
} from "./ocr-runtime-dispatcher.js";
|
||||
export {
|
||||
drainOcrDispatcher,
|
||||
handoffOcrDispatcher,
|
||||
probeOcrDispatcher,
|
||||
rotateOcrDispatcher,
|
||||
runOcrRuntime,
|
||||
shutdownOcrDispatcher,
|
||||
} from "./ocr-runtime-dispatcher.js";
|
||||
export type { OutpaintOptions } from "./outpainting.js";
|
||||
export { outpaint } from "./outpainting.js";
|
||||
export { removeRedEye } from "./red-eye-removal.js";
|
||||
export { restorePhoto } from "./restoration.js";
|
||||
export type {
|
||||
OcrRuntimeTrustKey,
|
||||
VerifiedOcrRuntimeIndex,
|
||||
} from "./runtime-index.js";
|
||||
export {
|
||||
canonicalRuntimeJson,
|
||||
loadOcrRuntimeTrustKeys,
|
||||
OCR_RUNTIME_INDEX_MAX_BYTES,
|
||||
verifyRuntimeIndex,
|
||||
} from "./runtime-index.js";
|
||||
export type { OcrRuntimeMemoryOptions } from "./runtime-resources.js";
|
||||
export {
|
||||
assertOcrRuntimeMemory,
|
||||
getOcrRuntimeEffectiveMemoryBytes,
|
||||
hasOcrRuntimeMemory,
|
||||
OCR_RUNTIME_MINIMUM_MEMORY_BYTES,
|
||||
} from "./runtime-resources.js";
|
||||
export type {
|
||||
ActiveRuntimeDescriptor,
|
||||
OcrRuntimeActivationIdentity,
|
||||
OcrRuntimeCapability,
|
||||
OcrRuntimeQuality,
|
||||
OcrRuntimeTarget,
|
||||
RuntimeIntegrityFile,
|
||||
RuntimeIntegrityFileId,
|
||||
RuntimePlatformOptions,
|
||||
RuntimeSignedIndex,
|
||||
RuntimeStateOptions,
|
||||
} from "./runtime-state.js";
|
||||
export {
|
||||
getOcrRuntimeCapability,
|
||||
OCR_RUNTIME_PROTOCOL_VERSION,
|
||||
readActiveRuntime,
|
||||
readCommittedOcrRuntimeActivationIdentity,
|
||||
readPendingOcrRuntimeActivationIdentity,
|
||||
resolveAiDataDir,
|
||||
selectOcrRuntimeTarget,
|
||||
} from "./runtime-state.js";
|
||||
export { seamCarve } from "./seam-carving.js";
|
||||
export type {
|
||||
RunTesseractOptions,
|
||||
TesseractLanguage,
|
||||
TesseractResult,
|
||||
TesseractRuntimeMetadata,
|
||||
} from "./tesseract.js";
|
||||
export {
|
||||
getTesseractRuntimeMetadata,
|
||||
resolveTesseractLanguage,
|
||||
runAdaptiveTesseract,
|
||||
runTesseract,
|
||||
selectTesseractLanguageFamily,
|
||||
selectTesseractLayout,
|
||||
TESSERACT_LANGUAGE_MAP,
|
||||
} from "./tesseract.js";
|
||||
export type {
|
||||
PreparedPdfOcrPage,
|
||||
PreparedPdfOcrPages,
|
||||
RunTesseractPdfOptions,
|
||||
TesseractPdfResult,
|
||||
} from "./tesseract-pdf.js";
|
||||
export {
|
||||
MAX_PDF_OCR_PAGES,
|
||||
parsePdfPageSpec,
|
||||
preparePdfOcrPages,
|
||||
runTesseractPdf,
|
||||
} from "./tesseract-pdf.js";
|
||||
export type { TranscribeOptions, TranscriptionResult, TranscriptSegment } from "./transcription.js";
|
||||
export { transcribeAudio } from "./transcription.js";
|
||||
export { upscale } from "./upscaling.js";
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+379
-48
@@ -1,9 +1,32 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import sharp from "sharp";
|
||||
import { type ProgressCallback, parseStdoutJson, runPythonWithProgress } from "./bridge.js";
|
||||
import type { ProgressCallback } from "./bridge.js";
|
||||
import { runOcrRuntime } from "./ocr-runtime-dispatcher.js";
|
||||
import { runAdaptiveTesseract, type TesseractLanguage } from "./tesseract.js";
|
||||
import { preparePdfOcrPages, runTesseractPdf } from "./tesseract-pdf.js";
|
||||
|
||||
export type OcrQuality = "fast" | "balanced" | "best";
|
||||
export const FAST_KOREAN_UNSUPPORTED_REASON =
|
||||
"Fast OCR does not support Korean. Install the Accurate OCR bundle and choose Balanced or Best.";
|
||||
export const MAX_OCR_INPUT_PIXELS = 40_000_000;
|
||||
/** Bound pathological aspect ratios so tiled OCR cannot fan out into thousands of sessions. */
|
||||
export const MAX_OCR_INPUT_DIMENSION = 40_000;
|
||||
/** Keep Fast aligned with the accurate runtime and durable-result database budget. */
|
||||
export const MAX_OCR_OUTPUT_BYTES = 1_000_000;
|
||||
const OCR_PROGRESS_HEARTBEAT_MS = 30_000;
|
||||
const FAST_LOW_CONTRAST_MAX_SHORT_SIDE = 512;
|
||||
const FAST_LOW_CONTRAST_MAX_LONG_SIDE = 1_024;
|
||||
const FAST_LOW_CONTRAST_MIN_MEAN = 200;
|
||||
const FAST_LOW_CONTRAST_MAX_STDEV = 20;
|
||||
const FAST_LOW_CONTRAST_GAIN = 4;
|
||||
const FAST_LOW_CONTRAST_TARGET_BACKGROUND = 250;
|
||||
const FAST_CJK_SCENE_MIN_PIXELS = 1_500_000;
|
||||
const FAST_CJK_SCENE_MIN_WIDTH = 1_000;
|
||||
const FAST_CJK_SCENE_MIN_HEIGHT = 1_200;
|
||||
const FAST_DENSE_CJK_MIN_PIXELS = 1_000_000;
|
||||
const FAST_DENSE_CJK_MAX_PIXELS = 2_500_000;
|
||||
const FAST_DENSE_CJK_MIN_WIDTH = 1_000;
|
||||
const FAST_DENSE_CJK_MIN_HEIGHT = 1_000;
|
||||
|
||||
export interface OcrOptions {
|
||||
quality?: OcrQuality;
|
||||
@@ -11,11 +34,116 @@ export interface OcrOptions {
|
||||
enhance?: boolean;
|
||||
/** @deprecated Use quality instead. Kept for backward compat. */
|
||||
engine?: "tesseract" | "paddleocr";
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface OcrResult {
|
||||
export interface OcrExecutionMetadata {
|
||||
engine: string;
|
||||
requestedQuality: OcrQuality;
|
||||
actualQuality: OcrQuality;
|
||||
device: "cpu" | "cuda";
|
||||
degraded: boolean;
|
||||
warnings: string[];
|
||||
provider: string;
|
||||
runtimeVersion?: string;
|
||||
modelVersion?: string;
|
||||
}
|
||||
|
||||
export interface OcrResult extends OcrExecutionMetadata {
|
||||
text: string;
|
||||
engine?: string;
|
||||
}
|
||||
|
||||
function resolveQuality(options: OcrOptions): OcrQuality {
|
||||
if (options.quality) return options.quality;
|
||||
if (options.engine) return options.engine === "tesseract" ? "fast" : "balanced";
|
||||
return "fast";
|
||||
}
|
||||
|
||||
function assertFastLanguageSupported(quality: OcrQuality, language: string | undefined): void {
|
||||
if (quality === "fast" && language === "ko") {
|
||||
throw new Error(FAST_KOREAN_UNSUPPORTED_REASON);
|
||||
}
|
||||
}
|
||||
|
||||
function parseAccurateResult(resultValue: unknown, quality: OcrQuality): OcrResult {
|
||||
if (typeof resultValue !== "object" || resultValue === null || Array.isArray(resultValue)) {
|
||||
throw new Error("OCR runtime returned invalid metadata");
|
||||
}
|
||||
const result = resultValue as Record<string, unknown>;
|
||||
if (result.success !== true) {
|
||||
throw new Error((result.error as string | undefined) || "OCR failed");
|
||||
}
|
||||
|
||||
const metadataValid =
|
||||
typeof result.text === "string" &&
|
||||
typeof result.engine === "string" &&
|
||||
(result.requestedQuality === "fast" ||
|
||||
result.requestedQuality === "balanced" ||
|
||||
result.requestedQuality === "best") &&
|
||||
(result.actualQuality === "fast" ||
|
||||
result.actualQuality === "balanced" ||
|
||||
result.actualQuality === "best") &&
|
||||
(result.device === "cpu" || result.device === "cuda") &&
|
||||
typeof result.provider === "string" &&
|
||||
typeof result.degraded === "boolean" &&
|
||||
Array.isArray(result.warnings) &&
|
||||
result.warnings.every((warning) => typeof warning === "string") &&
|
||||
(result.runtimeVersion === undefined || typeof result.runtimeVersion === "string") &&
|
||||
(result.modelVersion === undefined || typeof result.modelVersion === "string");
|
||||
|
||||
if (!metadataValid) {
|
||||
throw new Error("OCR runtime returned invalid metadata");
|
||||
}
|
||||
|
||||
if (result.requestedQuality !== quality || result.actualQuality !== quality) {
|
||||
throw new Error(
|
||||
`OCR runtime tier mismatch: requested ${quality}, reported ${String(result.requestedQuality)}/${String(result.actualQuality)}`,
|
||||
);
|
||||
}
|
||||
if (Buffer.byteLength(result.text as string, "utf8") > MAX_OCR_OUTPUT_BYTES) {
|
||||
throw new Error(
|
||||
`OCR runtime output exceeds the ${MAX_OCR_OUTPUT_BYTES.toLocaleString("en-US")} byte safety limit`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
text: result.text as string,
|
||||
engine: result.engine as string,
|
||||
requestedQuality: quality,
|
||||
actualQuality: quality,
|
||||
device: result.device as "cpu" | "cuda",
|
||||
provider: result.provider as string,
|
||||
degraded: result.degraded as boolean,
|
||||
warnings: result.warnings as string[],
|
||||
...(typeof result.runtimeVersion === "string" && { runtimeVersion: result.runtimeVersion }),
|
||||
...(typeof result.modelVersion === "string" && { modelVersion: result.modelVersion }),
|
||||
};
|
||||
}
|
||||
|
||||
async function withProgressHeartbeat<T>(
|
||||
operation: (progress: ProgressCallback | undefined) => Promise<T>,
|
||||
onProgress: ProgressCallback | undefined,
|
||||
percent: number,
|
||||
stage: string,
|
||||
): Promise<T> {
|
||||
if (!onProgress) return operation(undefined);
|
||||
let latestPercent = percent;
|
||||
let latestStage = stage;
|
||||
const relayProgress: ProgressCallback = (nextPercent, nextStage) => {
|
||||
latestPercent = nextPercent;
|
||||
latestStage = nextStage;
|
||||
onProgress(nextPercent, nextStage);
|
||||
};
|
||||
const heartbeat = setInterval(
|
||||
() => onProgress(latestPercent, latestStage),
|
||||
OCR_PROGRESS_HEARTBEAT_MS,
|
||||
);
|
||||
heartbeat.unref();
|
||||
try {
|
||||
return await operation(relayProgress);
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
}
|
||||
|
||||
export async function extractText(
|
||||
@@ -25,33 +153,181 @@ export async function extractText(
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<OcrResult> {
|
||||
const inputPath = join(outputDir, "input_ocr.png");
|
||||
const quality = resolveQuality(options);
|
||||
assertFastLanguageSupported(quality, options.language);
|
||||
|
||||
// Convert to PNG and cap at 2048px to prevent PaddleOCR OOM on large images.
|
||||
const MAX_OCR_DIM = 2048;
|
||||
const pngBuffer = await sharp(inputBuffer)
|
||||
.resize({ width: MAX_OCR_DIM, height: MAX_OCR_DIM, fit: "inside", withoutEnlargement: true })
|
||||
.png()
|
||||
.toBuffer();
|
||||
await writeFile(inputPath, pngBuffer);
|
||||
// Normalize the format without discarding source pixels. The old 2048px cap
|
||||
// made small text permanently unreadable before either OCR engine saw it.
|
||||
const image = sharp(inputBuffer);
|
||||
const meta = await image.metadata();
|
||||
const width = meta.width ?? 0;
|
||||
const height = meta.height ?? 0;
|
||||
const pixels = width * height;
|
||||
if (!Number.isSafeInteger(pixels) || pixels <= 0) {
|
||||
throw new Error("OCR input has invalid image dimensions");
|
||||
}
|
||||
if (width > MAX_OCR_INPUT_DIMENSION || height > MAX_OCR_INPUT_DIMENSION) {
|
||||
throw new Error(
|
||||
`OCR input exceeds the ${MAX_OCR_INPUT_DIMENSION.toLocaleString("en-US")} pixel dimension safety limit`,
|
||||
);
|
||||
}
|
||||
if (pixels > MAX_OCR_INPUT_PIXELS) {
|
||||
throw new Error(
|
||||
`OCR input exceeds the ${MAX_OCR_INPUT_PIXELS.toLocaleString("en-US")} pixel safety limit`,
|
||||
);
|
||||
}
|
||||
let recognitionImage: ReturnType<typeof sharp> | undefined;
|
||||
let automaticLowContrast = false;
|
||||
if (
|
||||
quality === "fast" &&
|
||||
Math.min(width, height) <= FAST_LOW_CONTRAST_MAX_SHORT_SIDE &&
|
||||
Math.max(width, height) <= FAST_LOW_CONTRAST_MAX_LONG_SIDE
|
||||
) {
|
||||
const stats = await image.clone().grayscale().stats();
|
||||
const luminance = stats.channels[0];
|
||||
if (
|
||||
luminance &&
|
||||
Number.isFinite(luminance.mean) &&
|
||||
Number.isFinite(luminance.stdev) &&
|
||||
luminance.mean >= FAST_LOW_CONTRAST_MIN_MEAN &&
|
||||
luminance.stdev <= FAST_LOW_CONTRAST_MAX_STDEV
|
||||
) {
|
||||
// Development receipts showed that a fixed gain with a mean-derived
|
||||
// offset recovers faint thermal text without upscaling or threshold
|
||||
// artifacts. The strict size/statistics gate leaves ordinary images
|
||||
// byte-for-byte equivalent apart from the existing PNG normalization.
|
||||
recognitionImage = image
|
||||
.clone()
|
||||
.grayscale()
|
||||
.linear(
|
||||
FAST_LOW_CONTRAST_GAIN,
|
||||
Math.round(FAST_LOW_CONTRAST_TARGET_BACKGROUND - FAST_LOW_CONTRAST_GAIN * luminance.mean),
|
||||
);
|
||||
automaticLowContrast = true;
|
||||
}
|
||||
}
|
||||
if (quality === "fast" && options.enhance && !automaticLowContrast) {
|
||||
recognitionImage = image.clone().clahe({
|
||||
width: Math.max(1, Math.min(256, Math.round(width / 8))),
|
||||
height: Math.max(1, Math.min(256, Math.round(height / 8))),
|
||||
maxSlope: 2,
|
||||
});
|
||||
}
|
||||
// Stream normalized rasters straight to scratch. When preprocessing is
|
||||
// active, preserve an original PNG for auto script-family probes and keep
|
||||
// the enhanced raster separate for recognition. This prevents contrast
|
||||
// transforms from turning faint Latin noise into false CJK evidence.
|
||||
await image.png().toFile(inputPath);
|
||||
const recognitionInputPath = recognitionImage
|
||||
? join(outputDir, "input_ocr_recognition.png")
|
||||
: inputPath;
|
||||
if (recognitionImage) await recognitionImage.png().toFile(recognitionInputPath);
|
||||
|
||||
const meta = await sharp(pngBuffer).metadata();
|
||||
const megapixels = ((meta.width ?? 0) * (meta.height ?? 0)) / 1_000_000;
|
||||
const selectedLanguage = options.language ?? "auto";
|
||||
const canContainCjkSceneText =
|
||||
selectedLanguage === "auto" || selectedLanguage === "ja" || selectedLanguage === "zh";
|
||||
const fallbackInputProvider =
|
||||
quality === "fast" &&
|
||||
canContainCjkSceneText &&
|
||||
pixels >= FAST_CJK_SCENE_MIN_PIXELS &&
|
||||
width >= FAST_CJK_SCENE_MIN_WIDTH &&
|
||||
height >= FAST_CJK_SCENE_MIN_HEIGHT
|
||||
? async () => {
|
||||
// A whole mixed-polarity scene can hide dense light-on-dark CJK text
|
||||
// from Tesseract even when each local band is clean. Split lazily so
|
||||
// ordinary and strong primary results pay no extra raster/process cost.
|
||||
const splitY = Math.floor(height / 2);
|
||||
const paths = [
|
||||
join(outputDir, "input_ocr_scene_upper.png"),
|
||||
join(outputDir, "input_ocr_scene_lower.png"),
|
||||
] as const;
|
||||
await sharp(recognitionInputPath)
|
||||
.extract({ left: 0, top: 0, width, height: splitY })
|
||||
.png()
|
||||
.toFile(paths[0]);
|
||||
await sharp(recognitionInputPath)
|
||||
.extract({ left: 0, top: splitY, width, height: height - splitY })
|
||||
.png()
|
||||
.toFile(paths[1]);
|
||||
return paths;
|
||||
}
|
||||
: undefined;
|
||||
const denseCjkInputProvider =
|
||||
quality === "fast" &&
|
||||
canContainCjkSceneText &&
|
||||
pixels >= FAST_DENSE_CJK_MIN_PIXELS &&
|
||||
pixels <= FAST_DENSE_CJK_MAX_PIXELS &&
|
||||
width >= FAST_DENSE_CJK_MIN_WIDTH &&
|
||||
height >= FAST_DENSE_CJK_MIN_HEIGHT
|
||||
? async () => {
|
||||
// Small, dense CJK boards are the one scene class where a confident
|
||||
// sparse fragment can hide most of the page. Build this candidate
|
||||
// lazily only after the primary pass is weak; the adaptive runner
|
||||
// accepts it only when confidence and recovered coverage both rise.
|
||||
const denseCjkInputPath = join(outputDir, "input_ocr_dense_cjk.png");
|
||||
await sharp(inputPath)
|
||||
.grayscale()
|
||||
.clahe({
|
||||
width: Math.max(1, Math.min(256, Math.round(width / 8))),
|
||||
height: Math.max(1, Math.min(256, Math.round(height / 8))),
|
||||
maxSlope: 2,
|
||||
})
|
||||
.sharpen({ sigma: 1 })
|
||||
.png()
|
||||
.toFile(denseCjkInputPath);
|
||||
return denseCjkInputPath;
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const megapixels = pixels / 1_000_000;
|
||||
const timeout = Math.max(600_000, megapixels * 30 * 1000);
|
||||
|
||||
const { stdout } = await runPythonWithProgress("ocr.py", [inputPath, JSON.stringify(options)], {
|
||||
onProgress,
|
||||
timeout,
|
||||
});
|
||||
|
||||
const result = parseStdoutJson(stdout);
|
||||
if (!result.success) {
|
||||
throw new Error(result.error || "OCR failed");
|
||||
if (quality === "fast") {
|
||||
const result = await withProgressHeartbeat(
|
||||
(heartbeatProgress) =>
|
||||
runAdaptiveTesseract(inputPath, {
|
||||
language: (options.language ?? "auto") as TesseractLanguage,
|
||||
...(automaticLowContrast && { blockLayoutOnly: true }),
|
||||
...(recognitionInputPath !== inputPath && { recognitionInputPath }),
|
||||
...(fallbackInputProvider && { fallbackInputProvider }),
|
||||
...(denseCjkInputProvider && { denseCjkInputProvider }),
|
||||
timeoutMs: timeout,
|
||||
maxStdoutBytes: MAX_OCR_OUTPUT_BYTES,
|
||||
signal: options.signal,
|
||||
onProgress: heartbeatProgress,
|
||||
}),
|
||||
onProgress,
|
||||
10,
|
||||
"Running Fast OCR",
|
||||
);
|
||||
return {
|
||||
...result,
|
||||
requestedQuality: "fast",
|
||||
actualQuality: "fast",
|
||||
degraded: false,
|
||||
warnings: automaticLowContrast ? ["Applied automatic low-contrast OCR preprocessing."] : [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
text: result.text,
|
||||
engine: result.engine,
|
||||
const runtimeOptions = {
|
||||
quality,
|
||||
...(options.language !== undefined && { language: options.language }),
|
||||
enhance: options.enhance ?? quality === "best",
|
||||
};
|
||||
|
||||
onProgress?.(10, "Starting accurate OCR");
|
||||
const { result } = await withProgressHeartbeat(
|
||||
() =>
|
||||
runOcrRuntime("ocr", [inputPath, JSON.stringify(runtimeOptions)], {
|
||||
timeoutMs: timeout,
|
||||
signal: options.signal,
|
||||
}),
|
||||
onProgress,
|
||||
10,
|
||||
"Running accurate OCR",
|
||||
);
|
||||
onProgress?.(100, "Accurate OCR complete");
|
||||
return parseAccurateResult(result, quality);
|
||||
}
|
||||
|
||||
// ── PDF OCR ───────────────────────────────────────────────────────────
|
||||
@@ -60,11 +336,12 @@ export interface PdfOcrOptions {
|
||||
quality?: OcrQuality;
|
||||
language?: string;
|
||||
pages?: string;
|
||||
enhance?: boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface PdfOcrResult {
|
||||
export interface PdfOcrResult extends OcrExecutionMetadata {
|
||||
text: string;
|
||||
engine: string;
|
||||
pages: number;
|
||||
}
|
||||
|
||||
@@ -73,28 +350,82 @@ export async function extractPdfText(
|
||||
opts: PdfOcrOptions = {},
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<PdfOcrResult> {
|
||||
const optionsJson = JSON.stringify({
|
||||
quality: opts.quality ?? "balanced",
|
||||
language: opts.language ?? "auto",
|
||||
pages: opts.pages ?? "all",
|
||||
});
|
||||
const quality = opts.quality ?? "fast";
|
||||
assertFastLanguageSupported(quality, opts.language);
|
||||
if (quality === "fast") {
|
||||
const result = await withProgressHeartbeat(
|
||||
(heartbeatProgress) =>
|
||||
runTesseractPdf(inputPath, dirname(inputPath), {
|
||||
pages: opts.pages ?? "all",
|
||||
language: (opts.language ?? "auto") as TesseractLanguage,
|
||||
enhance: opts.enhance ?? false,
|
||||
signal: opts.signal,
|
||||
onProgress: heartbeatProgress,
|
||||
}),
|
||||
onProgress,
|
||||
10,
|
||||
"Running Fast PDF OCR",
|
||||
);
|
||||
return {
|
||||
text: result.text,
|
||||
pages: result.pages,
|
||||
engine: result.engine,
|
||||
provider: result.provider,
|
||||
device: result.device,
|
||||
requestedQuality: "fast",
|
||||
actualQuality: "fast",
|
||||
degraded: false,
|
||||
warnings: [],
|
||||
};
|
||||
}
|
||||
|
||||
const { stdout } = await runPythonWithProgress("ocr_pdf.py", [inputPath, optionsJson], {
|
||||
timeout: 30 * 60_000,
|
||||
const prepared = await withProgressHeartbeat(
|
||||
(heartbeatProgress) =>
|
||||
preparePdfOcrPages(inputPath, dirname(inputPath), {
|
||||
pages: opts.pages ?? "all",
|
||||
signal: opts.signal,
|
||||
onProgress: heartbeatProgress,
|
||||
}),
|
||||
onProgress,
|
||||
});
|
||||
|
||||
const result = parseStdoutJson(stdout);
|
||||
if (result.error) {
|
||||
throw new Error(result.error);
|
||||
10,
|
||||
"Preparing accurate PDF OCR",
|
||||
);
|
||||
try {
|
||||
onProgress?.(50, "Starting accurate PDF OCR");
|
||||
const { result } = await withProgressHeartbeat(
|
||||
() =>
|
||||
runOcrRuntime(
|
||||
"ocr_pdf",
|
||||
[
|
||||
JSON.stringify(prepared.pages),
|
||||
JSON.stringify({
|
||||
quality,
|
||||
language: opts.language ?? "auto",
|
||||
enhance: opts.enhance ?? quality === "best",
|
||||
}),
|
||||
],
|
||||
{
|
||||
timeoutMs: prepared.remainingTimeoutMs(),
|
||||
signal: opts.signal,
|
||||
},
|
||||
),
|
||||
onProgress,
|
||||
50,
|
||||
"Running accurate PDF OCR",
|
||||
);
|
||||
const ocr = parseAccurateResult(result, quality);
|
||||
const resultRecord = result as Record<string, unknown>;
|
||||
if (resultRecord.pages !== prepared.pages.length) {
|
||||
throw new Error(
|
||||
`PDF OCR runtime page count mismatch: expected ${prepared.pages.length}, received ${String(resultRecord.pages)}`,
|
||||
);
|
||||
}
|
||||
onProgress?.(100, "Accurate PDF OCR complete");
|
||||
return {
|
||||
...ocr,
|
||||
pages: resultRecord.pages as number,
|
||||
};
|
||||
} finally {
|
||||
await prepared.cleanup();
|
||||
}
|
||||
if (!result.success) {
|
||||
throw new Error(result.error || "PDF OCR failed");
|
||||
}
|
||||
|
||||
return {
|
||||
text: result.text ?? "",
|
||||
engine: result.engine ?? "unknown",
|
||||
pages: result.pages ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
import { createPublicKey, verify } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export type OcrRuntimeTarget = "linux-amd64-cpu-py312" | "linux-arm64-cpu-py311";
|
||||
|
||||
export interface OcrRuntimeTrustKey {
|
||||
keyId: string;
|
||||
algorithm: "ed25519";
|
||||
publicKey: string;
|
||||
}
|
||||
|
||||
export interface VerifiedOcrRuntimeIndex {
|
||||
artifact: Record<string, unknown>;
|
||||
canonicalIndex: Buffer;
|
||||
archiveFile: string;
|
||||
archiveSha256: string;
|
||||
archiveSize: number;
|
||||
archiveExpandedSize: number;
|
||||
minimumMemoryBytes: number;
|
||||
}
|
||||
|
||||
export const OCR_RUNTIME_INDEX_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
||||
const SAFE_COMPONENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
||||
const PROJECT_ROOT = resolve(fileURLToPath(new URL("../../..", import.meta.url)));
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function sortJson(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(sortJson);
|
||||
if (!isRecord(value)) return value;
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, sortJson(value[key])]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Canonical representation shared with install_runtime.py and release signing. */
|
||||
export function canonicalRuntimeJson(value: unknown): string {
|
||||
return `${JSON.stringify(sortJson(value))}\n`;
|
||||
}
|
||||
|
||||
/** Resolve either the image-pinned release key or an operator-supplied trust store. */
|
||||
export function loadOcrRuntimeTrustKeys(path?: string): OcrRuntimeTrustKey[] {
|
||||
if (!path) {
|
||||
const keyId = process.env.OCR_RUNTIME_INDEX_KEY_ID;
|
||||
const encodedPublicKey = process.env.OCR_RUNTIME_INDEX_PUBLIC_KEY_PEM_B64;
|
||||
if (keyId || encodedPublicKey) {
|
||||
if (
|
||||
!keyId ||
|
||||
!SAFE_COMPONENT_PATTERN.test(keyId) ||
|
||||
!encodedPublicKey ||
|
||||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encodedPublicKey)
|
||||
) {
|
||||
throw new Error("OCR runtime trust environment is incomplete or invalid");
|
||||
}
|
||||
const decoded = Buffer.from(encodedPublicKey, "base64");
|
||||
if (decoded.toString("base64") !== encodedPublicKey) {
|
||||
throw new Error("OCR runtime public key is not canonical base64");
|
||||
}
|
||||
return [{ keyId, algorithm: "ed25519", publicKey: decoded.toString("utf8") }];
|
||||
}
|
||||
}
|
||||
|
||||
const trustPath = path ?? join(PROJECT_ROOT, "docker", "ocr-runtime-trust.json");
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(trustPath, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to read the OCR runtime trust store at ${trustPath}`, { cause: error });
|
||||
}
|
||||
if (!isRecord(parsed) || parsed.schemaVersion !== 1 || !Array.isArray(parsed.keys)) {
|
||||
throw new Error("OCR runtime trust store uses an unsupported schema");
|
||||
}
|
||||
const keys: OcrRuntimeTrustKey[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const value of parsed.keys) {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
typeof value.keyId !== "string" ||
|
||||
!SAFE_COMPONENT_PATTERN.test(value.keyId) ||
|
||||
value.algorithm !== "ed25519" ||
|
||||
typeof value.publicKey !== "string" ||
|
||||
!value.publicKey
|
||||
) {
|
||||
throw new Error("OCR runtime trust store contains an invalid key");
|
||||
}
|
||||
if (seen.has(value.keyId)) throw new Error(`Duplicate OCR runtime trust key: ${value.keyId}`);
|
||||
seen.add(value.keyId);
|
||||
keys.push({
|
||||
keyId: value.keyId,
|
||||
algorithm: "ed25519",
|
||||
publicKey: value.publicKey,
|
||||
});
|
||||
}
|
||||
if (keys.length === 0) throw new Error("OCR runtime trust store contains no keys");
|
||||
return keys;
|
||||
}
|
||||
|
||||
function safeRelativeReleasePath(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || !value || value.includes("\\") || value.includes("\0")) {
|
||||
throw new Error(`OCR runtime index contains an invalid ${label}`);
|
||||
}
|
||||
const parts = value.split("/");
|
||||
if (
|
||||
value.startsWith("/") ||
|
||||
value.endsWith("/") ||
|
||||
parts.some((part) => !SAFE_COMPONENT_PATTERN.test(part)) ||
|
||||
value.includes("://")
|
||||
) {
|
||||
throw new Error(`OCR runtime index contains an unsafe ${label}`);
|
||||
}
|
||||
return parts.join("/");
|
||||
}
|
||||
|
||||
function decodeSignature(value: unknown): Buffer {
|
||||
if (
|
||||
typeof value !== "string" ||
|
||||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)
|
||||
) {
|
||||
throw new Error("OCR runtime index contains an invalid signature encoding");
|
||||
}
|
||||
const decoded = Buffer.from(value, "base64");
|
||||
if (decoded.length !== 64 || decoded.toString("base64") !== value) {
|
||||
throw new Error("OCR runtime index contains an invalid Ed25519 signature");
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/** Authenticate a canonical release index and select its one compatible OCR artifact. */
|
||||
export function verifyRuntimeIndex(
|
||||
raw: Buffer,
|
||||
target: OcrRuntimeTarget,
|
||||
trustKeys: readonly OcrRuntimeTrustKey[],
|
||||
snapotterVersion: string,
|
||||
): VerifiedOcrRuntimeIndex {
|
||||
if (raw.length === 0 || raw.length > OCR_RUNTIME_INDEX_MAX_BYTES) {
|
||||
throw new Error("OCR runtime index exceeds its size limit");
|
||||
}
|
||||
if (raw.some((byte) => byte > 0x7f)) {
|
||||
throw new Error("OCR runtime index metadata must be canonical ASCII JSON");
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw.toString("utf8"));
|
||||
} catch (error) {
|
||||
throw new Error("OCR runtime index is not valid JSON", { cause: error });
|
||||
}
|
||||
if (!isRecord(parsed) || parsed.schemaVersion !== 1) {
|
||||
throw new Error("OCR runtime index uses an unsupported schema");
|
||||
}
|
||||
if (!raw.equals(Buffer.from(canonicalRuntimeJson(parsed)))) {
|
||||
throw new Error("OCR runtime index is not canonical JSON");
|
||||
}
|
||||
|
||||
const signature = parsed.signature;
|
||||
if (
|
||||
!isRecord(signature) ||
|
||||
typeof signature.keyId !== "string" ||
|
||||
signature.algorithm !== "ed25519"
|
||||
) {
|
||||
throw new Error("OCR runtime index has an invalid signature envelope");
|
||||
}
|
||||
const trustKey = trustKeys.find(
|
||||
(candidate) =>
|
||||
candidate.keyId === signature.keyId && candidate.algorithm === signature.algorithm,
|
||||
);
|
||||
if (!trustKey) {
|
||||
throw new Error(`OCR runtime index key "${signature.keyId}" is not trusted`);
|
||||
}
|
||||
|
||||
const unsigned = { ...parsed };
|
||||
delete unsigned.signature;
|
||||
let publicKey: ReturnType<typeof createPublicKey>;
|
||||
try {
|
||||
publicKey = createPublicKey(trustKey.publicKey);
|
||||
} catch (error) {
|
||||
throw new Error(`Trusted OCR runtime key "${trustKey.keyId}" is invalid`, { cause: error });
|
||||
}
|
||||
if (publicKey.asymmetricKeyType !== "ed25519") {
|
||||
throw new Error(`Trusted OCR runtime key "${trustKey.keyId}" is not Ed25519`);
|
||||
}
|
||||
const valid = verify(
|
||||
null,
|
||||
Buffer.from(canonicalRuntimeJson(unsigned)),
|
||||
publicKey,
|
||||
decodeSignature(signature.value),
|
||||
);
|
||||
if (!valid) throw new Error("OCR runtime index signature verification failed");
|
||||
|
||||
if (!Array.isArray(parsed.artifacts)) {
|
||||
throw new Error("OCR runtime index artifacts must be an array");
|
||||
}
|
||||
const matches = parsed.artifacts.filter(
|
||||
(artifact): artifact is Record<string, unknown> =>
|
||||
isRecord(artifact) && artifact.family === "ocr" && artifact.target === target,
|
||||
);
|
||||
if (matches.length !== 1) {
|
||||
throw new Error(`OCR runtime index must contain exactly one artifact for ocr/${target}`);
|
||||
}
|
||||
const artifact = matches[0];
|
||||
const compatibility = artifact.compatibility;
|
||||
if (
|
||||
!isRecord(compatibility) ||
|
||||
compatibility.protocolVersion !== 1 ||
|
||||
compatibility.snapotterVersion !== snapotterVersion ||
|
||||
artifact.version !== snapotterVersion
|
||||
) {
|
||||
throw new Error(
|
||||
`OCR runtime artifact version is incompatible with SnapOtter ${snapotterVersion}`,
|
||||
);
|
||||
}
|
||||
const expectedArch = target === "linux-amd64-cpu-py312" ? "amd64" : "arm64";
|
||||
if (artifact.platform !== "linux" || artifact.arch !== expectedArch) {
|
||||
throw new Error("OCR runtime artifact platform does not match the selected target");
|
||||
}
|
||||
const capabilities = artifact.capabilities;
|
||||
if (
|
||||
!isRecord(capabilities) ||
|
||||
!Array.isArray(capabilities.qualities) ||
|
||||
capabilities.qualities.length !== 2 ||
|
||||
!capabilities.qualities.includes("balanced") ||
|
||||
!capabilities.qualities.includes("best") ||
|
||||
!Array.isArray(capabilities.providers) ||
|
||||
capabilities.providers.length !== 1 ||
|
||||
capabilities.providers[0] !== "CPUExecutionProvider"
|
||||
) {
|
||||
throw new Error("OCR runtime artifact declares unsupported capabilities");
|
||||
}
|
||||
const resources = artifact.resources;
|
||||
if (
|
||||
!isRecord(resources) ||
|
||||
typeof resources.minimumMemoryBytes !== "number" ||
|
||||
!Number.isSafeInteger(resources.minimumMemoryBytes) ||
|
||||
resources.minimumMemoryBytes <= 0
|
||||
) {
|
||||
throw new Error("OCR runtime artifact has an invalid minimum memory requirement");
|
||||
}
|
||||
const archive = artifact.archive;
|
||||
if (!isRecord(archive)) throw new Error("OCR runtime artifact has no archive metadata");
|
||||
const archiveFile = safeRelativeReleasePath(archive.file, "archive file");
|
||||
if (typeof archive.sha256 !== "string" || !SHA256_PATTERN.test(archive.sha256)) {
|
||||
throw new Error("OCR runtime artifact has an invalid archive digest");
|
||||
}
|
||||
if (
|
||||
typeof archive.size !== "number" ||
|
||||
!Number.isSafeInteger(archive.size) ||
|
||||
archive.size <= 0
|
||||
) {
|
||||
throw new Error("OCR runtime artifact has an invalid archive size");
|
||||
}
|
||||
if (
|
||||
typeof archive.expandedSize !== "number" ||
|
||||
!Number.isSafeInteger(archive.expandedSize) ||
|
||||
archive.expandedSize < 0
|
||||
) {
|
||||
throw new Error("OCR runtime artifact has an invalid expanded archive size");
|
||||
}
|
||||
|
||||
return {
|
||||
artifact,
|
||||
canonicalIndex: raw,
|
||||
archiveFile,
|
||||
archiveSha256: archive.sha256,
|
||||
archiveSize: archive.size,
|
||||
archiveExpandedSize: archive.expandedSize,
|
||||
minimumMemoryBytes: resources.minimumMemoryBytes,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { totalmem } from "node:os";
|
||||
import { posix } from "node:path";
|
||||
|
||||
export const OCR_RUNTIME_MINIMUM_MEMORY_BYTES = 4 * 1024 * 1024 * 1024;
|
||||
|
||||
const CGROUP_MEMORY_LIMIT_PATHS = [
|
||||
"/sys/fs/cgroup/memory.max",
|
||||
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
|
||||
"/sys/fs/cgroup/memory.limit_in_bytes",
|
||||
] as const;
|
||||
const CGROUP_MEMBERSHIP_RESOLUTION_ATTEMPTS = 3;
|
||||
|
||||
export interface OcrRuntimeMemoryOptions {
|
||||
/** Exact test/caller override after physical and cgroup limits are resolved. */
|
||||
effectiveMemoryBytes?: number;
|
||||
/** Test seam for the host's configured physical capacity. */
|
||||
physicalMemoryBytes?: number;
|
||||
/** Test seam for cgroup v1/v2 capacity files. */
|
||||
readTextFile?: (path: string) => string;
|
||||
/** Test seam for Linux fail-closed cgroup discovery. */
|
||||
hostPlatform?: NodeJS.Platform;
|
||||
}
|
||||
|
||||
function positiveSafeBytes(value: number, label: string): number {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new Error(`${label} must be a positive safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCgroupLimit(raw: string, zeroIsLimit = false): bigint | null {
|
||||
const value = raw.trim();
|
||||
if (value === "max" || !/^[0-9]+$/.test(value)) return null;
|
||||
const parsed = BigInt(value);
|
||||
return parsed > 0n || zeroIsLimit ? parsed : null;
|
||||
}
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return typeof error === "object" && error !== null && "code" in error && error.code === code;
|
||||
}
|
||||
|
||||
function hasParentPathSegment(value: string): boolean {
|
||||
return value.split("/").includes("..");
|
||||
}
|
||||
|
||||
function decodeMountInfoPath(value: string): string {
|
||||
return value.replace(/\\([0-7]{3})/g, (_match, octal: string) =>
|
||||
String.fromCharCode(Number.parseInt(octal, 8)),
|
||||
);
|
||||
}
|
||||
|
||||
interface MountInfoRecord {
|
||||
id: number;
|
||||
parentId: number;
|
||||
device: string;
|
||||
filesystem: string;
|
||||
root: string;
|
||||
mountPoint: string;
|
||||
controllers: Set<string>;
|
||||
}
|
||||
|
||||
interface CgroupMount extends MountInfoRecord {
|
||||
filesystem: "cgroup" | "cgroup2";
|
||||
}
|
||||
|
||||
function parseMountInfo(raw: string): MountInfoRecord[] {
|
||||
const mounts: MountInfoRecord[] = [];
|
||||
for (const line of raw.split("\n")) {
|
||||
if (!line) continue;
|
||||
const fields = line.split(" ");
|
||||
const separator = fields.indexOf("-");
|
||||
if (
|
||||
separator < 6 ||
|
||||
!/^[1-9][0-9]*$/.test(fields[0]) ||
|
||||
!/^[1-9][0-9]*$/.test(fields[1]) ||
|
||||
!/^[0-9]+:[0-9]+$/.test(fields[2])
|
||||
) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
const id = Number(fields[0]);
|
||||
const parentId = Number(fields[1]);
|
||||
if (!Number.isSafeInteger(id) || !Number.isSafeInteger(parentId)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
const filesystem = fields[separator + 1];
|
||||
const root = decodeMountInfoPath(fields[3]);
|
||||
const decodedMountPoint = decodeMountInfoPath(fields[4]);
|
||||
if (!decodedMountPoint.startsWith("/") || hasParentPathSegment(decodedMountPoint)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
mounts.push({
|
||||
id,
|
||||
parentId,
|
||||
device: fields[2],
|
||||
filesystem,
|
||||
root,
|
||||
mountPoint: posix.normalize(decodedMountPoint),
|
||||
controllers: new Set(
|
||||
fields
|
||||
.slice(separator + 3)
|
||||
.join(",")
|
||||
.split(","),
|
||||
),
|
||||
});
|
||||
}
|
||||
return mounts;
|
||||
}
|
||||
|
||||
function isStrictPathPrefix(parent: string, child: string): boolean {
|
||||
return parent === "/" ? child !== "/" : child.startsWith(`${parent}/`);
|
||||
}
|
||||
|
||||
function pathDepth(value: string): number {
|
||||
return value.split("/").filter(Boolean).length;
|
||||
}
|
||||
|
||||
function parseVisibleMounts(raw: string): MountInfoRecord[] {
|
||||
const mounts = parseMountInfo(raw);
|
||||
const mountsById = new Map<number, MountInfoRecord>();
|
||||
for (const mount of mounts) {
|
||||
if (mountsById.has(mount.id)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
mountsById.set(mount.id, mount);
|
||||
}
|
||||
|
||||
const parentStates = new Map<number, "visiting" | "visited">();
|
||||
const validateParentChain = (mount: MountInfoRecord): void => {
|
||||
if (mount.parentId === mount.id) {
|
||||
if (mount.mountPoint !== "/") {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
parentStates.set(mount.id, "visited");
|
||||
return;
|
||||
}
|
||||
const state = parentStates.get(mount.id);
|
||||
if (state === "visiting") {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
if (state === "visited") return;
|
||||
parentStates.set(mount.id, "visiting");
|
||||
const parent = mountsById.get(mount.parentId);
|
||||
if (parent) validateParentChain(parent);
|
||||
parentStates.set(mount.id, "visited");
|
||||
};
|
||||
for (const mount of mounts) validateParentChain(mount);
|
||||
|
||||
const coveredIds = new Set<number>();
|
||||
for (const mount of mounts) {
|
||||
const parent = mountsById.get(mount.parentId);
|
||||
if (parent && parent.id !== mount.id && parent.mountPoint === mount.mountPoint) {
|
||||
coveredIds.add(parent.id);
|
||||
}
|
||||
}
|
||||
|
||||
const visibleMounts: MountInfoRecord[] = [];
|
||||
const visibleIds = new Set<number>();
|
||||
const topMounts = mounts
|
||||
.filter((mount) => !coveredIds.has(mount.id))
|
||||
.sort((left, right) => pathDepth(left.mountPoint) - pathDepth(right.mountPoint));
|
||||
for (const mount of topMounts) {
|
||||
let containingParent = mountsById.get(mount.parentId);
|
||||
while (containingParent?.mountPoint === mount.mountPoint) {
|
||||
if (containingParent.parentId === containingParent.id) {
|
||||
containingParent = undefined;
|
||||
break;
|
||||
}
|
||||
containingParent = mountsById.get(containingParent.parentId);
|
||||
}
|
||||
|
||||
let longestVisiblePrefix: MountInfoRecord | undefined;
|
||||
for (const visibleMount of visibleMounts) {
|
||||
if (
|
||||
isStrictPathPrefix(visibleMount.mountPoint, mount.mountPoint) &&
|
||||
(!longestVisiblePrefix ||
|
||||
visibleMount.mountPoint.length > longestVisiblePrefix.mountPoint.length)
|
||||
) {
|
||||
longestVisiblePrefix = visibleMount;
|
||||
}
|
||||
}
|
||||
|
||||
const visible = containingParent
|
||||
? visibleIds.has(containingParent.id) && longestVisiblePrefix?.id === containingParent.id
|
||||
: longestVisiblePrefix === undefined;
|
||||
if (visible) {
|
||||
if (visibleMounts.some((candidate) => candidate.mountPoint === mount.mountPoint)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
visibleMounts.push(mount);
|
||||
visibleIds.add(mount.id);
|
||||
}
|
||||
}
|
||||
|
||||
return visibleMounts;
|
||||
}
|
||||
|
||||
function isCgroupMount(mount: MountInfoRecord): mount is CgroupMount {
|
||||
return mount.filesystem === "cgroup" || mount.filesystem === "cgroup2";
|
||||
}
|
||||
|
||||
function isPathPrefix(parent: string, child: string): boolean {
|
||||
return parent === child || isStrictPathPrefix(parent, child);
|
||||
}
|
||||
|
||||
function normalizedAbsolutePath(value: string): string | null {
|
||||
if (!value.startsWith("/") || hasParentPathSegment(value)) return null;
|
||||
return posix.normalize(value);
|
||||
}
|
||||
|
||||
function hasConsistentCgroupPath(
|
||||
selected: CgroupMount,
|
||||
path: string,
|
||||
visibleMounts: MountInfoRecord[],
|
||||
): boolean {
|
||||
let owner: MountInfoRecord | undefined;
|
||||
for (const mount of visibleMounts) {
|
||||
if (
|
||||
isPathPrefix(mount.mountPoint, path) &&
|
||||
(!owner || mount.mountPoint.length > owner.mountPoint.length)
|
||||
) {
|
||||
owner = mount;
|
||||
}
|
||||
}
|
||||
if (!owner) return false;
|
||||
if (owner.id === selected.id) return true;
|
||||
if (
|
||||
owner.filesystem !== selected.filesystem ||
|
||||
owner.device !== selected.device ||
|
||||
!isStrictPathPrefix(selected.mountPoint, owner.mountPoint)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const selectedRoot = normalizedAbsolutePath(selected.root);
|
||||
const ownerRoot = normalizedAbsolutePath(owner.root);
|
||||
if (!selectedRoot || !ownerRoot) return false;
|
||||
const relativeMountPoint = posix.relative(selected.mountPoint, owner.mountPoint);
|
||||
return ownerRoot === posix.normalize(posix.join(selectedRoot, relativeMountPoint));
|
||||
}
|
||||
|
||||
function validLinuxMembership(fields: RegExpExecArray | null): fields is RegExpExecArray {
|
||||
if (fields === null || !fields[3]?.startsWith("/") || hasParentPathSegment(fields[3])) {
|
||||
return false;
|
||||
}
|
||||
const hierarchy = fields[1];
|
||||
const controllers = fields[2];
|
||||
return hierarchy === "0"
|
||||
? controllers === ""
|
||||
: controllers.length > 0 && /^[1-9][0-9]*$/.test(hierarchy);
|
||||
}
|
||||
|
||||
function cgroupProcessPath(mount: CgroupMount, membership: string): string | null {
|
||||
if (
|
||||
hasParentPathSegment(mount.root) ||
|
||||
hasParentPathSegment(mount.mountPoint) ||
|
||||
hasParentPathSegment(membership)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const root = posix.normalize(mount.root);
|
||||
const member = posix.normalize(membership);
|
||||
if (!root.startsWith("/") || !member.startsWith("/") || !mount.mountPoint.startsWith("/")) {
|
||||
return null;
|
||||
}
|
||||
let suffix: string;
|
||||
if (root === "/") suffix = member.slice(1);
|
||||
else if (member === root) suffix = "";
|
||||
else if (member.startsWith(`${root}/`)) suffix = member.slice(root.length + 1);
|
||||
else return null;
|
||||
const candidate = posix.normalize(posix.join(mount.mountPoint, suffix));
|
||||
return candidate === mount.mountPoint || candidate.startsWith(`${mount.mountPoint}/`)
|
||||
? candidate
|
||||
: null;
|
||||
}
|
||||
|
||||
function resolveMembershipMemoryLimits(
|
||||
readTextFile: (path: string) => string,
|
||||
failClosed: boolean,
|
||||
membershipRaw: string,
|
||||
): bigint[] | null {
|
||||
const membershipLines = membershipRaw.split("\n").filter(Boolean);
|
||||
if (failClosed && membershipLines.length === 0) {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
const parsedMemberships = membershipLines.map((line) => /^([^:]*):([^:]*):(.*)$/.exec(line));
|
||||
if (parsedMemberships.some((fields) => fields !== null && hasParentPathSegment(fields[3]))) {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
if (failClosed && parsedMemberships.some((fields) => !validLinuxMembership(fields))) {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
const allMemberships = parsedMemberships
|
||||
.filter(
|
||||
(fields): fields is RegExpExecArray => fields !== null && fields[3]?.startsWith("/") === true,
|
||||
)
|
||||
.map(([, hierarchy, controllers, membershipPath]) => ({
|
||||
kind: hierarchy === "0" && controllers === "" ? "cgroup2" : "cgroup",
|
||||
controllers: new Set(controllers.split(",").filter(Boolean)),
|
||||
path: membershipPath,
|
||||
}));
|
||||
const v1MemoryMemberships = allMemberships.filter(
|
||||
(membership) => membership.kind === "cgroup" && membership.controllers.has("memory"),
|
||||
);
|
||||
const memberships =
|
||||
v1MemoryMemberships.length > 0
|
||||
? v1MemoryMemberships
|
||||
: allMemberships.filter((membership) => membership.kind === "cgroup2");
|
||||
if (memberships.length === 0) return null;
|
||||
|
||||
let mountInfoRaw: string;
|
||||
try {
|
||||
mountInfoRaw = readTextFile("/proc/self/mountinfo");
|
||||
} catch {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
|
||||
const visibleMounts = parseVisibleMounts(mountInfoRaw);
|
||||
const mounts = visibleMounts.filter(isCgroupMount);
|
||||
const selectedMemberships = memberships.flatMap((membership) => {
|
||||
const selected: Array<{ mount: CgroupMount; processPath: string; limitFile: string }> = [];
|
||||
for (const mount of mounts) {
|
||||
if (
|
||||
mount.filesystem !== membership.kind ||
|
||||
(mount.filesystem === "cgroup" && !mount.controllers.has("memory"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const processPath = cgroupProcessPath(mount, membership.path);
|
||||
if (!processPath) continue;
|
||||
selected.push({
|
||||
mount,
|
||||
processPath,
|
||||
limitFile: mount.filesystem === "cgroup2" ? "memory.max" : "memory.limit_in_bytes",
|
||||
});
|
||||
}
|
||||
if (selected.length === 0) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
return selected;
|
||||
});
|
||||
|
||||
const limits: bigint[] = [];
|
||||
for (const selected of selectedMemberships) {
|
||||
if (!hasConsistentCgroupPath(selected.mount, selected.processPath, visibleMounts)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
let current = selected.processPath;
|
||||
while (true) {
|
||||
let raw: string | undefined;
|
||||
const limitPath = posix.join(current, selected.limitFile);
|
||||
if (!hasConsistentCgroupPath(selected.mount, limitPath, visibleMounts)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
try {
|
||||
raw = readTextFile(limitPath);
|
||||
} catch (error) {
|
||||
const absentV2Limit =
|
||||
selected.mount.filesystem === "cgroup2" && hasErrorCode(error, "ENOENT");
|
||||
if (absentV2Limit) {
|
||||
const controllersPath = posix.join(current, "cgroup.controllers");
|
||||
if (!hasConsistentCgroupPath(selected.mount, controllersPath, visibleMounts)) {
|
||||
throw new Error("unable to resolve the process cgroup memory capacity");
|
||||
}
|
||||
try {
|
||||
readTextFile(controllersPath);
|
||||
} catch {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
} else {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
}
|
||||
if (raw !== undefined) {
|
||||
const normalized = raw.trim();
|
||||
if (normalized !== "max" && !/^[0-9]+$/.test(normalized)) {
|
||||
throw new Error("malformed cgroup memory capacity");
|
||||
}
|
||||
const limit = parseCgroupLimit(raw, true);
|
||||
if (limit !== null) limits.push(limit);
|
||||
}
|
||||
if (current === selected.mount.mountPoint) break;
|
||||
const parent = posix.dirname(current);
|
||||
if (parent === current || !parent.startsWith(selected.mount.mountPoint)) break;
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
return limits;
|
||||
}
|
||||
|
||||
function membershipMemoryLimits(
|
||||
readTextFile: (path: string) => string,
|
||||
failClosed: boolean,
|
||||
): bigint[] | null {
|
||||
for (let attempt = 0; attempt < CGROUP_MEMBERSHIP_RESOLUTION_ATTEMPTS; attempt += 1) {
|
||||
let membershipRaw: string;
|
||||
try {
|
||||
membershipRaw = readTextFile("/proc/self/cgroup");
|
||||
} catch {
|
||||
if (failClosed || attempt > 0) {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
let limits: bigint[] | null;
|
||||
try {
|
||||
limits = resolveMembershipMemoryLimits(readTextFile, failClosed, membershipRaw);
|
||||
} catch (error) {
|
||||
let failedMembershipRaw: string;
|
||||
try {
|
||||
failedMembershipRaw = readTextFile("/proc/self/cgroup");
|
||||
} catch {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
if (failedMembershipRaw === membershipRaw) throw error;
|
||||
continue;
|
||||
}
|
||||
let confirmedMembershipRaw: string;
|
||||
try {
|
||||
confirmedMembershipRaw = readTextFile("/proc/self/cgroup");
|
||||
} catch {
|
||||
throw new Error("unable to read the process cgroup memory capacity");
|
||||
}
|
||||
if (confirmedMembershipRaw === membershipRaw) return limits;
|
||||
}
|
||||
throw new Error("unable to read a stable process cgroup memory capacity");
|
||||
}
|
||||
|
||||
/** Configured capacity available to this process, including container limits. */
|
||||
export function getOcrRuntimeEffectiveMemoryBytes(options: OcrRuntimeMemoryOptions = {}): number {
|
||||
if (options.effectiveMemoryBytes !== undefined) {
|
||||
return positiveSafeBytes(options.effectiveMemoryBytes, "effective OCR runtime memory");
|
||||
}
|
||||
|
||||
const physical = positiveSafeBytes(
|
||||
options.physicalMemoryBytes ?? totalmem(),
|
||||
"physical OCR runtime memory",
|
||||
);
|
||||
let effective = BigInt(physical);
|
||||
const readTextFile = options.readTextFile ?? ((path: string) => readFileSync(path, "utf8"));
|
||||
const hostPlatform = options.hostPlatform ?? process.platform;
|
||||
const isLinux = hostPlatform === "linux";
|
||||
const membershipLimits = membershipMemoryLimits(readTextFile, isLinux);
|
||||
if (membershipLimits === null) {
|
||||
for (const path of CGROUP_MEMORY_LIMIT_PATHS) {
|
||||
try {
|
||||
const limit = parseCgroupLimit(readTextFile(path), isLinux);
|
||||
if (limit !== null && limit < effective) effective = limit;
|
||||
} catch {
|
||||
// A host normally exposes either cgroup v2, one v1 layout, or neither.
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const limit of membershipLimits) if (limit < effective) effective = limit;
|
||||
}
|
||||
const constrained =
|
||||
options.physicalMemoryBytes === undefined && typeof process.constrainedMemory === "function"
|
||||
? process.constrainedMemory()
|
||||
: undefined;
|
||||
if (constrained && Number.isSafeInteger(constrained) && constrained > 0) {
|
||||
effective = effective < BigInt(constrained) ? effective : BigInt(constrained);
|
||||
}
|
||||
return Number(effective);
|
||||
}
|
||||
|
||||
export function hasOcrRuntimeMemory(
|
||||
minimumMemoryBytes: number,
|
||||
options: OcrRuntimeMemoryOptions = {},
|
||||
): boolean {
|
||||
positiveSafeBytes(minimumMemoryBytes, "OCR runtime minimum memory");
|
||||
return getOcrRuntimeEffectiveMemoryBytes(options) >= minimumMemoryBytes;
|
||||
}
|
||||
|
||||
export function assertOcrRuntimeMemory(
|
||||
minimumMemoryBytes: number,
|
||||
options: OcrRuntimeMemoryOptions = {},
|
||||
): void {
|
||||
positiveSafeBytes(minimumMemoryBytes, "OCR runtime minimum memory");
|
||||
const effectiveMemoryBytes = getOcrRuntimeEffectiveMemoryBytes(options);
|
||||
if (effectiveMemoryBytes < minimumMemoryBytes) {
|
||||
throw new Error(
|
||||
`insufficient memory for accurate OCR runtime: ${minimumMemoryBytes} bytes required, ${effectiveMemoryBytes} available; Fast OCR remains available`,
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,214 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
const MAX_INVENTORY_OUTPUT_BYTES = 64 * 1024;
|
||||
const FORCE_KILL_DELAY_MS = 1_000;
|
||||
|
||||
export const SUPPORTED_TESSERACT_TRAINEDDATA = [
|
||||
"eng",
|
||||
"deu",
|
||||
"fra",
|
||||
"spa",
|
||||
"chi_sim",
|
||||
"jpn",
|
||||
] as const;
|
||||
|
||||
export interface TesseractLanguageInventoryOptions {
|
||||
executable: string;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const inventoryCache = new Map<string, ReadonlySet<string>>();
|
||||
|
||||
function cacheKey(executable: string): string {
|
||||
return `${executable}\0${process.env.TESSDATA_PREFIX ?? ""}`;
|
||||
}
|
||||
|
||||
/** Clear the process-local inventory after an installation or in isolated tests. */
|
||||
export function clearTesseractLanguageInventoryCache(): void {
|
||||
inventoryCache.clear();
|
||||
}
|
||||
|
||||
/** Return a defensive copy when this executable was already preflighted. */
|
||||
export function getCachedTesseractLanguages(executable: string): ReadonlySet<string> | undefined {
|
||||
const cached = inventoryCache.get(cacheKey(executable));
|
||||
return cached ? new Set(cached) : undefined;
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("Tesseract language-pack preflight was canceled");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function parseLanguageInventory(stdout: string): ReadonlySet<string> {
|
||||
const lines = stdout.replaceAll("\r\n", "\n").split("\n");
|
||||
while (lines.at(-1) === "") lines.pop();
|
||||
const header = lines.shift();
|
||||
const match = header?.match(/^List of available languages in .+ \((\d+)\):$/u);
|
||||
if (!match) {
|
||||
throw new Error(
|
||||
"Tesseract --list-langs returned malformed output; cannot verify installed traineddata.",
|
||||
);
|
||||
}
|
||||
const declaredCount = Number(match[1]);
|
||||
if (
|
||||
!Number.isSafeInteger(declaredCount) ||
|
||||
lines.length !== declaredCount ||
|
||||
lines.some(
|
||||
(language) =>
|
||||
!/^[A-Za-z0-9][A-Za-z0-9_./-]*$/u.test(language) ||
|
||||
language.includes("..") ||
|
||||
language.endsWith("/"),
|
||||
) ||
|
||||
new Set(lines).size !== lines.length
|
||||
) {
|
||||
if (Number.isSafeInteger(declaredCount) && lines.length !== declaredCount) {
|
||||
throw new Error(
|
||||
`Tesseract --list-langs declared ${declaredCount} languages but returned ${lines.length}.`,
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
"Tesseract --list-langs returned malformed output; cannot verify installed traineddata.",
|
||||
);
|
||||
}
|
||||
return new Set(lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the selected executable which traineddata it can actually load. Successful
|
||||
* inventories are cached per executable and TESSDATA_PREFIX for the process.
|
||||
*/
|
||||
export async function getInstalledTesseractLanguages(
|
||||
options: TesseractLanguageInventoryOptions,
|
||||
): Promise<ReadonlySet<string>> {
|
||||
if (!options.executable) throw new Error("Tesseract executable path is empty");
|
||||
if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) {
|
||||
throw new Error("Tesseract language-pack preflight timeout must be positive");
|
||||
}
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
|
||||
const key = cacheKey(options.executable);
|
||||
const cached = inventoryCache.get(key);
|
||||
if (cached) return new Set(cached);
|
||||
|
||||
const installed = await new Promise<ReadonlySet<string>>((resolve, reject) => {
|
||||
const child = spawn(options.executable, ["--list-langs"], {
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
let stdoutBytes = 0;
|
||||
let stderrBytes = 0;
|
||||
let settled = false;
|
||||
let terminationError: Error | undefined;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
terminate(
|
||||
new Error(
|
||||
`Tesseract language-pack preflight timed out after ${Math.floor(options.timeoutMs)}ms`,
|
||||
),
|
||||
);
|
||||
}, options.timeoutMs);
|
||||
timeoutTimer.unref();
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutTimer);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
const finish = (error?: Error, result?: ReadonlySet<string>) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
if (error) reject(error);
|
||||
else resolve(result as ReadonlySet<string>);
|
||||
};
|
||||
|
||||
function terminate(error: Error) {
|
||||
if (settled || terminationError) return;
|
||||
terminationError = error;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// The close/error event retains process ownership and settles the call.
|
||||
}
|
||||
forceKillTimer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// Wait for close before releasing the request.
|
||||
}
|
||||
}
|
||||
}, FORCE_KILL_DELAY_MS);
|
||||
forceKillTimer.unref();
|
||||
}
|
||||
|
||||
const onAbort = () => terminate(abortError());
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
if (options.signal?.aborted) terminate(abortError());
|
||||
|
||||
child.stdout.on("data", (chunk: Buffer | string) => {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
stdoutBytes += buffer.length;
|
||||
if (stdoutBytes > MAX_INVENTORY_OUTPUT_BYTES) {
|
||||
terminate(new Error("Tesseract --list-langs stdout exceeded 65536 bytes"));
|
||||
return;
|
||||
}
|
||||
stdoutChunks.push(buffer);
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
stderrBytes += buffer.length;
|
||||
if (stderrBytes > MAX_INVENTORY_OUTPUT_BYTES) {
|
||||
terminate(new Error("Tesseract --list-langs stderr exceeded 65536 bytes"));
|
||||
return;
|
||||
}
|
||||
stderrChunks.push(buffer);
|
||||
});
|
||||
|
||||
child.once("error", (error: NodeJS.ErrnoException) => {
|
||||
if (terminationError) return;
|
||||
if (error.code === "ENOENT") {
|
||||
finish(
|
||||
new Error(
|
||||
"Tesseract executable not found while checking installed language packs. Install Tesseract or set TESSERACT_PATH.",
|
||||
{ cause: error },
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
finish(new Error(`Unable to run Tesseract --list-langs: ${error.message}`, { cause: error }));
|
||||
});
|
||||
|
||||
child.once("close", (code, signal) => {
|
||||
if (terminationError) {
|
||||
finish(terminationError);
|
||||
return;
|
||||
}
|
||||
if (code !== 0) {
|
||||
const detail = Buffer.concat(stderrChunks).toString("utf8").trim();
|
||||
const status = code === null ? `signal ${signal ?? "unknown"}` : `code ${code}`;
|
||||
finish(
|
||||
new Error(
|
||||
`Unable to inspect Tesseract language packs: --list-langs exited with ${status}${detail ? `: ${detail}` : ""}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
finish(undefined, parseLanguageInventory(Buffer.concat(stdoutChunks).toString("utf8")));
|
||||
} catch (error) {
|
||||
finish(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
inventoryCache.set(key, installed);
|
||||
return new Set(installed);
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, mkdtemp, realpath, rm, stat, statfs } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import sharp from "sharp";
|
||||
import {
|
||||
type RunTesseractOptions,
|
||||
runAdaptiveTesseract,
|
||||
type TesseractLanguage,
|
||||
type TesseractRuntimeMetadata,
|
||||
} from "./tesseract.js";
|
||||
|
||||
export const MAX_PDF_OCR_PAGES = 50;
|
||||
/** Shared Fast/accurate UTF-8 text ceiling, including PDF page headings. */
|
||||
export const MAX_PDF_OCR_OUTPUT_BYTES = 1_000_000;
|
||||
|
||||
const DEFAULT_DPI = 300;
|
||||
const MIN_DPI = 72;
|
||||
const MAX_DPI = 600;
|
||||
const DEFAULT_TIMEOUT_MS = 30 * 60_000;
|
||||
const FORCE_KILL_DELAY_MS = 1_000;
|
||||
const MAX_RASTER_DIMENSION = 6_000;
|
||||
const MAX_RASTER_PIXELS = 25_000_000;
|
||||
const MAX_PREPARED_RASTER_BYTES = 512n * 1024n * 1024n;
|
||||
const MIN_SCRATCH_FREE_BYTES = 256n * 1024n * 1024n;
|
||||
const MAX_DIAGNOSTIC_OUTPUT = 16_384;
|
||||
|
||||
const PAGE_COUNT_PROGRAM = "PDFname (r) file runpdfbegin pdfpagecount = quit";
|
||||
const PAGE_BOX_PROGRAM =
|
||||
"PDFname (r) file runpdfbegin /page PageNumber pdfgetpage def /box page /CropBox known { page /CropBox get } { page /MediaBox get } ifelse def box == page /UserUnit known { page /UserUnit get } { 1 } ifelse == quit";
|
||||
|
||||
export interface RunTesseractPdfOptions {
|
||||
pages?: string;
|
||||
language?: TesseractLanguage;
|
||||
/** Apply conservative local-contrast preprocessing before Tesseract. */
|
||||
enhance?: boolean;
|
||||
/** Requested raster resolution. Oversized pages are automatically rendered lower. */
|
||||
dpi?: number;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: number, stage: string) => void;
|
||||
/** Override for deployments where Ghostscript is not on PATH. */
|
||||
ghostscriptPath?: string;
|
||||
/** Override for deployments where Tesseract is not on PATH. */
|
||||
tesseractPath?: string;
|
||||
}
|
||||
|
||||
export interface TesseractPdfResult extends TesseractRuntimeMetadata {
|
||||
text: string;
|
||||
pages: number;
|
||||
pageNumbers: number[];
|
||||
}
|
||||
|
||||
export interface PreparedPdfOcrPage {
|
||||
page: number;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface PreparedPdfOcrPages {
|
||||
pages: PreparedPdfOcrPage[];
|
||||
totalPages: number;
|
||||
remainingTimeoutMs: () => number;
|
||||
cleanup: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface ProcessResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
interface PageBox {
|
||||
widthPoints: number;
|
||||
heightPoints: number;
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("PDF OCR was canceled");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function validatePositiveInteger(value: number, label: string, maximum?: number): number {
|
||||
if (!Number.isInteger(value) || value <= 0 || (maximum !== undefined && value > maximum)) {
|
||||
const range = maximum === undefined ? "a positive integer" : `an integer from 1 to ${maximum}`;
|
||||
throw new Error(`${label} must be ${range}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Parse a strict 1-based page list such as `all`, `1-3,5`, or `2,4-6`. */
|
||||
export function parsePdfPageSpec(spec: string, totalPages: number): number[] {
|
||||
validatePositiveInteger(totalPages, "PDF page count");
|
||||
const normalized = spec.trim();
|
||||
if (!normalized) throw new Error("No pages specified");
|
||||
|
||||
if (normalized.toLowerCase() === "all") {
|
||||
if (totalPages > MAX_PDF_OCR_PAGES) {
|
||||
throw new Error(`Too many pages for OCR (max ${MAX_PDF_OCR_PAGES})`);
|
||||
}
|
||||
return Array.from({ length: totalPages }, (_, index) => index + 1);
|
||||
}
|
||||
|
||||
const selected = new Set<number>();
|
||||
for (const rawPart of normalized.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
if (!part) throw new Error(`Invalid page selection: "${spec}"`);
|
||||
|
||||
const match = /^(\d+)(?:\s*-\s*(\d+))?$/.exec(part);
|
||||
if (!match) throw new Error(`Invalid page selection: "${part}"`);
|
||||
|
||||
const start = Number(match[1]);
|
||||
const end = match[2] === undefined ? start : Number(match[2]);
|
||||
if (start < 1 || end < 1) {
|
||||
throw new Error(`Invalid page selection: "${part}" (pages start at 1)`);
|
||||
}
|
||||
if (start > end) {
|
||||
throw new Error(`Invalid page selection: "${part}" (range start is after range end)`);
|
||||
}
|
||||
if (end > totalPages) {
|
||||
throw new Error(`Invalid page selection: "${part}" (document has ${totalPages} pages)`);
|
||||
}
|
||||
|
||||
for (let page = start; page <= end; page += 1) {
|
||||
selected.add(page);
|
||||
if (selected.size > MAX_PDF_OCR_PAGES) {
|
||||
throw new Error(`Too many pages for OCR (max ${MAX_PDF_OCR_PAGES})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (selected.size === 0) throw new Error("No pages specified");
|
||||
return [...selected].sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function appendDiagnostic(current: string, chunk: Buffer | string): string {
|
||||
const next = current + chunk.toString();
|
||||
return next.length <= MAX_DIAGNOSTIC_OUTPUT ? next : next.slice(-MAX_DIAGNOSTIC_OUTPUT);
|
||||
}
|
||||
|
||||
function runGhostscript(
|
||||
executable: string,
|
||||
args: string[],
|
||||
timeoutMs: number,
|
||||
totalTimeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ProcessResult> {
|
||||
if (signal?.aborted) return Promise.reject(abortError());
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(executable, args, {
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
let termination: "abort" | "timeout" | undefined;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
const timeoutTimer = setTimeout(() => terminate("timeout"), timeoutMs);
|
||||
timeoutTimer.unref();
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutTimer);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
const finish = (error?: Error, result?: ProcessResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
if (error) reject(error);
|
||||
else resolve(result as ProcessResult);
|
||||
};
|
||||
|
||||
const finishTermination = () => {
|
||||
if (termination === "abort") finish(abortError());
|
||||
else if (termination === "timeout") {
|
||||
finish(new Error(`PDF OCR timed out after ${totalTimeoutMs}ms`));
|
||||
}
|
||||
};
|
||||
|
||||
function terminate(reason: "abort" | "timeout") {
|
||||
if (settled || termination) return;
|
||||
termination = reason;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// A concurrent process exit owns settlement through close/error.
|
||||
}
|
||||
forceKillTimer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// Wait for close before deleting the raster scratch directory.
|
||||
}
|
||||
}
|
||||
}, FORCE_KILL_DELAY_MS);
|
||||
forceKillTimer.unref();
|
||||
}
|
||||
|
||||
const onAbort = () => terminate("abort");
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
if (signal?.aborted) terminate("abort");
|
||||
|
||||
child.stdout.on("data", (chunk: Buffer | string) => {
|
||||
stdout = appendDiagnostic(stdout, chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
stderr = appendDiagnostic(stderr, chunk);
|
||||
});
|
||||
|
||||
child.once("error", (error: NodeJS.ErrnoException) => {
|
||||
if (termination) return;
|
||||
if (error.code === "ENOENT") {
|
||||
finish(
|
||||
new Error("Ghostscript executable not found. Install Ghostscript or set GS_PATH.", {
|
||||
cause: error,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
finish(new Error(`Unable to start Ghostscript: ${error.message}`, { cause: error }));
|
||||
});
|
||||
|
||||
child.once("close", (code, closeSignal) => {
|
||||
if (termination) {
|
||||
finishTermination();
|
||||
return;
|
||||
}
|
||||
if (code !== 0) {
|
||||
const detail = stderr.trim();
|
||||
const status = code === null ? `signal ${closeSignal ?? "unknown"}` : `code ${code}`;
|
||||
finish(new Error(`Ghostscript exited with ${status}${detail ? `: ${detail}` : ""}`));
|
||||
return;
|
||||
}
|
||||
finish(undefined, { stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function parsePageCount(stdout: string): number {
|
||||
const count = Number(stdout.trim());
|
||||
if (!Number.isSafeInteger(count) || count < 1) {
|
||||
throw new Error("Ghostscript returned an invalid or empty PDF page count");
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function parsePageBox(stdout: string, pageNumber: number): PageBox {
|
||||
const arrayMatch = stdout.match(/\[([^\]]+)\]\s*$/m);
|
||||
const values = arrayMatch?.[1]
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.map((value) => Number(value));
|
||||
if (values?.length !== 4 || values.some((value) => !Number.isFinite(value))) {
|
||||
throw new Error(`Ghostscript returned invalid dimensions for PDF page ${pageNumber}`);
|
||||
}
|
||||
|
||||
const userUnitMatch = stdout.match(/\]\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)\s*$/);
|
||||
const userUnit = userUnitMatch ? Number(userUnitMatch[1]) : 1;
|
||||
if (!Number.isFinite(userUnit) || userUnit <= 0) {
|
||||
throw new Error(`PDF page ${pageNumber} has an invalid UserUnit`);
|
||||
}
|
||||
|
||||
const widthPoints = Math.abs(values[2] - values[0]) * userUnit;
|
||||
const heightPoints = Math.abs(values[3] - values[1]) * userUnit;
|
||||
if (widthPoints <= 0 || heightPoints <= 0) {
|
||||
throw new Error(`PDF page ${pageNumber} has invalid dimensions`);
|
||||
}
|
||||
return { widthPoints, heightPoints };
|
||||
}
|
||||
|
||||
async function validateRasterPage(path: string, pageNumber: number): Promise<string> {
|
||||
const canonicalPath = await realpath(path);
|
||||
let metadata: Awaited<ReturnType<ReturnType<typeof sharp>["metadata"]>>;
|
||||
try {
|
||||
metadata = await sharp(canonicalPath, { limitInputPixels: MAX_RASTER_PIXELS }).metadata();
|
||||
} catch (error) {
|
||||
throw new Error(`PDF page ${pageNumber} produced an unsafe or invalid OCR raster`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
const width = metadata.width ?? 0;
|
||||
const height = metadata.height ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(width) ||
|
||||
!Number.isSafeInteger(height) ||
|
||||
width <= 0 ||
|
||||
height <= 0 ||
|
||||
width > MAX_RASTER_DIMENSION ||
|
||||
height > MAX_RASTER_DIMENSION ||
|
||||
width * height > MAX_RASTER_PIXELS
|
||||
) {
|
||||
throw new Error(`PDF page ${pageNumber} produced unsafe raster dimensions`);
|
||||
}
|
||||
return canonicalPath;
|
||||
}
|
||||
|
||||
async function retainRasterWithinScratchBudget(
|
||||
path: string,
|
||||
scratchDir: string,
|
||||
retainedBytes: bigint,
|
||||
): Promise<bigint> {
|
||||
const rasterInfo = await stat(path, { bigint: true });
|
||||
if (!rasterInfo.isFile()) throw new Error("PDF OCR produced a non-regular raster file");
|
||||
|
||||
const nextRetainedBytes = retainedBytes + rasterInfo.size;
|
||||
if (nextRetainedBytes > MAX_PREPARED_RASTER_BYTES) {
|
||||
throw new Error("PDF OCR rasters exceed the 512 MiB aggregate scratch limit");
|
||||
}
|
||||
|
||||
const scratchInfo = await statfs(scratchDir, { bigint: true });
|
||||
const availableBytes = scratchInfo.bavail * scratchInfo.bsize;
|
||||
if (availableBytes < MIN_SCRATCH_FREE_BYTES) {
|
||||
throw new Error("PDF OCR cannot preserve the 256 MiB free scratch space reserve");
|
||||
}
|
||||
return nextRetainedBytes;
|
||||
}
|
||||
|
||||
function safeDpi(pageBox: PageBox, requestedDpi: number, pageNumber: number): number {
|
||||
const requestedWidth = (pageBox.widthPoints / 72) * requestedDpi;
|
||||
const requestedHeight = (pageBox.heightPoints / 72) * requestedDpi;
|
||||
const dimensionScale = Math.min(
|
||||
1,
|
||||
MAX_RASTER_DIMENSION / requestedWidth,
|
||||
MAX_RASTER_DIMENSION / requestedHeight,
|
||||
);
|
||||
const pixelScale = Math.min(1, Math.sqrt(MAX_RASTER_PIXELS / (requestedWidth * requestedHeight)));
|
||||
const dpi = Math.max(1, Math.floor(requestedDpi * Math.min(dimensionScale, pixelScale)));
|
||||
if (dpi < MIN_DPI) {
|
||||
throw new Error(
|
||||
`PDF page ${pageNumber} is too large to rasterize at the ${MIN_DPI} DPI quality floor`,
|
||||
);
|
||||
}
|
||||
const width = Math.ceil((pageBox.widthPoints / 72) * dpi);
|
||||
const height = Math.ceil((pageBox.heightPoints / 72) * dpi);
|
||||
if (
|
||||
width > MAX_RASTER_DIMENSION ||
|
||||
height > MAX_RASTER_DIMENSION ||
|
||||
width * height > MAX_RASTER_PIXELS
|
||||
) {
|
||||
throw new Error(`PDF page ${pageNumber} is too large to rasterize safely`);
|
||||
}
|
||||
return dpi;
|
||||
}
|
||||
|
||||
function remainingTimeout(deadline: number, totalTimeoutMs: number): number {
|
||||
const remaining = deadline - performance.now();
|
||||
if (remaining <= 0) throw new Error(`PDF OCR timed out after ${totalTimeoutMs}ms`);
|
||||
return remaining;
|
||||
}
|
||||
|
||||
function ghostscriptBaseArgs(inputPath: string): string[] {
|
||||
return [
|
||||
"-q",
|
||||
"-dNODISPLAY",
|
||||
"-dBATCH",
|
||||
"-dSAFER",
|
||||
`--permit-file-read=${inputPath}`,
|
||||
`-sPDFname=${inputPath}`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Rasterize validated, selected PDF pages for an OCR engine. The caller owns
|
||||
* the returned lease-like object and must invoke cleanup in a finally block.
|
||||
*/
|
||||
export async function preparePdfOcrPages(
|
||||
inputPath: string,
|
||||
scratchDir: string,
|
||||
options: Pick<
|
||||
RunTesseractPdfOptions,
|
||||
"pages" | "dpi" | "timeoutMs" | "signal" | "onProgress" | "ghostscriptPath"
|
||||
> = {},
|
||||
): Promise<PreparedPdfOcrPages> {
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
|
||||
const requestedDpi = options.dpi ?? DEFAULT_DPI;
|
||||
if (!Number.isInteger(requestedDpi) || requestedDpi < MIN_DPI || requestedDpi > MAX_DPI) {
|
||||
throw new Error(`PDF OCR DPI must be an integer from ${MIN_DPI} to ${MAX_DPI}`);
|
||||
}
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("PDF OCR timeout must be a positive number");
|
||||
}
|
||||
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
const executable = options.ghostscriptPath ?? process.env.GS_PATH ?? "gs";
|
||||
const resolvedInputPath = await realpath(inputPath);
|
||||
await mkdir(scratchDir, { recursive: true });
|
||||
const jobScratchDir = await mkdtemp(join(scratchDir, "ocr-pdf-pages-"));
|
||||
const cleanup = () => rm(jobScratchDir, { recursive: true, force: true }).catch(() => {});
|
||||
|
||||
try {
|
||||
options.onProgress?.(0, "Opening PDF");
|
||||
const countResult = await runGhostscript(
|
||||
executable,
|
||||
[...ghostscriptBaseArgs(resolvedInputPath), "-c", PAGE_COUNT_PROGRAM],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
const totalPages = parsePageCount(countResult.stdout);
|
||||
const pageNumbers = parsePdfPageSpec(options.pages ?? "all", totalPages);
|
||||
const pages: PreparedPdfOcrPage[] = [];
|
||||
let retainedRasterBytes = 0n;
|
||||
|
||||
for (const [index, pageNumber] of pageNumbers.entries()) {
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
options.onProgress?.(
|
||||
5 + Math.floor((index / pageNumbers.length) * 40),
|
||||
`Rasterizing PDF page ${pageNumber}`,
|
||||
);
|
||||
const boxResult = await runGhostscript(
|
||||
executable,
|
||||
[
|
||||
...ghostscriptBaseArgs(resolvedInputPath),
|
||||
`-dPageNumber=${pageNumber}`,
|
||||
"-c",
|
||||
PAGE_BOX_PROGRAM,
|
||||
],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
const dpi = safeDpi(parsePageBox(boxResult.stdout, pageNumber), requestedDpi, pageNumber);
|
||||
const pagePath = join(jobScratchDir, `page-${pageNumber}.png`);
|
||||
await runGhostscript(
|
||||
executable,
|
||||
[
|
||||
"-q",
|
||||
"-dBATCH",
|
||||
"-dNOPAUSE",
|
||||
"-dSAFER",
|
||||
"-dUseCropBox",
|
||||
`-dFirstPage=${pageNumber}`,
|
||||
`-dLastPage=${pageNumber}`,
|
||||
"-sDEVICE=pnggray",
|
||||
"-dTextAlphaBits=4",
|
||||
"-dGraphicsAlphaBits=4",
|
||||
`-r${dpi}`,
|
||||
`-sOutputFile=${pagePath}`,
|
||||
resolvedInputPath,
|
||||
],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
const rasterPath = await validateRasterPage(pagePath, pageNumber);
|
||||
retainedRasterBytes = await retainRasterWithinScratchBudget(
|
||||
rasterPath,
|
||||
jobScratchDir,
|
||||
retainedRasterBytes,
|
||||
);
|
||||
pages.push({ page: pageNumber, path: rasterPath });
|
||||
}
|
||||
|
||||
return {
|
||||
pages,
|
||||
totalPages,
|
||||
remainingTimeoutMs: () => remainingTimeout(deadline, timeoutMs),
|
||||
cleanup,
|
||||
};
|
||||
} catch (error) {
|
||||
await cleanup();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Rasterize selected PDF pages with Ghostscript and OCR them with built-in Tesseract. */
|
||||
export async function runTesseractPdf(
|
||||
inputPath: string,
|
||||
scratchDir: string,
|
||||
options: RunTesseractPdfOptions = {},
|
||||
): Promise<TesseractPdfResult> {
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
|
||||
const requestedDpi = options.dpi ?? DEFAULT_DPI;
|
||||
if (!Number.isInteger(requestedDpi) || requestedDpi < MIN_DPI || requestedDpi > MAX_DPI) {
|
||||
throw new Error(`PDF OCR DPI must be an integer from ${MIN_DPI} to ${MAX_DPI}`);
|
||||
}
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("PDF OCR timeout must be a positive number");
|
||||
}
|
||||
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
const executable = options.ghostscriptPath ?? process.env.GS_PATH ?? "gs";
|
||||
const resolvedInputPath = await realpath(inputPath);
|
||||
await mkdir(scratchDir, { recursive: true });
|
||||
const jobScratchDir = await mkdtemp(join(scratchDir, "ocr-pdf-"));
|
||||
|
||||
try {
|
||||
options.onProgress?.(0, "Opening PDF");
|
||||
const pageCountResult = await runGhostscript(
|
||||
executable,
|
||||
[...ghostscriptBaseArgs(resolvedInputPath), "-c", PAGE_COUNT_PROGRAM],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
const totalPages = parsePageCount(pageCountResult.stdout);
|
||||
const pageNumbers = parsePdfPageSpec(options.pages ?? "all", totalPages);
|
||||
const pageTexts: string[] = [];
|
||||
let retainedOutputBytes = 0;
|
||||
|
||||
for (const [index, pageNumber] of pageNumbers.entries()) {
|
||||
if (options.signal?.aborted) throw abortError();
|
||||
const pageBaseProgress = 5 + Math.floor((index / pageNumbers.length) * 90);
|
||||
options.onProgress?.(pageBaseProgress, `Rasterizing PDF page ${pageNumber}`);
|
||||
|
||||
const pageBoxResult = await runGhostscript(
|
||||
executable,
|
||||
[
|
||||
...ghostscriptBaseArgs(resolvedInputPath),
|
||||
`-dPageNumber=${pageNumber}`,
|
||||
"-c",
|
||||
PAGE_BOX_PROGRAM,
|
||||
],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
const dpi = safeDpi(parsePageBox(pageBoxResult.stdout, pageNumber), requestedDpi, pageNumber);
|
||||
const pagePath = join(jobScratchDir, `page-${pageNumber}.png`);
|
||||
|
||||
await runGhostscript(
|
||||
executable,
|
||||
[
|
||||
"-q",
|
||||
"-dBATCH",
|
||||
"-dNOPAUSE",
|
||||
"-dSAFER",
|
||||
"-dUseCropBox",
|
||||
`-dFirstPage=${pageNumber}`,
|
||||
`-dLastPage=${pageNumber}`,
|
||||
"-sDEVICE=pnggray",
|
||||
"-dTextAlphaBits=4",
|
||||
"-dGraphicsAlphaBits=4",
|
||||
`-r${dpi}`,
|
||||
`-sOutputFile=${pagePath}`,
|
||||
resolvedInputPath,
|
||||
],
|
||||
remainingTimeout(deadline, timeoutMs),
|
||||
timeoutMs,
|
||||
options.signal,
|
||||
);
|
||||
|
||||
const separator = pageTexts.length === 0 ? "" : "\n\n";
|
||||
const pageHeading = `--- Page ${pageNumber} ---\n\n`;
|
||||
const framingBytes = Buffer.byteLength(separator) + Buffer.byteLength(pageHeading);
|
||||
const remainingOutputBytes = MAX_PDF_OCR_OUTPUT_BYTES - retainedOutputBytes - framingBytes;
|
||||
if (remainingOutputBytes <= 0) {
|
||||
throw new Error(
|
||||
`PDF OCR exceeded the ${MAX_PDF_OCR_OUTPUT_BYTES} byte aggregate output limit`,
|
||||
);
|
||||
}
|
||||
|
||||
const tesseractOptions: RunTesseractOptions = {
|
||||
language: options.language ?? "auto",
|
||||
timeoutMs: remainingTimeout(deadline, timeoutMs),
|
||||
signal: options.signal,
|
||||
tesseractPath: options.tesseractPath,
|
||||
maxStdoutBytes: remainingOutputBytes,
|
||||
onProgress: (progress, stage) => {
|
||||
const pageShare = 90 / pageNumbers.length;
|
||||
options.onProgress?.(
|
||||
Math.min(95, Math.floor(pageBaseProgress + (progress / 100) * pageShare)),
|
||||
stage,
|
||||
);
|
||||
},
|
||||
};
|
||||
const rasterPath = await validateRasterPage(pagePath, pageNumber);
|
||||
let ocrPath = rasterPath;
|
||||
if (options.enhance) {
|
||||
options.onProgress?.(pageBaseProgress, `Enhancing PDF page ${pageNumber}`);
|
||||
const metadata = await sharp(rasterPath, {
|
||||
limitInputPixels: MAX_RASTER_PIXELS,
|
||||
}).metadata();
|
||||
const width = metadata.width ?? 0;
|
||||
const height = metadata.height ?? 0;
|
||||
const enhancedPath = join(jobScratchDir, `enhanced-page-${pageNumber}.png`);
|
||||
await sharp(rasterPath, { limitInputPixels: MAX_RASTER_PIXELS })
|
||||
.clahe({
|
||||
width: Math.max(1, Math.min(256, Math.round(width / 8))),
|
||||
height: Math.max(1, Math.min(256, Math.round(height / 8))),
|
||||
maxSlope: 2,
|
||||
})
|
||||
.png()
|
||||
.toFile(enhancedPath);
|
||||
ocrPath = await validateRasterPage(enhancedPath, pageNumber);
|
||||
await rm(rasterPath, { force: true });
|
||||
}
|
||||
const result = await runAdaptiveTesseract(ocrPath, tesseractOptions).finally(() =>
|
||||
rm(ocrPath, { force: true }).catch(() => {}),
|
||||
);
|
||||
const pageText = `${pageHeading}${result.text.trim()}`;
|
||||
const addedBytes = Buffer.byteLength(separator) + Buffer.byteLength(pageText);
|
||||
if (retainedOutputBytes + addedBytes > MAX_PDF_OCR_OUTPUT_BYTES) {
|
||||
throw new Error(
|
||||
`PDF OCR exceeded the ${MAX_PDF_OCR_OUTPUT_BYTES} byte aggregate output limit`,
|
||||
);
|
||||
}
|
||||
pageTexts.push(pageText);
|
||||
retainedOutputBytes += addedBytes;
|
||||
}
|
||||
|
||||
options.onProgress?.(100, "Tesseract PDF OCR complete");
|
||||
return {
|
||||
text: pageTexts.join("\n\n"),
|
||||
pages: pageNumbers.length,
|
||||
pageNumbers,
|
||||
engine: "tesseract",
|
||||
provider: "native",
|
||||
device: "cpu",
|
||||
};
|
||||
} finally {
|
||||
await rm(jobScratchDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,803 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import {
|
||||
getCachedTesseractLanguages,
|
||||
getInstalledTesseractLanguages,
|
||||
} from "./tesseract-languages.js";
|
||||
|
||||
export type TesseractLanguage = "auto" | "en" | "de" | "fr" | "es" | "zh" | "ja";
|
||||
|
||||
export const TESSERACT_LANGUAGE_MAP = {
|
||||
en: "eng",
|
||||
de: "deu",
|
||||
fr: "fra",
|
||||
es: "spa",
|
||||
zh: "chi_sim",
|
||||
ja: "jpn",
|
||||
} as const satisfies Record<Exclude<TesseractLanguage, "auto">, string>;
|
||||
|
||||
const ALL_TESSERACT_LANGUAGES = Object.values(TESSERACT_LANGUAGE_MAP).join("+");
|
||||
|
||||
export function resolveTesseractLanguage(language: TesseractLanguage): string {
|
||||
if (language === "auto") return ALL_TESSERACT_LANGUAGES;
|
||||
|
||||
const mapped = TESSERACT_LANGUAGE_MAP[language];
|
||||
if (!mapped) {
|
||||
throw new Error(`Unsupported OCR language "${language}"`);
|
||||
}
|
||||
return mapped;
|
||||
}
|
||||
|
||||
export interface TesseractRuntimeMetadata {
|
||||
engine: "tesseract";
|
||||
provider: "native";
|
||||
device: "cpu";
|
||||
}
|
||||
|
||||
export interface TesseractResult extends TesseractRuntimeMetadata {
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface RunTesseractOptions {
|
||||
language?: TesseractLanguage;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: number, stage: string) => void;
|
||||
/** Override for deployments where Tesseract is not on PATH. */
|
||||
tesseractPath?: string;
|
||||
/** Maximum bytes retained independently for stdout and stderr. */
|
||||
maxOutputBytes?: number;
|
||||
/** Maximum stdout bytes retained; overrides maxOutputBytes for stdout only. */
|
||||
maxStdoutBytes?: number;
|
||||
/** Maximum stderr bytes retained; overrides maxOutputBytes for stderr only. */
|
||||
maxStderrBytes?: number;
|
||||
/** Internal page segmentation override used by the bounded adaptive runner. */
|
||||
pageSegmentationMode?: 6 | 11;
|
||||
/** Internal renderer override used to obtain confidence-bearing TSV output. */
|
||||
outputFormat?: "text" | "tsv";
|
||||
/** Internal, whitelisted language family used by adaptive auto detection. */
|
||||
tesseractLanguages?: string;
|
||||
/** Internal process-ownership grace reserved by the aggregate adaptive deadline. */
|
||||
terminationGraceMs?: number;
|
||||
/** Internal preprocessed raster; auto script probing remains on inputPath. */
|
||||
recognitionInputPath?: string;
|
||||
/** Internal calibrated mode for dense, faint low-resolution documents. */
|
||||
blockLayoutOnly?: boolean;
|
||||
/** Internal pre-split scene rasters used only when primary CJK evidence is weak. */
|
||||
fallbackInputPaths?: readonly string[];
|
||||
/** Internal lazy provider that avoids splitting a strong or Latin primary image. */
|
||||
fallbackInputProvider?: () => Promise<readonly string[]>;
|
||||
/** Internal lazy provider for one bounded dense-CJK preprocessing candidate. */
|
||||
denseCjkInputProvider?: () => Promise<string>;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const DEFAULT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
|
||||
const FORCE_KILL_DELAY_MS = 1_000;
|
||||
const AUTO_LATIN_LANGUAGES = "eng+deu+fra+spa";
|
||||
const AUTO_CJK_LANGUAGES = "jpn+chi_sim";
|
||||
const AUTO_LATIN_LANGUAGE_CODES = AUTO_LATIN_LANGUAGES.split("+");
|
||||
const AUTO_CJK_LANGUAGE_CODES = AUTO_CJK_LANGUAGES.split("+");
|
||||
const ALL_TESSERACT_LANGUAGE_CODES = ALL_TESSERACT_LANGUAGES.split("+");
|
||||
// CJK packs can hallucinate ideographs on faint Latin receipts. Require a
|
||||
// baseline script density, then either strong density or a material run whose
|
||||
// confidence-weighted score beats the Latin probe. This retains genuine mixed
|
||||
// CJK receipts dominated by addresses, prices, Latin text, and digits.
|
||||
const AUTO_CJK_MIN_SCRIPT_RATIO = 0.18;
|
||||
const AUTO_CJK_STRONG_SCRIPT_RATIO = 0.3;
|
||||
const AUTO_CJK_COMPARATIVE_MIN_CHARACTERS = 5;
|
||||
// Development-corpus calibration: smaller score gains were caused by sparse
|
||||
// layout emitting extra low-value tokens. Only a substantial gain justifies
|
||||
// replacing the stable block-layout result.
|
||||
const SPARSE_LAYOUT_MIN_SCORE_GAIN = 0.25;
|
||||
const CJK_SCENE_FALLBACK_MAX_PATHS = 2;
|
||||
const CJK_SCENE_FALLBACK_MAX_PRIMARY_SCORE = 1.5;
|
||||
const CJK_SCENE_FALLBACK_MAX_PRIMARY_CHARACTERS = 128;
|
||||
const CJK_SCENE_FALLBACK_MIN_SCORE_GAIN = 0.5;
|
||||
const CJK_SCENE_FALLBACK_MIN_CHARACTER_GAIN = 64;
|
||||
// A development-only board cohort showed a narrow failure mode where sparse
|
||||
// segmentation retained a confident fragment while losing most dense text.
|
||||
// Only try one enhanced block pass for a moderately weak CJK primary, and only
|
||||
// retain it when it carries both credible confidence and substantially more
|
||||
// text. Strong pages and low-confidence noise remain byte-for-byte unchanged.
|
||||
const CJK_DENSE_ENHANCEMENT_MAX_PRIMARY_SCORE = 2.2;
|
||||
const CJK_DENSE_ENHANCEMENT_MIN_BLOCK_SCORE = 1.5;
|
||||
const CJK_DENSE_ENHANCEMENT_MIN_CHARACTER_GAIN = 64;
|
||||
const DEBIAN_TESSERACT_PACKAGE_SUFFIX: Readonly<Record<string, string>> = {
|
||||
chi_sim: "chi-sim",
|
||||
};
|
||||
|
||||
function installedSubset(languageCodes: readonly string[], installed: ReadonlySet<string>): string {
|
||||
return languageCodes.filter((language) => installed.has(language)).join("+");
|
||||
}
|
||||
|
||||
function isAllowedInternalLanguageSet(languageSet: string): boolean {
|
||||
const languages = languageSet.split("+");
|
||||
const isOrderedSubset = (allowed: readonly string[]) => {
|
||||
let previousIndex = -1;
|
||||
for (const language of languages) {
|
||||
const index = allowed.indexOf(language);
|
||||
if (index <= previousIndex) return false;
|
||||
previousIndex = index;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
return (
|
||||
languages.length > 0 &&
|
||||
(isOrderedSubset(ALL_TESSERACT_LANGUAGE_CODES) ||
|
||||
isOrderedSubset(AUTO_LATIN_LANGUAGE_CODES) ||
|
||||
isOrderedSubset(AUTO_CJK_LANGUAGE_CODES))
|
||||
);
|
||||
}
|
||||
|
||||
function missingLanguagePackError(
|
||||
requestedLanguage: TesseractLanguage,
|
||||
missingTraineddata: readonly string[],
|
||||
): Error {
|
||||
if (requestedLanguage !== "auto" && missingTraineddata.length === 1) {
|
||||
const traineddata = missingTraineddata[0];
|
||||
const debianPackageSuffix = DEBIAN_TESSERACT_PACKAGE_SUFFIX[traineddata] ?? traineddata;
|
||||
return new Error(
|
||||
`Tesseract language "${requestedLanguage}" is unavailable: missing traineddata "${traineddata}". Install Debian/Ubuntu package tesseract-ocr-${debianPackageSuffix} or the equivalent traineddata pack (Homebrew: brew install tesseract-lang), then restart SnapOtter.`,
|
||||
);
|
||||
}
|
||||
return new Error(
|
||||
`Tesseract is missing required traineddata: ${missingTraineddata.join(", ")}. Install the matching tesseract-ocr-<lang> packages or the equivalent platform language pack, then restart SnapOtter.`,
|
||||
);
|
||||
}
|
||||
|
||||
function requireSupportedAutoLanguages(installed: ReadonlySet<string>): {
|
||||
latin: string;
|
||||
cjk: string;
|
||||
} {
|
||||
const latin = installedSubset(AUTO_LATIN_LANGUAGE_CODES, installed);
|
||||
const cjk = installedSubset(AUTO_CJK_LANGUAGE_CODES, installed);
|
||||
if (!latin && !cjk) {
|
||||
throw new Error(
|
||||
"Tesseract has no supported traineddata installed. Install at least tesseract-ocr-eng (or the equivalent platform language pack), then restart SnapOtter.",
|
||||
);
|
||||
}
|
||||
return { latin, cjk };
|
||||
}
|
||||
|
||||
function requireInstalledExplicitLanguage(
|
||||
requestedLanguage: Exclude<TesseractLanguage, "auto">,
|
||||
installed: ReadonlySet<string>,
|
||||
): string {
|
||||
const traineddata = resolveTesseractLanguage(requestedLanguage);
|
||||
if (!installed.has(traineddata)) {
|
||||
throw missingLanguagePackError(requestedLanguage, [traineddata]);
|
||||
}
|
||||
return traineddata;
|
||||
}
|
||||
|
||||
interface TesseractLayoutCandidate {
|
||||
text: string;
|
||||
score: number;
|
||||
cjkCharacters: number;
|
||||
visibleCharacters: number;
|
||||
}
|
||||
|
||||
interface TesseractLayoutSelection extends TesseractLayoutCandidate {
|
||||
pageSegmentationMode: 6 | 11;
|
||||
}
|
||||
|
||||
function isCjkCharacter(character: string): boolean {
|
||||
return (
|
||||
(character >= "\u3040" && character <= "\u30ff") ||
|
||||
(character >= "\u3400" && character <= "\u9fff") ||
|
||||
(character >= "\uac00" && character <= "\ud7af") ||
|
||||
(character >= "\u1100" && character <= "\u11ff")
|
||||
);
|
||||
}
|
||||
|
||||
function parseTesseractTsv(
|
||||
tsv: string,
|
||||
options: { stripStandaloneRuleArtifacts?: boolean } = {},
|
||||
): TesseractLayoutCandidate {
|
||||
const rows = tsv.split(/\r?\n/u);
|
||||
if (
|
||||
rows[0] !==
|
||||
"level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext"
|
||||
) {
|
||||
throw new Error("Tesseract returned malformed TSV output");
|
||||
}
|
||||
const lines = new Map<string, string[]>();
|
||||
const words: Array<{ characters: number; confidence: number }> = [];
|
||||
for (const row of rows.slice(1)) {
|
||||
if (!row) continue;
|
||||
const fields = row.split("\t");
|
||||
if (
|
||||
fields.length < 12 ||
|
||||
!/^[1-5]$/u.test(fields[0]) ||
|
||||
fields.slice(1, 6).some((value) => !/^\d+$/u.test(value))
|
||||
) {
|
||||
throw new Error("Tesseract returned malformed TSV output");
|
||||
}
|
||||
if (fields[0] !== "5") continue;
|
||||
const text = fields.slice(11).join("\t").trim();
|
||||
const confidence = Number(fields[10]);
|
||||
if (!text) continue;
|
||||
if (options.stripStandaloneRuleArtifacts && /^\|+$/u.test(text)) continue;
|
||||
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 100) {
|
||||
throw new Error("Tesseract returned malformed TSV confidence");
|
||||
}
|
||||
const lineKey = fields.slice(1, 5).join(":");
|
||||
const line = lines.get(lineKey) ?? [];
|
||||
line.push(text);
|
||||
lines.set(lineKey, line);
|
||||
words.push({
|
||||
characters: Array.from(text.replace(/\s/gu, "")).length,
|
||||
confidence: confidence / 100,
|
||||
});
|
||||
}
|
||||
const text = Array.from(lines.values(), (line) => line.join(" ")).join("\n");
|
||||
const visible = Array.from(text).filter((character) => !/\s/u.test(character));
|
||||
const scriptEvidence = {
|
||||
cjkCharacters: visible.filter(isCjkCharacter).length,
|
||||
visibleCharacters: visible.length,
|
||||
};
|
||||
const characterCount = words.reduce((sum, word) => sum + word.characters, 0);
|
||||
if (characterCount === 0) return { text, score: 0, ...scriptEvidence };
|
||||
const confidenceCoverage =
|
||||
words.reduce((sum, word) => sum + word.characters * word.confidence, 0) / characterCount;
|
||||
const highConfidenceCharacters = words.reduce(
|
||||
(sum, word) => sum + word.characters * Math.max(0, Math.min(1, (word.confidence - 0.3) / 0.7)),
|
||||
0,
|
||||
);
|
||||
return {
|
||||
text,
|
||||
score: confidenceCoverage * Math.log1p(highConfidenceCharacters),
|
||||
...scriptEvidence,
|
||||
};
|
||||
}
|
||||
|
||||
export function selectTesseractLanguageFamily(
|
||||
latinTsv: string,
|
||||
cjkTsv: string,
|
||||
): typeof AUTO_LATIN_LANGUAGES | typeof AUTO_CJK_LANGUAGES {
|
||||
const latin = parseTesseractTsv(latinTsv);
|
||||
const cjk = parseTesseractTsv(cjkTsv);
|
||||
const cjkRatio = cjk.cjkCharacters / Math.max(cjk.visibleCharacters, 1);
|
||||
const hasBaselineEvidence = cjk.cjkCharacters >= 2 && cjkRatio >= AUTO_CJK_MIN_SCRIPT_RATIO;
|
||||
const hasStrongDensity = cjkRatio >= AUTO_CJK_STRONG_SCRIPT_RATIO;
|
||||
const hasStrongerComparativeEvidence =
|
||||
cjk.cjkCharacters >= AUTO_CJK_COMPARATIVE_MIN_CHARACTERS && cjk.score >= latin.score;
|
||||
return hasBaselineEvidence && (hasStrongDensity || hasStrongerComparativeEvidence)
|
||||
? AUTO_CJK_LANGUAGES
|
||||
: AUTO_LATIN_LANGUAGES;
|
||||
}
|
||||
|
||||
export function selectTesseractLayout(
|
||||
blockTsv: string,
|
||||
sparseTsv: string,
|
||||
): { pageSegmentationMode: 6 | 11; text: string } {
|
||||
const selected = selectTesseractLayoutCandidate(blockTsv, sparseTsv);
|
||||
return {
|
||||
pageSegmentationMode: selected.pageSegmentationMode,
|
||||
text: selected.text,
|
||||
};
|
||||
}
|
||||
|
||||
function selectTesseractLayoutCandidate(
|
||||
blockTsv: string,
|
||||
sparseTsv: string,
|
||||
): TesseractLayoutSelection {
|
||||
const block = parseTesseractTsv(blockTsv);
|
||||
const sparse = parseTesseractTsv(sparseTsv);
|
||||
return sparse.score >= block.score + SPARSE_LAYOUT_MIN_SCORE_GAIN
|
||||
? { pageSegmentationMode: 11, ...sparse }
|
||||
: { pageSegmentationMode: 6, ...block };
|
||||
}
|
||||
|
||||
function remainingTimeout(deadline: number, timeoutMs: number, terminationGraceMs: number): number {
|
||||
const remaining = deadline - performance.now() - terminationGraceMs;
|
||||
const bounded = Math.floor(remaining);
|
||||
if (bounded <= 0) {
|
||||
throw new Error(`Tesseract OCR timed out after ${timeoutMs}ms`);
|
||||
}
|
||||
return bounded;
|
||||
}
|
||||
|
||||
/** Run bounded block and sparse-layout candidates and retain the calibrated winner. */
|
||||
export async function runAdaptiveTesseract(
|
||||
inputPath: string,
|
||||
options: RunTesseractOptions = {},
|
||||
): Promise<TesseractResult> {
|
||||
const requestedLanguage = options.language ?? "auto";
|
||||
resolveTesseractLanguage(requestedLanguage);
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new Error("Tesseract timeout must be a positive number");
|
||||
}
|
||||
const maxTextBytes = options.maxStdoutBytes ?? options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
||||
if (!Number.isSafeInteger(maxTextBytes) || maxTextBytes <= 0) {
|
||||
throw new Error("Tesseract output limit must be a positive integer");
|
||||
}
|
||||
const maxTsvBytes = Math.max(1024 * 1024, Math.min(DEFAULT_MAX_OUTPUT_BYTES, maxTextBytes * 8));
|
||||
// Reserve the actual per-process SIGTERM-to-SIGKILL grace once against the
|
||||
// shared monotonic deadline. Every sequential candidate receives only the
|
||||
// execution time left before that reserve, so cleanup cannot stack one
|
||||
// second of overrun per candidate.
|
||||
const terminationGraceMs = Math.min(FORCE_KILL_DELAY_MS, Math.floor(timeoutMs / 4));
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
const executable = options.tesseractPath ?? process.env.TESSERACT_PATH ?? "tesseract";
|
||||
const installedLanguages = await getInstalledTesseractLanguages({
|
||||
executable,
|
||||
timeoutMs: remainingTimeout(deadline, timeoutMs, terminationGraceMs),
|
||||
signal: options.signal,
|
||||
});
|
||||
const autoFamilies =
|
||||
requestedLanguage === "auto" ? requireSupportedAutoLanguages(installedLanguages) : undefined;
|
||||
const explicitLanguage =
|
||||
requestedLanguage === "auto"
|
||||
? undefined
|
||||
: requireInstalledExplicitLanguage(requestedLanguage, installedLanguages);
|
||||
let fallbackInputPaths = options.fallbackInputPaths ?? [];
|
||||
if (
|
||||
!Array.isArray(fallbackInputPaths) ||
|
||||
fallbackInputPaths.length > CJK_SCENE_FALLBACK_MAX_PATHS ||
|
||||
fallbackInputPaths.some((path) => typeof path !== "string" || path.length === 0)
|
||||
) {
|
||||
throw new Error("Tesseract scene fallback paths are invalid");
|
||||
}
|
||||
if (
|
||||
options.fallbackInputProvider !== undefined &&
|
||||
typeof options.fallbackInputProvider !== "function"
|
||||
) {
|
||||
throw new Error("Tesseract scene fallback provider is invalid");
|
||||
}
|
||||
if (fallbackInputPaths.length > 0 && options.fallbackInputProvider) {
|
||||
throw new Error("Tesseract scene fallback inputs are ambiguous");
|
||||
}
|
||||
if (
|
||||
options.denseCjkInputProvider !== undefined &&
|
||||
typeof options.denseCjkInputProvider !== "function"
|
||||
) {
|
||||
throw new Error("Tesseract dense CJK input provider is invalid");
|
||||
}
|
||||
const hasSceneFallback =
|
||||
fallbackInputPaths.length > 0 || options.fallbackInputProvider !== undefined;
|
||||
const hasFallback = hasSceneFallback || options.denseCjkInputProvider !== undefined;
|
||||
const primaryProgressScale = hasFallback ? 0.5 : 1;
|
||||
const runCandidate = (
|
||||
candidateInputPath: string,
|
||||
pageSegmentationMode: 6 | 11,
|
||||
progressBase: number,
|
||||
progressSpan: number,
|
||||
tesseractLanguages?: string,
|
||||
) =>
|
||||
runTesseract(candidateInputPath, {
|
||||
...options,
|
||||
timeoutMs: remainingTimeout(deadline, timeoutMs, terminationGraceMs),
|
||||
terminationGraceMs,
|
||||
maxStdoutBytes: maxTsvBytes,
|
||||
pageSegmentationMode,
|
||||
outputFormat: "tsv",
|
||||
...(tesseractLanguages !== undefined && { tesseractLanguages }),
|
||||
onProgress: (progress, stage) =>
|
||||
options.onProgress?.(Math.min(100, progressBase + (progress / 100) * progressSpan), stage),
|
||||
});
|
||||
|
||||
let block: TesseractResult;
|
||||
let sparse: TesseractResult;
|
||||
let selectedLanguages: string;
|
||||
const recognitionInputPath = options.recognitionInputPath ?? inputPath;
|
||||
if (!recognitionInputPath) throw new Error("Tesseract recognition input path is invalid");
|
||||
if (requestedLanguage === "auto") {
|
||||
if (!autoFamilies) throw new Error("Tesseract auto language inventory is unavailable");
|
||||
const separateRecognitionInput = recognitionInputPath !== inputPath;
|
||||
if (autoFamilies.latin && autoFamilies.cjk) {
|
||||
const probeSpan = separateRecognitionInput ? 25 : 33;
|
||||
const latinBlock = await runCandidate(
|
||||
inputPath,
|
||||
6,
|
||||
0,
|
||||
probeSpan * primaryProgressScale,
|
||||
autoFamilies.latin,
|
||||
);
|
||||
const cjkBlock = await runCandidate(
|
||||
inputPath,
|
||||
6,
|
||||
probeSpan * primaryProgressScale,
|
||||
probeSpan * primaryProgressScale,
|
||||
autoFamilies.cjk,
|
||||
);
|
||||
selectedLanguages =
|
||||
selectTesseractLanguageFamily(latinBlock.text, cjkBlock.text) === AUTO_CJK_LANGUAGES
|
||||
? autoFamilies.cjk
|
||||
: autoFamilies.latin;
|
||||
if (separateRecognitionInput) {
|
||||
block = await runCandidate(
|
||||
recognitionInputPath,
|
||||
6,
|
||||
50 * primaryProgressScale,
|
||||
25 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
sparse = options.blockLayoutOnly
|
||||
? block
|
||||
: await runCandidate(
|
||||
recognitionInputPath,
|
||||
11,
|
||||
75 * primaryProgressScale,
|
||||
25 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
} else {
|
||||
block = selectedLanguages === autoFamilies.cjk ? cjkBlock : latinBlock;
|
||||
sparse = options.blockLayoutOnly
|
||||
? block
|
||||
: await runCandidate(
|
||||
inputPath,
|
||||
11,
|
||||
66 * primaryProgressScale,
|
||||
34 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
selectedLanguages = autoFamilies.latin || autoFamilies.cjk;
|
||||
block = await runCandidate(
|
||||
recognitionInputPath,
|
||||
6,
|
||||
0,
|
||||
50 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
sparse = options.blockLayoutOnly
|
||||
? block
|
||||
: await runCandidate(
|
||||
recognitionInputPath,
|
||||
11,
|
||||
50 * primaryProgressScale,
|
||||
50 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if (!explicitLanguage) throw new Error("Tesseract explicit language inventory is unavailable");
|
||||
selectedLanguages = explicitLanguage;
|
||||
block = await runCandidate(
|
||||
recognitionInputPath,
|
||||
6,
|
||||
0,
|
||||
50 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
sparse = options.blockLayoutOnly
|
||||
? block
|
||||
: await runCandidate(
|
||||
recognitionInputPath,
|
||||
11,
|
||||
50 * primaryProgressScale,
|
||||
50 * primaryProgressScale,
|
||||
selectedLanguages,
|
||||
);
|
||||
}
|
||||
let selected = selectTesseractLayoutCandidate(block.text, sparse.text);
|
||||
const selectedCjkLanguages = selectedLanguages
|
||||
.split("+")
|
||||
.some((language) => AUTO_CJK_LANGUAGE_CODES.includes(language));
|
||||
let fallbackProgressBase = 50;
|
||||
const denseCjkInputProvider = options.denseCjkInputProvider;
|
||||
const weakDenseCjkScene =
|
||||
denseCjkInputProvider !== undefined &&
|
||||
selectedCjkLanguages &&
|
||||
selected.score < CJK_DENSE_ENHANCEMENT_MAX_PRIMARY_SCORE;
|
||||
if (weakDenseCjkScene) {
|
||||
const denseCjkInputPath = await denseCjkInputProvider();
|
||||
if (typeof denseCjkInputPath !== "string" || denseCjkInputPath.length === 0) {
|
||||
throw new Error("Tesseract dense CJK input path is invalid");
|
||||
}
|
||||
const denseProgressSpan = hasSceneFallback ? 20 : 50;
|
||||
const denseBlock = await runCandidate(
|
||||
denseCjkInputPath,
|
||||
6,
|
||||
fallbackProgressBase,
|
||||
denseProgressSpan,
|
||||
selectedLanguages,
|
||||
);
|
||||
fallbackProgressBase += denseProgressSpan;
|
||||
const denseCandidate = parseTesseractTsv(denseBlock.text, {
|
||||
stripStandaloneRuleArtifacts: true,
|
||||
});
|
||||
if (
|
||||
denseCandidate.score >= CJK_DENSE_ENHANCEMENT_MIN_BLOCK_SCORE &&
|
||||
denseCandidate.visibleCharacters >=
|
||||
selected.visibleCharacters + CJK_DENSE_ENHANCEMENT_MIN_CHARACTER_GAIN
|
||||
) {
|
||||
selected = { pageSegmentationMode: 6, ...denseCandidate };
|
||||
}
|
||||
}
|
||||
const weakPrimaryCjkScene =
|
||||
hasSceneFallback &&
|
||||
selectedCjkLanguages &&
|
||||
selected.score < CJK_SCENE_FALLBACK_MAX_PRIMARY_SCORE &&
|
||||
selected.visibleCharacters < CJK_SCENE_FALLBACK_MAX_PRIMARY_CHARACTERS;
|
||||
if (weakPrimaryCjkScene) {
|
||||
if (options.fallbackInputProvider) {
|
||||
fallbackInputPaths = await options.fallbackInputProvider();
|
||||
if (
|
||||
!Array.isArray(fallbackInputPaths) ||
|
||||
fallbackInputPaths.length === 0 ||
|
||||
fallbackInputPaths.length > CJK_SCENE_FALLBACK_MAX_PATHS ||
|
||||
fallbackInputPaths.some((path) => typeof path !== "string" || path.length === 0)
|
||||
) {
|
||||
throw new Error("Tesseract scene fallback paths are invalid");
|
||||
}
|
||||
}
|
||||
const tiledSelections: TesseractLayoutSelection[] = [];
|
||||
const tileProgressSpan = (100 - fallbackProgressBase) / fallbackInputPaths.length;
|
||||
for (const [index, fallbackInputPath] of fallbackInputPaths.entries()) {
|
||||
const tileProgressBase = fallbackProgressBase + index * tileProgressSpan;
|
||||
const tileBlock = await runCandidate(
|
||||
fallbackInputPath,
|
||||
6,
|
||||
tileProgressBase,
|
||||
tileProgressSpan / (options.blockLayoutOnly ? 1 : 2),
|
||||
selectedLanguages,
|
||||
);
|
||||
const tileSparse = options.blockLayoutOnly
|
||||
? tileBlock
|
||||
: await runCandidate(
|
||||
fallbackInputPath,
|
||||
11,
|
||||
tileProgressBase + tileProgressSpan / 2,
|
||||
tileProgressSpan / 2,
|
||||
selectedLanguages,
|
||||
);
|
||||
tiledSelections.push(selectTesseractLayoutCandidate(tileBlock.text, tileSparse.text));
|
||||
}
|
||||
const tiledScore = tiledSelections.reduce((sum, candidate) => sum + candidate.score, 0);
|
||||
const tiledVisibleCharacters = tiledSelections.reduce(
|
||||
(sum, candidate) => sum + candidate.visibleCharacters,
|
||||
0,
|
||||
);
|
||||
if (
|
||||
tiledScore >= selected.score + CJK_SCENE_FALLBACK_MIN_SCORE_GAIN &&
|
||||
tiledVisibleCharacters >= selected.visibleCharacters + CJK_SCENE_FALLBACK_MIN_CHARACTER_GAIN
|
||||
) {
|
||||
selected = {
|
||||
pageSegmentationMode: 6,
|
||||
text: tiledSelections
|
||||
.map((candidate) => candidate.text)
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
score: tiledScore,
|
||||
cjkCharacters: tiledSelections.reduce((sum, candidate) => sum + candidate.cjkCharacters, 0),
|
||||
visibleCharacters: tiledVisibleCharacters,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (Buffer.byteLength(selected.text, "utf8") > maxTextBytes) {
|
||||
throw new Error(`Tesseract stdout exceeded ${maxTextBytes} bytes`);
|
||||
}
|
||||
options.onProgress?.(100, "Tesseract OCR complete");
|
||||
return {
|
||||
text: selected.text,
|
||||
...getTesseractRuntimeMetadata(),
|
||||
};
|
||||
}
|
||||
|
||||
export function getTesseractRuntimeMetadata(): TesseractRuntimeMetadata {
|
||||
return {
|
||||
engine: "tesseract",
|
||||
provider: "native",
|
||||
device: "cpu",
|
||||
};
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("Tesseract OCR was canceled");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Run the built-in Tesseract binary without involving the Python AI runtime. */
|
||||
export function runTesseract(
|
||||
inputPath: string,
|
||||
options: RunTesseractOptions = {},
|
||||
): Promise<TesseractResult> {
|
||||
if (options.signal?.aborted) return Promise.reject(abortError());
|
||||
|
||||
const requestedLanguage = options.language ?? "auto";
|
||||
try {
|
||||
resolveTesseractLanguage(requestedLanguage);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
if (
|
||||
options.tesseractLanguages !== undefined &&
|
||||
!isAllowedInternalLanguageSet(options.tesseractLanguages)
|
||||
) {
|
||||
return Promise.reject(new Error("Unsupported internal Tesseract language set"));
|
||||
}
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
return Promise.reject(new Error("Tesseract timeout must be a positive number"));
|
||||
}
|
||||
const terminationGraceMs = options.terminationGraceMs ?? FORCE_KILL_DELAY_MS;
|
||||
if (
|
||||
!Number.isSafeInteger(terminationGraceMs) ||
|
||||
terminationGraceMs < 0 ||
|
||||
terminationGraceMs > FORCE_KILL_DELAY_MS
|
||||
) {
|
||||
return Promise.reject(new Error("Tesseract termination grace is invalid"));
|
||||
}
|
||||
const maxStdoutBytes =
|
||||
options.maxStdoutBytes ?? options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
||||
const maxStderrBytes =
|
||||
options.maxStderrBytes ?? options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
||||
if (
|
||||
!Number.isSafeInteger(maxStdoutBytes) ||
|
||||
maxStdoutBytes <= 0 ||
|
||||
!Number.isSafeInteger(maxStderrBytes) ||
|
||||
maxStderrBytes <= 0
|
||||
) {
|
||||
return Promise.reject(new Error("Tesseract output limit must be a positive integer"));
|
||||
}
|
||||
|
||||
const executable = options.tesseractPath ?? process.env.TESSERACT_PATH ?? "tesseract";
|
||||
const installedLanguages = getCachedTesseractLanguages(executable);
|
||||
if (!installedLanguages) {
|
||||
const preflightStarted = performance.now();
|
||||
return getInstalledTesseractLanguages({
|
||||
executable,
|
||||
timeoutMs,
|
||||
signal: options.signal,
|
||||
}).then(() => {
|
||||
const remainingMs = Math.floor(timeoutMs - (performance.now() - preflightStarted));
|
||||
if (remainingMs <= 0) {
|
||||
throw new Error(`Tesseract OCR timed out after ${timeoutMs}ms`);
|
||||
}
|
||||
return runTesseract(inputPath, { ...options, timeoutMs: remainingMs });
|
||||
});
|
||||
}
|
||||
let language: string;
|
||||
if (options.tesseractLanguages !== undefined) {
|
||||
language = options.tesseractLanguages;
|
||||
const missing = language
|
||||
.split("+")
|
||||
.filter((traineddata) => !installedLanguages.has(traineddata));
|
||||
if (missing.length > 0) {
|
||||
return Promise.reject(missingLanguagePackError(requestedLanguage, missing));
|
||||
}
|
||||
} else if (requestedLanguage === "auto") {
|
||||
try {
|
||||
requireSupportedAutoLanguages(installedLanguages);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
language = installedSubset(ALL_TESSERACT_LANGUAGE_CODES, installedLanguages);
|
||||
} else {
|
||||
try {
|
||||
language = requireInstalledExplicitLanguage(requestedLanguage, installedLanguages);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
}
|
||||
options.onProgress?.(0, "Starting Tesseract OCR");
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const args = [inputPath, "stdout", "-l", language];
|
||||
if (options.pageSegmentationMode !== undefined) {
|
||||
args.push("--psm", String(options.pageSegmentationMode));
|
||||
}
|
||||
if (options.outputFormat === "tsv") args.push("tsv");
|
||||
const child = spawn(executable, args, {
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
let stdoutBytes = 0;
|
||||
let stderrBytes = 0;
|
||||
let settled = false;
|
||||
let terminationError: Error | undefined;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
const timeoutTimer = setTimeout(() => {
|
||||
terminate(new Error(`Tesseract OCR timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
timeoutTimer.unref();
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutTimer);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
|
||||
const finish = (error?: Error, result?: TesseractResult) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
if (error) reject(error);
|
||||
else resolve(result as TesseractResult);
|
||||
};
|
||||
|
||||
const finishTermination = () => {
|
||||
if (terminationError) finish(terminationError);
|
||||
};
|
||||
|
||||
function terminate(error: Error) {
|
||||
if (settled || terminationError) return;
|
||||
terminationError = error;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// A concurrent process exit owns settlement through close/error.
|
||||
}
|
||||
if (settled) return;
|
||||
forceKillTimer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// Wait for close before releasing request-owned scratch state.
|
||||
}
|
||||
}
|
||||
}, terminationGraceMs);
|
||||
forceKillTimer.unref();
|
||||
}
|
||||
|
||||
const onAbort = () => terminate(abortError());
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
if (options.signal?.aborted) terminate(abortError());
|
||||
|
||||
child.stdout.on("data", (chunk: Buffer | string) => {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
stdoutBytes += buffer.length;
|
||||
if (stdoutBytes > maxStdoutBytes) {
|
||||
terminate(new Error(`Tesseract stdout exceeded ${maxStdoutBytes} bytes`));
|
||||
return;
|
||||
}
|
||||
stdoutChunks.push(buffer);
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer | string) => {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
stderrBytes += buffer.length;
|
||||
if (stderrBytes > maxStderrBytes) {
|
||||
terminate(new Error(`Tesseract stderr exceeded ${maxStderrBytes} bytes`));
|
||||
return;
|
||||
}
|
||||
stderrChunks.push(buffer);
|
||||
});
|
||||
|
||||
child.once("error", (error: NodeJS.ErrnoException) => {
|
||||
if (terminationError) {
|
||||
return;
|
||||
}
|
||||
if (error.code === "ENOENT") {
|
||||
finish(
|
||||
new Error("Tesseract executable not found. Install Tesseract or set TESSERACT_PATH.", {
|
||||
cause: error,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
finish(new Error(`Unable to start Tesseract: ${error.message}`, { cause: error }));
|
||||
});
|
||||
|
||||
child.once("close", (code, signal) => {
|
||||
if (terminationError) {
|
||||
finishTermination();
|
||||
return;
|
||||
}
|
||||
if (code !== 0) {
|
||||
const detail = Buffer.concat(stderrChunks).toString("utf8").trim();
|
||||
const status = code === null ? `signal ${signal ?? "unknown"}` : `code ${code}`;
|
||||
finish(new Error(`Tesseract exited with ${status}${detail ? `: ${detail}` : ""}`));
|
||||
return;
|
||||
}
|
||||
|
||||
options.onProgress?.(100, "Tesseract OCR complete");
|
||||
finish(undefined, {
|
||||
text: Buffer.concat(stdoutChunks).toString("utf8"),
|
||||
...getTesseractRuntimeMetadata(),
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user