fix: GPU deployment robustness (6 fixes from end-to-end testing on an RTX 4070) (#334)

* fix(docker): pin CUDA base to 12.6 so the GPU image starts on R560+ drivers

The amd64 base nvidia/cuda:12.9.2-cudnn-runtime bakes a cuda>=12.9 driver gate enforced by nvidia-container-toolkit at container start, so the image fails to launch on common production drivers (e.g. 570.x / CUDA 12.8). The AI bundles are all cu126 wheels and the image installs libcublas-12-6, so 12.9 was misaligned with the workload. Pin to nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04 to match the wheels and lower the driver floor to R560+.

* fix(ai): broaden OOM detection so the rembg lighter-model fallback fires

onnxruntime/CUDA allocation failures surface as 'Failed to allocate memory for requested buffer', CUBLAS_STATUS_ALLOC_FAILED, or bad_alloc, not just 'out of memory'. The background-removal and transparency-fixer fallback-to-lighter-model paths only matched the literal 'out of memory', so the fallback was dead code and transparency-fixer (default birefnet-hr-matting) always failed with an allocation error. Add isMemoryAllocError() and use it in both checks.

* fix(ai): use bundled PaddleOCR models so OCR runs offline

ocr.py passed no model dirs to PaddleOCR, so PaddleX resolved models from ~/.paddlex and downloaded them from HuggingFace at runtime (slow first use, broken air-gapped), ignoring the models the OCR bundle ships in MODELS_PATH; it also pulled doc-orientation/unwarping models that are not bundled. Pin detection, recognition and textline models to the bundled dirs in MODELS_PATH (per language) and disable use_doc_orientation_classify / use_doc_unwarping, with per-component fallback when a model is absent. Verified: OCR runs with zero HuggingFace requests.

* fix(docker): add CAP_KILL so container shutdown is graceful

cap_drop: ALL without re-adding KILL meant tini (PID 1, root) could not forward SIGTERM to the gosu-dropped snapotter process (root minus CAP_KILL cannot signal a different UID). docker stop logged '[FATAL tini] forwarding signal: Operation not permitted', never delivered the signal, and fell back to SIGKILL after the 10s timeout. Add KILL to cap_add in both compose files. Verified: docker stop completes in 0s with SIGTERM delivered (exit 143) and no FATAL tini.

* fix(ai): serialize bundle installs against AI jobs to prevent sidecar segfault

A feature bundle install rewrites the shared Python venv (pip + copytree of site-packages/*.so) as a background subprocess, with no coordination against AI tool jobs that dlopen native libs (torch / onnxruntime CUDA) from the same venv; a job loading a shared object while it is overwritten segfaults the sidecar. Add a process-wide async mutex (venv-lock.ts): bridge.run() acquires it before every AI script and the install route holds it across the installer subprocess. Both run in the same Node process so a module-level lock suffices. Verified: concurrent install + AI job produces zero segfaults and the job serializes behind the install.

* fix(ai): make the venv lock read/write so concurrent AI jobs are not serialized

The first cut used an exclusive mutex, which (a) deferred the dispatcher spawn by a microtask and broke unit tests that synchronously drive the mocked spawn, and (b) serialized AI jobs against each other, removing the dispatcher's by-id request multiplexing. Make it a writer-preferring read/write lock: AI jobs are shared readers (with a synchronous fast path so spawn still happens in-tick) and a bundle install is the exclusive writer. Verified: all 764 AI unit tests pass.

* fix(ai): degrade OCR to Tesseract on CPU-only hosts instead of segfaulting

The amd64 AI bundle ships paddlepaddle-gpu, whose native libs dlopen
libcuda.so.1 at import and segfault on a host without a GPU (libcuda is the
driver lib, injected only by nvidia-container-toolkit on GPU hosts). The
segfault crashed the shared long-lived AI dispatcher and, after a few attempts,
tripped the bridge crash-recovery permanent-disable, wedging all AI until a
container restart. The standalone ocr tool defaults to quality=balanced
(PaddleOCR), so it hit this on every CPU-only deployment; ocr-pdf already
hardcoded Tesseract and was unaffected.

ocr.py now gates the PaddleOCR tiers on gpu_available(): balanced/best
transparently fall back to fast (Tesseract, CPU-capable) when no usable GPU is
present, and run_paddleocr_v5/run_paddleocr_vl refuse before importing paddle so
the GPU build is never dlopen'd on CPU. GPU hosts are unchanged.

Verified on a CPU-only Windows/WSL2 box: ocr returns Tesseract text across
repeated runs with the dispatcher staying healthy (no wedge).

apps/api/src/routes/features.ts

@@ -20,7 +20,7 @@ import {
   unlinkSync,
 } from "node:fs";
 import { join } from "node:path";
-import { shutdownDispatcher } from "@snapotter/ai";
+import { acquireVenvLock, shutdownDispatcher } from "@snapotter/ai";
 import { ANALYTICS_EVENTS, FEATURE_BUNDLES } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { trackEvent } from "../lib/analytics.js";
@@ -167,6 +167,19 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
       const installStartTime = Date.now();
       const reqRef = request;
 
+      // Hold the venv lock across the whole install so no AI tool job loads
+      // native libs from the venv while pip is rewriting them (that segfaults
+      // the sidecar). This awaits any in-flight AI job before the installer
+      // starts; the lock is released when the installer process exits.
+      const releaseVenv = await acquireVenvLock();
+      let venvReleased = false;
+      const releaseVenvOnce = () => {
+        if (!venvReleased) {
+          venvReleased = true;
+          releaseVenv();
+        }
+      };
+
       const child = spawn(pythonPath, [scriptPath, bundleId, manifestPath, modelsDir], {
         stdio: ["ignore", "pipe", "pipe"],
         env: {
@@ -219,6 +232,7 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
       });
 
       child.on("close", (code) => {
+        releaseVenvOnce();
         releaseInstallLock();
 
         if (code === 0) {
@@ -275,6 +289,7 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
       });
 
       child.on("error", (err) => {
+        releaseVenvOnce();
         releaseInstallLock();
         const errorMsg = `Failed to spawn install process: ${err.message}`;
         setInstallProgress(bundleId, null, errorMsg);

apps/api/src/routes/tools/transparency-fixer.ts

@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
 import { mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { removeBackground } from "@snapotter/ai";
+import { isMemoryAllocError, removeBackground } from "@snapotter/ai";
 import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import sharp from "sharp";
@@ -104,7 +104,7 @@ async function processTransparencyFix(
       onProgress,
     );
   } catch (err) {
-    const isOom = err instanceof Error && err.message.includes("out of memory");
+    const isOom = isMemoryAllocError(err);
     if (!isOom) throw err;
 
     onProgress?.(5, `Retrying with fallback model (${FALLBACK_MODEL})`);

docker/Dockerfile

@@ -166,7 +166,12 @@ RUN curl -fsSL --retry 3 --retry-delay 5 "https://github.com/strukturag/libheif/
 # Pin tags to specific major.minor for reproducible builds.
 # ============================================
 FROM node:22-bookworm@sha256:e0d149b4727ac0c20d9774e801e423d7a946a0bffced886f42cfe9cd3c67820a AS base-linux-arm64
-FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04@sha256:070f8f2672df1b05b84c0409a5fd1d54ddfd646e5b9d8dee7878131271b563fc AS base-linux-amd64
+# CUDA base must match the AI bundles' wheels (torch/paddle/onnxruntime-gpu are all
+# cu126) and the libcublas-12-6 install below. It also sets the NVIDIA_REQUIRE_CUDA
+# driver gate enforced by nvidia-container-toolkit at container start: a 12.6 base
+# needs driver R560+, vs 12.9 which needs R575+ and fails to start on common
+# production drivers (e.g. 570.x / CUDA 12.8). Keep this at 12.6.x.
+FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04@sha256:8aef630a54bc5c5146ae5ce68e6af5caa3df0fb690bb91544175c91f307e4356 AS base-linux-amd64
 
 # Node.js donor: provides Node binaries for the CUDA amd64 image without
 # relying on NodeSource apt repos or Ubuntu mirrors (which are flaky on CI).

docker/docker-compose-gpu.yml

@@ -84,6 +84,11 @@ services:
       - SETGID
       - DAC_OVERRIDE
       - FOWNER
+      # KILL lets tini (PID 1, root) forward SIGTERM to the gosu-dropped
+      # snapotter process on shutdown. Without it, root minus CAP_KILL cannot
+      # signal a different-UID process, so docker stop is ungraceful
+      # ("[FATAL tini] forwarding signal: Operation not permitted" -> SIGKILL).
+      - KILL
     # NOTE: security_opt: [no-new-privileges:true] is intentionally omitted.
     # gosu requires setuid to drop from root to the snapotter user.
     # Mitigation: cap_drop: ALL limits available capabilities after privilege drop.

docker/docker-compose.yml

@@ -83,6 +83,11 @@ services:
       - SETGID
       - DAC_OVERRIDE
       - FOWNER
+      # KILL lets tini (PID 1, root) forward SIGTERM to the gosu-dropped
+      # snapotter process on shutdown. Without it, root minus CAP_KILL cannot
+      # signal a different-UID process, so docker stop is ungraceful
+      # ("[FATAL tini] forwarding signal: Operation not permitted" -> SIGKILL).
+      - KILL
     # NOTE: security_opt: [no-new-privileges:true] is intentionally omitted.
     # gosu requires setuid to drop from root to the snapotter user.
     # Mitigation: cap_drop: ALL limits available capabilities after privilege drop.

packages/ai/python/ocr.py

@@ -7,8 +7,8 @@ import os
 # Without these, paddlepaddle-gpu can segfault during import on machines without
 # a GPU, because the C++ layer attempts GPU initialization before Python-level
 # device routing takes effect.  Must run before any PaddleOCR import.
-from gpu import gpu_available as _gpu_available
-if not _gpu_available():
+from gpu import gpu_available
+if not gpu_available():
     if not os.environ.get("FLAGS_use_cuda"):
         os.environ["FLAGS_use_cuda"] = "0"
     if not os.environ.get("FLAGS_use_cudnn"):
@@ -23,6 +23,25 @@ def emit_progress(percent, stage):
     print(json.dumps({"progress": percent, "stage": stage}), file=sys.stderr, flush=True)
 
 
+# OCR quality tiers backed by PaddleOCR. PaddleOCR ships as the GPU build
+# (paddlepaddle-gpu) in the amd64 bundle; its native libs dlopen libcuda.so.1 at
+# import and segfault on a CPU-only host, so these tiers need a usable GPU.
+PADDLE_QUALITY_TIERS = ("balanced", "best")
+
+
+def effective_quality(requested):
+    """Return the OCR quality tier that can actually run on this host.
+
+    On a CPU-only host the PaddleOCR tiers (balanced/best) cannot load, so they
+    transparently fall back to "fast" (Tesseract), which runs on CPU. This keeps a
+    GPU-less host from importing paddlepaddle-gpu, whose import segfaults and wedges
+    the shared AI dispatcher.
+    """
+    if requested in PADDLE_QUALITY_TIERS and not gpu_available():
+        return "fast"
+    return requested
+
+
 TESSERACT_LANG_MAP = {
     "en": "eng", "de": "deu", "fr": "fra", "es": "spa",
     "zh": "chi_sim", "ja": "jpn", "ko": "kor",
@@ -33,6 +52,60 @@ PADDLE_LANG_MAP = {
     "zh": "ch", "ja": "japan", "ko": "korean",
 }
 
+# Bundled PaddleOCR models (shipped by the OCR feature bundle into MODELS_PATH).
+# Pinning the constructor at these dirs keeps OCR fully offline / air-gapped and
+# skips slow HuggingFace model resolution on first use. PP-OCRv5 server rec
+# covers Chinese+English; latin covers en/de/fr/es; plus a dedicated Korean rec.
+PADDLE_DET_MODEL = "PP-OCRv5_server_det"
+PADDLE_TEXTLINE_MODEL = "PP-LCNet_x1_0_textline_ori"
+PADDLE_REC_MODEL = {
+    "ch": "PP-OCRv5_server_rec",
+    "en": "latin_PP-OCRv5_mobile_rec",
+    "latin": "latin_PP-OCRv5_mobile_rec",
+    "korean": "korean_PP-OCRv5_mobile_rec",
+}
+
+
+def _bundled_paddle_kwargs(paddle_lang):
+    """Build PaddleOCR kwargs that use the bundled models in MODELS_PATH.
+
+    Only pins a component when its model is actually present on disk, so a
+    partial bundle (or an unbundled language such as Japanese) falls back to
+    PaddleOCR's default resolution for that component. The doc-orientation and
+    doc-unwarping models are not bundled and not needed for plain OCR, so they
+    are disabled to avoid a runtime HuggingFace download.
+    """
+    models_dir = os.environ.get("MODELS_PATH", "/data/ai/models")
+
+    def model_dir(name):
+        if not name:
+            return None
+        path = os.path.join(models_dir, name)
+        return path if os.path.isdir(path) else None
+
+    kwargs = {"use_doc_orientation_classify": False, "use_doc_unwarping": False}
+
+    det = model_dir(PADDLE_DET_MODEL)
+    if det:
+        kwargs["text_detection_model_name"] = PADDLE_DET_MODEL
+        kwargs["text_detection_model_dir"] = det
+
+    rec_name = PADDLE_REC_MODEL.get(paddle_lang)
+    rec = model_dir(rec_name)
+    if rec:
+        kwargs["text_recognition_model_name"] = rec_name
+        kwargs["text_recognition_model_dir"] = rec
+
+    textline = model_dir(PADDLE_TEXTLINE_MODEL)
+    if textline:
+        kwargs["textline_orientation_model_name"] = PADDLE_TEXTLINE_MODEL
+        kwargs["textline_orientation_model_dir"] = textline
+        kwargs["use_textline_orientation"] = True
+    else:
+        kwargs["use_textline_orientation"] = False
+
+    return kwargs
+
 
 def auto_detect_language(input_path):
     """Detect the predominant script in the image using Tesseract multi-lang.
@@ -121,6 +194,13 @@ def _extract_ocr_texts(results):
 
 def run_paddleocr_v5(input_path, language):
     """Run PaddleOCR PP-OCRv5 server models (Balanced tier)."""
+    # GPU-only: paddlepaddle-gpu segfaults at import on a CPU-only host (libcuda
+    # absent). Refuse before importing so the caller falls back to Tesseract.
+    if not gpu_available():
+        raise ImportError(
+            "PaddleOCR (paddlepaddle-gpu) requires a GPU; the amd64 bundle ships the "
+            "GPU build, which cannot load on a CPU-only host. Use quality=fast (Tesseract)."
+        )
     os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True"
 
     stdout_fd = os.dup(1)
@@ -129,7 +209,6 @@ def run_paddleocr_v5(input_path, language):
     try:
         import logging
         from paddleocr import PaddleOCR
-        from gpu import gpu_available
 
         # Suppress PaddleOCR internal logging (replaces removed show_log param)
         for name in ("ppocr", "paddleocr", "paddle"):
@@ -139,11 +218,17 @@ def run_paddleocr_v5(input_path, language):
         device = "gpu:0" if gpu_available() else "cpu"
 
         emit_progress(20, "Loading")
+        mk = _bundled_paddle_kwargs(paddle_lang)
+        # When a bundled recognizer is pinned, the model selects the script, so
+        # we omit lang (this is the proven fully-offline path). Only fall back to
+        # lang-based (online) resolution when no bundled rec exists (e.g. ja).
+        if "text_recognition_model_dir" not in mk:
+            mk["lang"] = paddle_lang
         ocr = PaddleOCR(
-            lang=paddle_lang,
             device=device,
             ocr_version="PP-OCRv5",
             enable_mkldnn=False,
+            **mk,
         )
         emit_progress(30, "Scanning")
         results = ocr.predict(input=input_path)
@@ -165,6 +250,12 @@ def run_paddleocr_vl(input_path):
     Requires PaddlePaddle >= 3.2 for fused_rms_norm_ext.
     """
     global _paddleocr_vl_instance
+    # GPU-only: see run_paddleocr_v5. Refuse before importing paddle on CPU.
+    if not gpu_available():
+        raise ImportError(
+            "PaddleOCR-VL (paddlepaddle-gpu) requires a GPU; the amd64 bundle ships the "
+            "GPU build, which cannot load on a CPU-only host. Use quality=balanced or fast."
+        )
     os.environ["PADDLE_PDX_DISABLE_MODEL_SOURCE_CHECK"] = "True"
 
     stdout_fd = os.dup(1)
@@ -174,7 +265,6 @@ def run_paddleocr_vl(input_path):
         if _paddleocr_vl_instance is None:
             emit_progress(15, "Loading model")
             from paddleocr import PaddleOCRVL
-            from gpu import gpu_available
 
             device = "gpu" if gpu_available() else "cpu"
             _paddleocr_vl_instance = PaddleOCRVL(device=device)
@@ -225,6 +315,17 @@ def main():
         engine = settings.get("engine", "tesseract")
         quality = "fast" if engine == "tesseract" else "balanced"
 
+    # On a CPU-only host, downgrade GPU-only tiers (PaddleOCR) to Tesseract so we
+    # never import paddlepaddle-gpu, whose import segfaults and wedges the dispatcher.
+    downgraded = effective_quality(quality)
+    if downgraded != quality:
+        print(
+            json.dumps({"info": f"{quality} OCR needs a GPU; using {downgraded} (Tesseract) on CPU"}),
+            file=sys.stderr,
+            flush=True,
+        )
+        quality = downgraded
+
     preprocessed_path = None
     try:
         emit_progress(5, "Preparing")

packages/ai/python/tests/test_ocr_cpu_fallback.py

@@ -0,0 +1,44 @@
+"""OCR must degrade gracefully on CPU-only hosts.
+
+The amd64 AI bundle ships paddlepaddle-gpu, whose native libraries dlopen
+libcuda.so.1 at import time and segfault on a host without a GPU (libcuda is the
+driver lib, injected only by nvidia-container-toolkit on GPU hosts). That segfault
+crashes the shared long-lived AI dispatcher and wedges all AI. So on a CPU-only
+host the PaddleOCR tiers (balanced/best) must transparently fall back to Tesseract
+and must never reach the paddle import.
+"""
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+import ocr  # noqa: E402
+
+
+def test_effective_quality_downgrades_paddle_tiers_on_cpu(monkeypatch):
+    monkeypatch.setattr(ocr, "gpu_available", lambda: False)
+    assert ocr.effective_quality("balanced") == "fast"
+    assert ocr.effective_quality("best") == "fast"
+    assert ocr.effective_quality("fast") == "fast"
+
+
+def test_effective_quality_preserves_paddle_tiers_on_gpu(monkeypatch):
+    monkeypatch.setattr(ocr, "gpu_available", lambda: True)
+    assert ocr.effective_quality("balanced") == "balanced"
+    assert ocr.effective_quality("best") == "best"
+    assert ocr.effective_quality("fast") == "fast"
+
+
+def test_run_paddleocr_v5_refuses_on_cpu_before_import(monkeypatch):
+    # Must raise a GPU-specific error (the guard), NOT attempt the paddle import
+    # that would segfault on a CPU-only host.
+    monkeypatch.setattr(ocr, "gpu_available", lambda: False)
+    with pytest.raises(ImportError, match="GPU"):
+        ocr.run_paddleocr_v5("/nonexistent.png", "en")
+
+
+def test_run_paddleocr_vl_refuses_on_cpu_before_import(monkeypatch):
+    monkeypatch.setattr(ocr, "gpu_available", lambda: False)
+    with pytest.raises(ImportError, match="GPU"):
+        ocr.run_paddleocr_vl("/nonexistent.png")

packages/ai/src/background-removal.ts

@@ -15,6 +15,19 @@ export interface RemoveBackgroundOptions {
 const MAX_REMBG_PX = Number(process.env.MAX_REMBG_PX) || 2048;
 const OOM_FALLBACK_MODEL = "u2net";
 
+/**
+ * onnxruntime / CUDA surface allocation failures with several different
+ * messages ("out of memory", "Failed to allocate memory for requested buffer",
+ * CUBLAS_STATUS_ALLOC_FAILED, bad_alloc). Match them all so the lighter-model
+ * fallback actually triggers instead of failing the job.
+ */
+export function isMemoryAllocError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  return /out of memory|failed to allocate|cudaerrormemoryallocation|cublas_status_alloc_failed|bad_alloc/i.test(
+    err.message,
+  );
+}
+
 export async function removeBackground(
   inputBuffer: Buffer,
   outputDir: string,
@@ -80,7 +93,7 @@ async function runAndParse(
     }
     return readFile(outputPath);
   } catch (err) {
-    const isOom = err instanceof Error && err.message.includes("out of memory");
+    const isOom = isMemoryAllocError(err);
     const canFallback = isOom && options.model !== OOM_FALLBACK_MODEL;
 
     if (!canFallback) throw err;

packages/ai/src/bridge.ts

@@ -4,6 +4,7 @@ import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { context, propagation, SpanStatusCode, trace } from "@opentelemetry/api";
 import { missingBundleForScript } from "./feature-gate.js";
+import { acquireVenvRead, tryAcquireVenvRead } from "./venv-lock.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PYTHON_DIR = resolve(__dirname, "../python");
@@ -624,23 +625,33 @@ export class PythonDispatcher {
       return this.runPerRequest(scriptName, args, options);
     };
 
-    if (!span) return doRun();
+    const runWithSpan = (): Promise<{ stdout: string; stderr: string }> => {
+      if (!span) return doRun();
+      return doRun().then(
+        (result) => {
+          span.end();
+          return result;
+        },
+        (err) => {
+          span.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: err instanceof Error ? err.message : String(err),
+          });
+          span.recordException(err instanceof Error ? err : new Error(String(err)));
+          span.end();
+          throw err;
+        },
+      );
+    };
 
-    return doRun().then(
-      (result) => {
-        span.end();
-        return result;
-      },
-      (err) => {
-        span.setStatus({
-          code: SpanStatusCode.ERROR,
-          message: err instanceof Error ? err.message : String(err),
-        });
-        span.recordException(err instanceof Error ? err : new Error(String(err)));
-        span.end();
-        throw err;
-      },
-    );
+    // Serialize against bundle installs that mutate the shared venv: a job that
+    // dlopens native libs (torch / onnxruntime CUDA) while pip rewrites them
+    // segfaults the sidecar. In the common case the lock is free, so acquire it
+    // synchronously and run inline (no microtask deferral, preserving the prior
+    // call timing); only when an install holds it do we await.
+    const release = tryAcquireVenvRead();
+    if (release) return runWithSpan().finally(release);
+    return acquireVenvRead().then((r) => runWithSpan().finally(r));
   }
 }
 

packages/ai/src/index.ts

@@ -1,4 +1,4 @@
-export { removeBackground } from "./background-removal.js";
+export { isMemoryAllocError, removeBackground } from "./background-removal.js";
 export type { DispatcherStatus } from "./bridge.js";
 export {
   getDispatcherStatus,
@@ -29,3 +29,4 @@ export { seamCarve } from "./seam-carving.js";
 export type { TranscribeOptions, TranscriptionResult, TranscriptSegment } from "./transcription.js";
 export { transcribeAudio } from "./transcription.js";
 export { upscale } from "./upscaling.js";
+export { acquireVenvLock } from "./venv-lock.js";

packages/ai/src/venv-lock.ts

@@ -0,0 +1,87 @@
+/**
+ * Read/write lock between AI tool jobs (readers) and bundle installs (writers).
+ *
+ * A bundle install rewrites the shared venv's site-packages (pip + copytree of
+ * *.so), while AI tool jobs dlopen native libraries (torch / onnxruntime CUDA)
+ * from that same venv. Loading a shared object while it is being overwritten
+ * segfaults the sidecar, so an install must not overlap with any AI job.
+ *
+ * AI jobs may run concurrently with each other -- the persistent dispatcher
+ * multiplexes requests by id -- so they are shared readers; only an install
+ * needs exclusivity, so it is the writer. Writer-preferring, so an install is
+ * not starved by a steady stream of jobs.
+ *
+ * Both sides live in the same Node process (the Fastify route spawns the
+ * installer; in-process BullMQ workers run the jobs), so a module-level lock
+ * shared via Node's module cache is sufficient.
+ *
+ * Readers have a synchronous fast path (tryAcquireVenvRead) so a job's work
+ * begins in the same tick when no install is active or pending -- exactly as
+ * before this lock existed. Every acquire returns a release function; always
+ * call it (it is idempotent).
+ */
+type Release = () => void;
+
+let readers = 0;
+let writerActive = false;
+const writeWaiters: Array<() => void> = [];
+const readWaiters: Array<() => void> = [];
+
+function readRelease(): Release {
+  let done = false;
+  return () => {
+    if (done) return;
+    done = true;
+    readers--;
+    if (readers === 0 && writeWaiters.length > 0) {
+      writerActive = true;
+      writeWaiters.shift()?.();
+    }
+  };
+}
+
+function writeRelease(): Release {
+  let done = false;
+  return () => {
+    if (done) return;
+    done = true;
+    writerActive = false;
+    if (writeWaiters.length > 0) {
+      writerActive = true;
+      writeWaiters.shift()?.();
+    } else {
+      const granted = readWaiters.splice(0);
+      for (const grant of granted) grant();
+    }
+  };
+}
+
+/** AI-job (reader) sync fast path; null if an install is active or waiting. */
+export function tryAcquireVenvRead(): Release | null {
+  if (writerActive || writeWaiters.length > 0) return null;
+  readers++;
+  return readRelease();
+}
+
+/** AI-job (reader) acquire; waits while an install holds or is awaiting the lock. */
+export function acquireVenvRead(): Promise<Release> {
+  const r = tryAcquireVenvRead();
+  if (r) return Promise.resolve(r);
+  return new Promise<Release>((resolve) => {
+    readWaiters.push(() => {
+      readers++;
+      resolve(readRelease());
+    });
+  });
+}
+
+/** Bundle-install (writer) exclusive acquire; waits for in-flight AI jobs. */
+export function acquireVenvLock(): Promise<Release> {
+  if (!writerActive && readers === 0 && writeWaiters.length === 0) {
+    writerActive = true;
+    return Promise.resolve(writeRelease());
+  }
+  return new Promise<Release>((resolve) => {
+    writeWaiters.push(() => resolve(writeRelease()));
+  });
+}