Enhance logging and error handling across tools; add full tool audit and Playwright tests

- Added model mismatch warnings in colorize, enhance-faces, and upscale routes.
- Improved error handling in colorize, enhance_faces, remove_bg, restore, and upscale scripts with detailed logging.
- Updated Dockerfile to align NCCL versions for compatibility.
- Introduced a new full tool audit script to test all tools for functionality and GPU usage.
- Created Playwright E2E tests for GPU-dependent tools to ensure proper functionality and performance.

docker/Dockerfile

@@ -108,7 +108,7 @@ RUN set -e; \
 # Stage 3: Platform-specific base images
 # ============================================
 FROM node:22-bookworm AS base-linux-arm64
-FROM nvidia/cuda:12.6.3-runtime-ubuntu24.04 AS base-linux-amd64
+FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04 AS base-linux-amd64
 
 # Node.js donor: provides Node binaries for the CUDA amd64 image without
 # relying on NodeSource apt repos or Ubuntu mirrors (which are flaky on CI).
@@ -212,6 +212,18 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 RUN --mount=type=cache,target=/root/.cache/pip \
     /opt/venv/bin/pip install numpy==1.26.4
 
+# Re-align NCCL to match torch's requirement.  paddlepaddle-gpu pins an older
+# nvidia-nccl-cu12 which silently downgrades the version installed by torch,
+# causing "undefined symbol: ncclCommShrink" at import time.  NCCL is ABI-
+# backwards-compatible, so the newer version satisfies both packages.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    if [ "$TARGETARCH" = "amd64" ]; then \
+    /opt/venv/bin/pip install $(/opt/venv/bin/python3 -c \
+        "from importlib.metadata import requires; \
+         print([r.split(';')[0].strip() for r in requires('torch') \
+                if 'nccl' in r][0])") \
+    ; fi
+
 # Pin rembg model storage to a fixed path so models downloaded at build time
 # (as root) are found at runtime (as the non-root ashim user, home=/app).
 # Without this, rembg defaults to ~/.u2net which differs between users.
@@ -227,7 +239,7 @@ COPY docker/download_models.py /tmp/download_models.py
 RUN if [ "$SKIP_MODEL_DOWNLOADS" = "true" ]; then \
         echo "Skipping model downloads (CI build)"; \
     else \
-        /opt/venv/bin/python3 /tmp/download_models.py; \
+        CUDA_VISIBLE_DEVICES="" /opt/venv/bin/python3 /tmp/download_models.py; \
     fi && rm -f /tmp/download_models.py && \
     # Symlink PaddleX model dir into both possible HOME locations so models are
     # found regardless of whether HOME=/root (build/root context) or HOME=/app

docker/download_models.py

@@ -70,6 +70,37 @@ os.environ["PADDLE_DEVICE"] = "cpu"
 os.environ["FLAGS_use_cuda"] = "0"
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 
+# Prevent ONNX Runtime from loading the CUDA Execution Provider at build time.
+# The cudnn-runtime base image includes cuDNN, which makes ONNX try to init
+# the CUDA EP.  Without a GPU driver (only available at container runtime),
+# this segfaults.  Temporarily renaming the provider .so is the most reliable
+# way to prevent this — env vars alone are not enough.
+def _hide_cuda_provider():
+    """Rename ONNX CUDA provider .so to prevent load at build time."""
+    try:
+        import onnxruntime as _ort
+        ep_dir = os.path.join(os.path.dirname(_ort.__file__), "capi")
+        for name in ("libonnxruntime_providers_cuda.so", "libonnxruntime_providers_tensorrt.so"):
+            src = os.path.join(ep_dir, name)
+            if os.path.exists(src):
+                os.rename(src, src + ".build_hide")
+    except ImportError:
+        pass
+
+def _restore_cuda_provider():
+    """Restore hidden ONNX CUDA provider .so after build-time downloads."""
+    try:
+        import onnxruntime as _ort
+        ep_dir = os.path.join(os.path.dirname(_ort.__file__), "capi")
+        for name in ("libonnxruntime_providers_cuda.so", "libonnxruntime_providers_tensorrt.so"):
+            bak = os.path.join(ep_dir, name + ".build_hide")
+            if os.path.exists(bak):
+                os.rename(bak, os.path.join(ep_dir, name))
+    except ImportError:
+        pass
+
+_hide_cuda_provider()
+
 LAMA_MODEL_DIR = "/opt/models/lama"
 LAMA_MODEL_URL = "https://huggingface.co/Carve/LaMa-ONNX/resolve/main/lama_fp32.onnx"
 LAMA_MODEL_PATH = os.path.join(LAMA_MODEL_DIR, "lama_fp32.onnx")
@@ -686,6 +717,7 @@ def main():
     print("\nAll downloads complete. Running verification...\n")
     verify_mediapipe()
     smoke_test()
+    _restore_cuda_provider()
     print("All models downloaded and verified.")