Enhance logging and error handling across tools; add full tool audit and Playwright tests

- Added model mismatch warnings in colorize, enhance-faces, and upscale routes.
- Improved error handling in colorize, enhance_faces, remove_bg, restore, and upscale scripts with detailed logging.
- Updated Dockerfile to align NCCL versions for compatibility.
- Introduced a new full tool audit script to test all tools for functionality and GPU usage.
- Created Playwright E2E tests for GPU-dependent tools to ensure proper functionality and performance.

packages/ai/python/colorize.py

@@ -188,18 +188,29 @@ def main():
                 if os.path.exists(DDCOLOR_MODEL_PATH):
                     result_bgr, method = colorize_ddcolor(img_bgr, intensity)
                 elif model_choice == "ddcolor":
-                    emit_progress(10, "DDColor model not found, using fallback")
+                    raise FileNotFoundError(f"DDColor model not found: {DDCOLOR_MODEL_PATH}")
             except Exception as e:
+                import traceback
+                print(f"[colorize] DDColor failed: {e}", file=sys.stderr, flush=True)
+                traceback.print_exc(file=sys.stderr)
                 if model_choice == "ddcolor":
-                    emit_progress(10, f"DDColor failed: {str(e)[:50]}")
+                    # User explicitly requested ddcolor — fail, don't degrade
+                    raise
                 result_bgr = None
 
-        # Try OpenCV fallback
+        # Try OpenCV fallback only in auto mode
         if result_bgr is None and model_choice in ("auto", "opencv"):
             try:
                 if os.path.exists(OPENCV_PROTO_PATH) and os.path.exists(OPENCV_MODEL_PATH):
                     result_bgr, method = colorize_opencv(img_bgr, intensity)
-            except Exception:
+                elif model_choice == "opencv":
+                    raise FileNotFoundError(f"OpenCV colorize models not found: {OPENCV_PROTO_PATH}")
+            except Exception as e:
+                import traceback
+                print(f"[colorize] OpenCV fallback failed: {e}", file=sys.stderr, flush=True)
+                traceback.print_exc(file=sys.stderr)
+                if model_choice == "opencv":
+                    raise
                 result_bgr = None
 
         if result_bgr is None:

packages/ai/python/dispatcher.py

@@ -32,8 +32,8 @@ available_modules = {}
 def _try_import(name, import_fn):
     try:
         available_modules[name] = import_fn()
-    except ImportError:
-        pass
+    except ImportError as e:
+        print(f"[dispatcher] Module '{name}' not available: {e}", file=sys.stderr, flush=True)
 
 
 _try_import("PIL", lambda: __import__("PIL"))
@@ -94,6 +94,8 @@ def _run_script_main(script_name, args):
     except SystemExit as e:
         exit_code = e.code if isinstance(e.code, int) else 1
     except Exception as e:
+        # Log full traceback to stderr for diagnostics
+        traceback.print_exc(file=sys.stderr)
         # Write error to the captured stdout
         sys.stdout.write(json.dumps({"success": False, "error": str(e)}) + "\n")
         sys.stdout.flush()
@@ -129,9 +131,10 @@ def main():
     try:
         from gpu import gpu_available
         gpu = gpu_available()
-    except ImportError:
-        pass
+    except ImportError as e:
+        print(f"[dispatcher] GPU detection failed: {e}", file=sys.stderr, flush=True)
     print(json.dumps({"ready": True, "gpu": gpu}), file=sys.stderr, flush=True)
+    print(f"[dispatcher] Ready. GPU: {gpu}. Modules: {list(available_modules.keys())}", file=sys.stderr, flush=True)
 
     for line in sys.stdin:
         line = line.strip()

packages/ai/python/enhance_faces.py

@@ -17,8 +17,8 @@ except (ImportError, ModuleNotFoundError):
         _shim = types.ModuleType("torchvision.transforms.functional_tensor")
         _shim.rgb_to_grayscale = _F.rgb_to_grayscale
         sys.modules["torchvision.transforms.functional_tensor"] = _shim
-    except ImportError:
-        pass  # torchvision not installed at all
+    except ImportError as e:
+        print(f"[enhance-faces] torchvision shim failed: {e}", file=sys.stderr, flush=True)
 
 
 def emit_progress(percent, stage):
@@ -196,6 +196,9 @@ def enhance_with_codeformer(img_array, fidelity_weight):
     finally:
         torch.cuda.is_available = _orig_cuda_check
 
+    if restored_bgr is None:
+        raise RuntimeError("CodeFormer returned no result (face detection may have failed)")
+
     restored_rgb = restored_bgr[:, :, ::-1].copy()
     return restored_rgb
 
@@ -258,7 +261,9 @@ def main():
         # progress and init messages to stdout which would corrupt
         # our JSON result.
         stdout_fd = os.dup(1)
+        sys.stdout.flush()  # Flush before redirect to avoid mixing buffers
         os.dup2(2, 1)
+        sys.stdout = os.fdopen(1, "w", closefd=False)  # Rebind sys.stdout to new fd 1
 
         enhanced = None
         model_used = None
@@ -281,14 +286,19 @@ def main():
                     fidelity_weight = 1.0 - strength
                     enhanced = enhance_with_codeformer(img_array, fidelity_weight)
                     model_used = "codeformer"
-                except Exception:
+                except Exception as e:
+                    import traceback
+                    print(f"[enhance-faces] CodeFormer failed, falling back to GFPGAN: {e}", file=sys.stderr, flush=True)
+                    traceback.print_exc(file=sys.stderr)
                     enhanced = enhance_with_gfpgan(img_array, only_center_face)
                     model_used = "gfpgan"
 
         finally:
             # Restore stdout after ALL AI processing
+            sys.stdout.flush()
             os.dup2(stdout_fd, 1)
             os.close(stdout_fd)
+            sys.stdout = sys.__stdout__  # Restore Python-level stdout
 
         if enhanced is None:
             raise RuntimeError("Face enhancement failed: no model available")

packages/ai/python/gpu.py

@@ -1,6 +1,8 @@
 """Runtime GPU/CUDA detection utility."""
+import ctypes
 import functools
 import os
+import sys
 
 
 @functools.lru_cache(maxsize=1)
@@ -11,16 +13,36 @@ def gpu_available():
     if override is not None and override.lower() in ("0", "false", "no"):
         return False
 
-    # Use torch.cuda as the source of truth. It actually probes
-    # the hardware. onnxruntime's get_available_providers() only
-    # reports compiled-in backends, not whether a GPU exists.
+    # Use torch.cuda as the source of truth when available. It actually
+    # probes the hardware. Fall back to onnxruntime provider detection
+    # when torch is not installed (e.g. CPU-only images without PyTorch).
     try:
         import torch
-        return torch.cuda.is_available()
-    except ImportError:
-        pass
+        avail = torch.cuda.is_available()
+        if avail:
+            name = torch.cuda.get_device_name(0)
+            print(f"[gpu] CUDA available via torch: {name}", file=sys.stderr, flush=True)
+        else:
+            print("[gpu] torch loaded but CUDA not available", file=sys.stderr, flush=True)
+        return avail
+    except ImportError as e:
+        print(f"[gpu] torch not importable: {e}", file=sys.stderr, flush=True)
 
-    return False
+    # Fallback: check if onnxruntime's CUDA provider can actually load.
+    # get_available_providers() only reports *compiled-in* backends, not whether
+    # the required libraries (cuDNN, etc.) are present at runtime.  We verify
+    # by trying to load the provider shared library — this transitively checks
+    # that cuDNN is installed.
+    try:
+        import onnxruntime as _ort
+        if "CUDAExecutionProvider" not in _ort.get_available_providers():
+            return False
+        ep_dir = os.path.dirname(_ort.__file__)
+        ctypes.CDLL(os.path.join(ep_dir, "capi", "libonnxruntime_providers_cuda.so"))
+        return True
+    except (ImportError, OSError) as e:
+        print(f"[gpu] ONNX CUDA provider not functional: {e}", file=sys.stderr, flush=True)
+        return False
 
 
 def onnx_providers():

packages/ai/python/remove_bg.py

@@ -93,7 +93,8 @@ def main():
                 alpha_matting_foreground_threshold=240,
                 alpha_matting_background_threshold=10,
             )
-        except Exception:
+        except Exception as e:
+            print(f"[remove-bg] Alpha matting failed ({e}), using standard removal", file=sys.stderr, flush=True)
             output_data = remove(input_data, session=session)
 
         emit_progress(80, "Background removed")
@@ -107,11 +108,12 @@ def main():
 
         result = json.dumps({"success": True, "model": model})
 
-    except ImportError:
+    except ImportError as e:
+        print(f"[remove-bg] Import failed: {e}", file=sys.stderr, flush=True)
         result = json.dumps(
             {
                 "success": False,
-                "error": "rembg is not installed. Install with: pip install rembg[cpu]",
+                "error": f"rembg import failed: {e}",
             }
         )
     except Exception as e:

packages/ai/python/restore.py

@@ -368,7 +368,8 @@ def enhance_faces(img_bgr, fidelity=0.7):
         # Run inference
         try:
             output = session.run(None, model_inputs)[0][0]  # (3, 512, 512)
-        except Exception:
+        except Exception as e:
+            print(f"[restore] CodeFormer inference failed for face {i}: {e}", file=sys.stderr, flush=True)
             continue
 
         # Postprocess: [-1, 1] -> [0, 255], RGB -> BGR
@@ -478,8 +479,8 @@ def colorize_bw(img_bgr, intensity=0.85):
         from gpu import gpu_available
         if gpu_available():
             providers.insert(0, "CUDAExecutionProvider")
-    except ImportError:
-        pass
+    except ImportError as e:
+        print(f"[restore] GPU detection unavailable: {e}", file=sys.stderr, flush=True)
 
     session = ort.InferenceSession(DDCOLOR_MODEL_PATH, providers=providers)
     input_name = session.get_inputs()[0].name

packages/ai/python/upscale.py

@@ -17,8 +17,8 @@ except (ImportError, ModuleNotFoundError):
         _shim = types.ModuleType("torchvision.transforms.functional_tensor")
         _shim.rgb_to_grayscale = _F.rgb_to_grayscale
         sys.modules["torchvision.transforms.functional_tensor"] = _shim
-    except ImportError:
-        pass  # torchvision not installed at all, Real-ESRGAN unavailable
+    except ImportError as e:
+        print(f"[upscale] torchvision shim failed: {e}", file=sys.stderr, flush=True)
 
 
 def emit_progress(percent, stage):
@@ -167,14 +167,19 @@ def main():
                     os.dup2(stdout_fd, 1)
                     os.close(stdout_fd)
 
-            except (ImportError, FileNotFoundError, RuntimeError, OSError):
-                # RealESRGAN unavailable or failed
+            except (ImportError, FileNotFoundError, RuntimeError, OSError) as e:
+                import traceback
+                print(f"[upscale] Real-ESRGAN failed: {e}", file=sys.stderr, flush=True)
+                traceback.print_exc(file=sys.stderr)
                 if model_choice == "realesrgan":
-                    emit_progress(15, "AI model not available, using fast resize")
+                    # User explicitly requested realesrgan — fail, don't degrade
+                    raise RuntimeError(f"Real-ESRGAN unavailable: {e}") from e
                 result = None
 
-        # Fall back to Lanczos
+        # Lanczos path: used when explicitly requested or as auto fallback
         if result is None:
+            if model_choice not in ("auto", "lanczos"):
+                raise RuntimeError(f"Requested model '{model_choice}' is not available")
             emit_progress(50, "Upscaling with Lanczos")
             result = img.resize(new_size, Image.LANCZOS)
             method = "lanczos"