feat(telemetry): readable Sentry errors, Python tracebacks, and diagnostic mode

Keeps a real, redacted error message instead of "Error: Error", surfaces Python tracebacks in Sentry as a vetted context, and adds an opt-in SNAPOTTER_SENTRY_DIAGNOSTIC verbose mode plus SNAPOTTER_SENTRY_DSN_OVERRIDE. The default fleet path ships nothing on the never-collect list; raw detail is reachable only via the opt-in flag. Also classifies Redis OOM/READONLY replies as operational and removes a ReDoS in stack-frame extraction.

packages/ai/python/dispatcher.py

@@ -27,6 +27,8 @@ import io
 import os
 import traceback
 
+from sidecar_errors import build_error_envelope
+
 
 # ── Optional OpenTelemetry tracing (enterprise only) ─────────────
 _tracer = None
@@ -294,10 +296,14 @@ def _run_script_main(script_name, args):
     except SystemExit as e:
         exit_code = e.code if isinstance(e.code, int) else 1
     except Exception as e:
-        # Log full traceback to stderr for diagnostics
+        # Log full traceback to stderr for local diagnostics (unchanged).
         traceback.print_exc(file=sys.stderr)
-        # Write error to the captured stdout
-        sys.stdout.write(json.dumps({"success": False, "error": str(e)}) + "\n")
+        info = build_error_envelope(e)
+        # Keep `error` as the redacted string for back-compatible consumers;
+        # add `errorInfo` (type + our frames) for the structured Sentry path.
+        sys.stdout.write(
+            json.dumps({"success": False, "error": info["message"], "errorInfo": info}) + "\n"
+        )
         sys.stdout.flush()
         exit_code = 1
     finally:

packages/ai/python/sidecar_errors.py

@@ -0,0 +1,78 @@
+"""
+Structured error envelope for the sidecar. Mirrors the Node redactMessage so a
+Python failure reaches Sentry with its type, a redacted message, and our own
+stack frames (basename only) instead of a bare "Error: Error".
+"""
+import os
+import re
+import traceback
+
+# _CTRL: matches ASCII control chars 0x00-0x1F and 0x7F only. Written with \x
+# hex escapes so no literal control bytes appear in this file. It must NOT match
+# spaces, "/", ".", or any printable character.
+_CTRL = re.compile(r"[\x00-\x1f\x7f]")
+_BLOB = re.compile(r"blob:[^\s\"')]+")
+_DATA = re.compile(r"data:[^\s\"')]+")
+_URL = re.compile(r"https?://[^\s\"')]+")
+_PATH = re.compile(r"(?:/(?:Users|home|root|data|tmp|var|app|opt|mnt|srv)|[A-Za-z]:\\)[^\s\"')]*")
+# Relative object-storage keys (uploads/<jobId>/…, outputs/…, previews/…) carry a
+# user-supplied filename tail; mask them like absolute paths. Runs after _PATH,
+# which already swallows the absolute /data/uploads/… form.
+_RELKEY = re.compile(r"\b(?:uploads|outputs|previews)/[^\s\"')]+")
+_IP = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b")
+# IPv6: a full 8-group form, or any ::-compressed form (::1, fe80::…, …::). The
+# negative lookbehind/lookahead ((?<![\w:]) … (?![\w:])) require the address to
+# stand alone, so C++/Rust scope resolution (std::bad_alloc, core::result) is left
+# intact. A plain decimal version like 2.2.0 has no colons, and a bare HH:MM needs
+# no ::, so both survive too.
+_IPV6 = re.compile(
+    r"(?<![\w:])(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|"
+    r"(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?::(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?)(?![\w:])"
+)
+_EMAIL = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_USER_FILE_EXT = (
+    "jpe?g|png|gif|webp|avif|heif?|tiff?|bmp|svg|raw|psd|mp4|mov|avi|mkv|webm|"
+    "flv|wmv|m4v|mp3|wav|flac|aac|ogg|m4a|opus|pdf|docx?|xlsx?|pptx?|odt|ods|"
+    "odp|txt|csv|epub|zip"
+)
+_FILE = re.compile(r"\b[\w-]{1,80}\.(?:" + _USER_FILE_EXT + r")\b", re.IGNORECASE)
+_QUOTED = re.compile(r"(['\"])(.{24,}?)\1")
+_HEX = re.compile(r"\b[0-9a-fA-F]{16,}\b")
+_MAX_LEN = 300
+_SIDECAR_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def redact(message):
+    s = _CTRL.sub(" ", str(message or ""))
+    s = _BLOB.sub("<blob>", s)
+    s = _DATA.sub("<data>", s)
+    s = _URL.sub("<url>", s)
+    s = _PATH.sub("<path>", s)
+    s = _RELKEY.sub("<path>", s)
+    s = _IP.sub("<ip>", s)
+    s = _IPV6.sub("<ip>", s)
+    s = _EMAIL.sub("<email>", s)
+    s = _QUOTED.sub(lambda m: m.group(1) + "<value>" + m.group(1), s)
+    s = _HEX.sub("<hex>", s)
+    s = _FILE.sub("<file>", s)
+    s = re.sub(r"\s+", " ", s).strip()
+    return (s[:_MAX_LEN] + "…") if len(s) > _MAX_LEN else s
+
+
+def _our_frames(exc):
+    frames = []
+    for fr in traceback.extract_tb(exc.__traceback__):
+        # Keep only our sidecar-script frames; drop stdlib and venv/site-packages.
+        if os.path.dirname(os.path.abspath(fr.filename)) != _SIDECAR_DIR:
+            continue
+        frames.append({"file": os.path.basename(fr.filename), "line": fr.lineno, "func": fr.name})
+    return frames[-20:]
+
+
+def build_error_envelope(exc):
+    """A JSON-serializable {type, message, frames} describing a caught exception."""
+    return {
+        "type": type(exc).__name__,
+        "message": redact(str(exc)),
+        "frames": _our_frames(exc),
+    }

packages/ai/python/test_sidecar_errors.py

@@ -0,0 +1,31 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from sidecar_errors import build_error_envelope, redact  # noqa: E402
+
+
+def test_redact_masks_paths_and_files():
+    assert redact("open /data/uploads/9f/in.bin") == "open <path>"
+    assert redact("cannot read family_photo.JPG") == "cannot read <file>"
+    assert redact("torch 2.2.0 ok") == "torch 2.2.0 ok"
+
+
+def test_envelope_shape_and_frames():
+    try:
+        raise RuntimeError("CUDA out of memory for /data/x.png")
+    except RuntimeError as exc:
+        env = build_error_envelope(exc)
+    assert env["type"] == "RuntimeError"
+    assert env["message"] == "CUDA out of memory for <path>"
+    assert isinstance(env["frames"], list) and len(env["frames"]) >= 1
+    top = env["frames"][-1]
+    assert top["file"] == "test_sidecar_errors.py"
+    assert isinstance(top["line"], int)
+    assert top["func"] == "test_envelope_shape_and_frames"
+
+
+if __name__ == "__main__":
+    test_redact_masks_paths_and_files()
+    test_envelope_shape_and_frames()
+    print("ok")

packages/ai/src/bridge.ts

@@ -108,7 +108,12 @@ const OOM_EXIT_TEXT =
  * memory-aware callers still retry on a lighter model. Any other non-zero exit
  * is a bug, with the extracted reason (or an OOM reported in-band) preserved.
  */
-function pythonExitError(code: number | null, signal: string | null, extracted: string): SafeError {
+function pythonExitError(
+  code: number | null,
+  signal: string | null,
+  extracted: string,
+  info?: PythonErrorInfo | null,
+): SafeError {
   if (signal === "SIGSEGV" || code === 139) {
     return new SafeError("Process crashed (segmentation fault)", {
       kind: "operational",
@@ -122,10 +127,14 @@ function pythonExitError(code: number | null, signal: string | null, extracted:
     });
   }
   const message = extracted || `Python script exited with code ${code}`;
-  return new SafeError(message, {
+  const err = new SafeError(message, {
     kind: OOM_EXIT_TEXT.test(message) ? "operational" : "bug",
     code: `exit-${code ?? "unknown"}`,
   });
+  if (info) {
+    Object.assign(err, { pythonType: info.type, pythonFrames: info.frames });
+  }
+  return err;
 }
 
 /**
@@ -186,6 +195,41 @@ function extractPythonError(error: unknown): string {
   return String(error);
 }
 
+export interface PythonErrorInfo {
+  type?: string;
+  frames: Array<{ file: string; line: number; func: string }>;
+}
+
+/** Read the structured {type, frames} envelope from a sidecar failure, or null. */
+export function extractPythonErrorInfo(error: unknown): PythonErrorInfo | null {
+  if (!error || typeof error !== "object") return null;
+  const pErr = error as { stdout?: string; stderr?: string };
+  for (const output of [pErr.stdout, pErr.stderr]) {
+    if (!output) continue;
+    try {
+      const parsed = JSON.parse(output.trim()) as { errorInfo?: unknown };
+      const info = parsed.errorInfo as PythonErrorInfo | undefined;
+      if (info && Array.isArray(info.frames)) {
+        return {
+          type: typeof info.type === "string" ? info.type : undefined,
+          frames: info.frames
+            .filter(
+              (f): f is { file: string; line: number; func: string } =>
+                !!f &&
+                typeof (f as { file?: unknown }).file === "string" &&
+                typeof (f as { line?: unknown }).line === "number" &&
+                typeof (f as { func?: unknown }).func === "string",
+            )
+            .slice(0, 20),
+        };
+      }
+    } catch {
+      // not JSON; nothing structured to read
+    }
+  }
+  return null;
+}
+
 export type ProgressCallback = (percent: number, stage: string) => void;
 
 export interface PythonRunOptions {
@@ -420,11 +464,11 @@ export class PythonDispatcher {
             if (req) {
               this.pending.delete(reqId);
               if (response.exitCode !== 0) {
-                const extracted = extractPythonError({
-                  stdout: response.stdout,
-                  stderr: req.stderrLines.join("\n"),
-                });
-                req.reject(pythonExitError(response.exitCode, null, extracted));
+                const raw = { stdout: response.stdout, stderr: req.stderrLines.join("\n") };
+                const extracted = extractPythonError(raw);
+                req.reject(
+                  pythonExitError(response.exitCode, null, extracted, extractPythonErrorInfo(raw)),
+                );
               } else {
                 req.resolve({
                   stdout: response.stdout || "",
@@ -737,8 +781,9 @@ export class PythonDispatcher {
           const stderr = stderrLines.join("\n");
 
           if (code !== 0) {
+            const raw = { stdout: stdout.trim(), stderr };
             rejectOnce(
-              pythonExitError(code, signal, extractPythonError({ stdout: stdout.trim(), stderr })),
+              pythonExitError(code, signal, extractPythonError(raw), extractPythonErrorInfo(raw)),
             );
             return;
           }