feat(telemetry): readable Sentry errors, Python tracebacks, and diagnostic mode

Keeps a real, redacted error message instead of "Error: Error", surfaces Python tracebacks in Sentry as a vetted context, and adds an opt-in SNAPOTTER_SENTRY_DIAGNOSTIC verbose mode plus SNAPOTTER_SENTRY_DSN_OVERRIDE. The default fleet path ships nothing on the never-collect list; raw detail is reachable only via the opt-in flag. Also classifies Redis OOM/READONLY replies as operational and removes a ReDoS in stack-frame extraction.

packages/ai/python/dispatcher.py

@@ -27,6 +27,8 @@ import io
 import os
 import traceback
 
+from sidecar_errors import build_error_envelope
+
 
 # ── Optional OpenTelemetry tracing (enterprise only) ─────────────
 _tracer = None
@@ -294,10 +296,14 @@ def _run_script_main(script_name, args):
     except SystemExit as e:
         exit_code = e.code if isinstance(e.code, int) else 1
     except Exception as e:
-        # Log full traceback to stderr for diagnostics
+        # Log full traceback to stderr for local diagnostics (unchanged).
         traceback.print_exc(file=sys.stderr)
-        # Write error to the captured stdout
-        sys.stdout.write(json.dumps({"success": False, "error": str(e)}) + "\n")
+        info = build_error_envelope(e)
+        # Keep `error` as the redacted string for back-compatible consumers;
+        # add `errorInfo` (type + our frames) for the structured Sentry path.
+        sys.stdout.write(
+            json.dumps({"success": False, "error": info["message"], "errorInfo": info}) + "\n"
+        )
         sys.stdout.flush()
         exit_code = 1
     finally:

packages/ai/python/sidecar_errors.py

@@ -0,0 +1,78 @@
+"""
+Structured error envelope for the sidecar. Mirrors the Node redactMessage so a
+Python failure reaches Sentry with its type, a redacted message, and our own
+stack frames (basename only) instead of a bare "Error: Error".
+"""
+import os
+import re
+import traceback
+
+# _CTRL: matches ASCII control chars 0x00-0x1F and 0x7F only. Written with \x
+# hex escapes so no literal control bytes appear in this file. It must NOT match
+# spaces, "/", ".", or any printable character.
+_CTRL = re.compile(r"[\x00-\x1f\x7f]")
+_BLOB = re.compile(r"blob:[^\s\"')]+")
+_DATA = re.compile(r"data:[^\s\"')]+")
+_URL = re.compile(r"https?://[^\s\"')]+")
+_PATH = re.compile(r"(?:/(?:Users|home|root|data|tmp|var|app|opt|mnt|srv)|[A-Za-z]:\\)[^\s\"')]*")
+# Relative object-storage keys (uploads/<jobId>/…, outputs/…, previews/…) carry a
+# user-supplied filename tail; mask them like absolute paths. Runs after _PATH,
+# which already swallows the absolute /data/uploads/… form.
+_RELKEY = re.compile(r"\b(?:uploads|outputs|previews)/[^\s\"')]+")
+_IP = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b")
+# IPv6: a full 8-group form, or any ::-compressed form (::1, fe80::…, …::). The
+# negative lookbehind/lookahead ((?<![\w:]) … (?![\w:])) require the address to
+# stand alone, so C++/Rust scope resolution (std::bad_alloc, core::result) is left
+# intact. A plain decimal version like 2.2.0 has no colons, and a bare HH:MM needs
+# no ::, so both survive too.
+_IPV6 = re.compile(
+    r"(?<![\w:])(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|"
+    r"(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?::(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?)(?![\w:])"
+)
+_EMAIL = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+_USER_FILE_EXT = (
+    "jpe?g|png|gif|webp|avif|heif?|tiff?|bmp|svg|raw|psd|mp4|mov|avi|mkv|webm|"
+    "flv|wmv|m4v|mp3|wav|flac|aac|ogg|m4a|opus|pdf|docx?|xlsx?|pptx?|odt|ods|"
+    "odp|txt|csv|epub|zip"
+)
+_FILE = re.compile(r"\b[\w-]{1,80}\.(?:" + _USER_FILE_EXT + r")\b", re.IGNORECASE)
+_QUOTED = re.compile(r"(['\"])(.{24,}?)\1")
+_HEX = re.compile(r"\b[0-9a-fA-F]{16,}\b")
+_MAX_LEN = 300
+_SIDECAR_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def redact(message):
+    s = _CTRL.sub(" ", str(message or ""))
+    s = _BLOB.sub("<blob>", s)
+    s = _DATA.sub("<data>", s)
+    s = _URL.sub("<url>", s)
+    s = _PATH.sub("<path>", s)
+    s = _RELKEY.sub("<path>", s)
+    s = _IP.sub("<ip>", s)
+    s = _IPV6.sub("<ip>", s)
+    s = _EMAIL.sub("<email>", s)
+    s = _QUOTED.sub(lambda m: m.group(1) + "<value>" + m.group(1), s)
+    s = _HEX.sub("<hex>", s)
+    s = _FILE.sub("<file>", s)
+    s = re.sub(r"\s+", " ", s).strip()
+    return (s[:_MAX_LEN] + "…") if len(s) > _MAX_LEN else s
+
+
+def _our_frames(exc):
+    frames = []
+    for fr in traceback.extract_tb(exc.__traceback__):
+        # Keep only our sidecar-script frames; drop stdlib and venv/site-packages.
+        if os.path.dirname(os.path.abspath(fr.filename)) != _SIDECAR_DIR:
+            continue
+        frames.append({"file": os.path.basename(fr.filename), "line": fr.lineno, "func": fr.name})
+    return frames[-20:]
+
+
+def build_error_envelope(exc):
+    """A JSON-serializable {type, message, frames} describing a caught exception."""
+    return {
+        "type": type(exc).__name__,
+        "message": redact(str(exc)),
+        "frames": _our_frames(exc),
+    }

packages/ai/python/test_sidecar_errors.py

@@ -0,0 +1,31 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from sidecar_errors import build_error_envelope, redact  # noqa: E402
+
+
+def test_redact_masks_paths_and_files():
+    assert redact("open /data/uploads/9f/in.bin") == "open <path>"
+    assert redact("cannot read family_photo.JPG") == "cannot read <file>"
+    assert redact("torch 2.2.0 ok") == "torch 2.2.0 ok"
+
+
+def test_envelope_shape_and_frames():
+    try:
+        raise RuntimeError("CUDA out of memory for /data/x.png")
+    except RuntimeError as exc:
+        env = build_error_envelope(exc)
+    assert env["type"] == "RuntimeError"
+    assert env["message"] == "CUDA out of memory for <path>"
+    assert isinstance(env["frames"], list) and len(env["frames"]) >= 1
+    top = env["frames"][-1]
+    assert top["file"] == "test_sidecar_errors.py"
+    assert isinstance(top["line"], int)
+    assert top["func"] == "test_envelope_shape_and_frames"
+
+
+if __name__ == "__main__":
+    test_redact_masks_paths_and_files()
+    test_envelope_shape_and_frames()
+    print("ok")

packages/ai/src/bridge.ts

@@ -108,7 +108,12 @@ const OOM_EXIT_TEXT =
  * memory-aware callers still retry on a lighter model. Any other non-zero exit
  * is a bug, with the extracted reason (or an OOM reported in-band) preserved.
  */
-function pythonExitError(code: number | null, signal: string | null, extracted: string): SafeError {
+function pythonExitError(
+  code: number | null,
+  signal: string | null,
+  extracted: string,
+  info?: PythonErrorInfo | null,
+): SafeError {
   if (signal === "SIGSEGV" || code === 139) {
     return new SafeError("Process crashed (segmentation fault)", {
       kind: "operational",
@@ -122,10 +127,14 @@ function pythonExitError(code: number | null, signal: string | null, extracted:
     });
   }
   const message = extracted || `Python script exited with code ${code}`;
-  return new SafeError(message, {
+  const err = new SafeError(message, {
     kind: OOM_EXIT_TEXT.test(message) ? "operational" : "bug",
     code: `exit-${code ?? "unknown"}`,
   });
+  if (info) {
+    Object.assign(err, { pythonType: info.type, pythonFrames: info.frames });
+  }
+  return err;
 }
 
 /**
@@ -186,6 +195,41 @@ function extractPythonError(error: unknown): string {
   return String(error);
 }
 
+export interface PythonErrorInfo {
+  type?: string;
+  frames: Array<{ file: string; line: number; func: string }>;
+}
+
+/** Read the structured {type, frames} envelope from a sidecar failure, or null. */
+export function extractPythonErrorInfo(error: unknown): PythonErrorInfo | null {
+  if (!error || typeof error !== "object") return null;
+  const pErr = error as { stdout?: string; stderr?: string };
+  for (const output of [pErr.stdout, pErr.stderr]) {
+    if (!output) continue;
+    try {
+      const parsed = JSON.parse(output.trim()) as { errorInfo?: unknown };
+      const info = parsed.errorInfo as PythonErrorInfo | undefined;
+      if (info && Array.isArray(info.frames)) {
+        return {
+          type: typeof info.type === "string" ? info.type : undefined,
+          frames: info.frames
+            .filter(
+              (f): f is { file: string; line: number; func: string } =>
+                !!f &&
+                typeof (f as { file?: unknown }).file === "string" &&
+                typeof (f as { line?: unknown }).line === "number" &&
+                typeof (f as { func?: unknown }).func === "string",
+            )
+            .slice(0, 20),
+        };
+      }
+    } catch {
+      // not JSON; nothing structured to read
+    }
+  }
+  return null;
+}
+
 export type ProgressCallback = (percent: number, stage: string) => void;
 
 export interface PythonRunOptions {
@@ -420,11 +464,11 @@ export class PythonDispatcher {
             if (req) {
               this.pending.delete(reqId);
               if (response.exitCode !== 0) {
-                const extracted = extractPythonError({
-                  stdout: response.stdout,
-                  stderr: req.stderrLines.join("\n"),
-                });
-                req.reject(pythonExitError(response.exitCode, null, extracted));
+                const raw = { stdout: response.stdout, stderr: req.stderrLines.join("\n") };
+                const extracted = extractPythonError(raw);
+                req.reject(
+                  pythonExitError(response.exitCode, null, extracted, extractPythonErrorInfo(raw)),
+                );
               } else {
                 req.resolve({
                   stdout: response.stdout || "",
@@ -737,8 +781,9 @@ export class PythonDispatcher {
           const stderr = stderrLines.join("\n");
 
           if (code !== 0) {
+            const raw = { stdout: stdout.trim(), stderr };
             rejectOnce(
-              pythonExitError(code, signal, extractPythonError({ stdout: stdout.trim(), stderr })),
+              pythonExitError(code, signal, extractPythonError(raw), extractPythonErrorInfo(raw)),
             );
             return;
           }

packages/shared/src/analytics/error-sanitize.ts

@@ -5,6 +5,7 @@
  * author). Returning null means "no safe rebuild known, use type-only".
  */
 import { isSafeMessageError } from "../tool-errors.js";
+import { redactMessage } from "./redact-message.js";
 
 interface ErrLike {
   name?: unknown;
@@ -56,13 +57,39 @@ function looksLikePg(links: ErrLike[]): boolean {
   );
 }
 
+/** "at file.ts:NN" from the first in-app stack frame, mirroring errorSignature. */
+function frameHint(err: unknown): string | null {
+  const stack = (err as { stack?: unknown } | null)?.stack;
+  if (typeof stack !== "string") return null;
+  const line = stack.split("\n").find((l) => l.includes("/apps/") || l.includes("/packages/"));
+  const m = line?.slice(0, 300).match(/([^/\\\s():]+):(\d+)/);
+  return m ? `at ${m[1]}:${m[2]}` : null;
+}
+
 /** Safe replacement for exception.value, or null for type-only fallback. */
 export function rebuildErrorValue(err: unknown): string | null {
   try {
-    if (isSafeMessageError(err)) return err.message;
     const links = chain(err);
-    if (links.length === 0) return null;
 
+    // 1. SafeError: our authored (or toSidecarError-wrapped) message, redacted,
+    //    plus the redacted immediate cause so a wrapper title never hides detail.
+    if (isSafeMessageError(err)) {
+      let out = redactMessage(err.message);
+      for (let i = 1; i < links.length; i++) {
+        const m = links[i].message;
+        if (typeof m === "string" && m.trim()) {
+          const detail = redactMessage(m);
+          if (detail && !out.includes(detail)) out = `${out}: ${detail}`;
+          break;
+        }
+      }
+      return out;
+    }
+
+    if (links.length === 0) return null;
+    const top = links[0];
+
+    // 2. Structured rebuilds: pg SQLSTATE, node E-code, reply, zod, http status.
     for (const l of links) {
       if (typeof l.code === "string" && SQLSTATE.test(l.code) && !NODE_CODE.test(l.code)) {
         return typeof l.routine === "string" && PG_ROUTINE.test(l.routine)
@@ -72,11 +99,9 @@ export function rebuildErrorValue(err: unknown): string | null {
     }
     for (const l of links) {
       if (typeof l.code === "string" && NODE_CODE.test(l.code)) {
-        // syscall is libuv vocabulary or "spawn <server-binary>", never user args.
         return typeof l.syscall === "string" ? `${l.code} ${l.syscall}` : l.code;
       }
     }
-    const top = links[0];
     if (top.name === "ReplyError" && typeof top.message === "string") {
       const token = top.message.split(" ")[0];
       return REPLY_TOKEN.test(token) ? `reply ${token}` : "reply";
@@ -96,6 +121,17 @@ export function rebuildErrorValue(err: unknown): string | null {
         typeof top.name === "string" && SAFE_NAME.test(top.name) ? top.name : "HttpError";
       return `${name} ${top.status}`;
     }
+
+    // 3. Non-empty message: keep it, redacted (the rich default).
+    if (typeof top.message === "string" && top.message.trim().length > 0) {
+      return redactMessage(top.message);
+    }
+
+    // 4. Empty message with a stack: derive a title from the first in-app frame.
+    const hint = frameHint(err);
+    if (hint) return hint;
+
+    // 5. Nothing safe to surface: type-only.
     return null;
   } catch {
     return null;

packages/shared/src/analytics/redact-message.ts

@@ -0,0 +1,56 @@
+/**
+ * The single redactor for error text sent to Sentry. Denylist-shaped, so it is
+ * only trusted for messages we already gate (SafeError, our own throws, known
+ * libraries) and for the redacted fallback in rebuildErrorValue. It keeps the
+ * published never-collect promise (no file names / paths / contents) true while
+ * still surfacing the human-readable message.
+ */
+// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally strip ASCII C0 controls + DEL from error text.
+const CTRL_RE = /[\x00-\x1f\x7f]/g;
+const BLOB_RE = /blob:[^\s"')]+/g;
+const DATA_RE = /data:[^\s"')]+/g;
+const URL_RE = /https?:\/\/[^\s"')]+/g;
+const PATH_RE = /(?:\/(?:Users|home|root|data|tmp|var|app|opt|mnt|srv)|[A-Za-z]:\\)[^\s"')]*/g;
+// Relative object-storage keys (uploads/<jobId>/…, outputs/…, previews/…) carry a
+// user-supplied filename tail, so mask them like absolute paths. Runs after
+// PATH_RE, which already swallows the absolute /data/uploads/… form.
+const RELKEY_RE = /\b(?:uploads|outputs|previews)\/[^\s"')]+/g;
+const IP_RE = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g;
+// IPv6: a full 8-group form, or any ::-compressed form (::1, fe80::…, …::). The
+// negative lookbehind/lookahead ((?<![\w:]) … (?![\w:])) require the address to
+// stand alone, so C++/Rust scope resolution (std::bad_alloc, core::result) is left
+// intact. A plain decimal version like 2.2.0 has no colons, and a bare HH:MM needs
+// no ::, so both survive too.
+const IPV6_RE =
+  /(?<![\w:])(?:(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?::(?:[0-9a-fA-F]{1,4}(?::[0-9a-fA-F]{1,4})*)?)(?![\w:])/g;
+const EMAIL_RE = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;
+// A user file is a name plus one of the formats SnapOtter processes. Restricting
+// to this set avoids eating version strings ("2.2.0") and code filenames
+// ("rounded-crop.ts"), which carry no user data and aid triage. Unicode-aware
+// (\p{L} + the u flag) so a non-ASCII user filename (CJK, Arabic) is masked too.
+const USER_FILE_EXT =
+  "jpe?g|png|gif|webp|avif|heif?|tiff?|bmp|svg|raw|psd|mp4|mov|avi|mkv|webm|flv|wmv|m4v|mp3|wav|flac|aac|ogg|m4a|opus|pdf|docx?|xlsx?|pptx?|odt|ods|odp|txt|csv|epub|zip";
+const FILE_RE = new RegExp(`[\\p{L}\\p{N}_-]{1,80}\\.(?:${USER_FILE_EXT})`, "giu");
+const QUOTED_RE = /(['"])(.{24,}?)\1/g;
+const HEX_RE = /\b[0-9a-fA-F]{16,}\b/g;
+const MAX_LEN = 300;
+
+export function redactMessage(message: unknown, opts?: { raw?: boolean }): string {
+  let s = String(message ?? "").replace(CTRL_RE, " ");
+  if (!opts?.raw) {
+    s = s
+      .replace(BLOB_RE, "<blob>")
+      .replace(DATA_RE, "<data>")
+      .replace(URL_RE, "<url>")
+      .replace(PATH_RE, "<path>")
+      .replace(RELKEY_RE, "<path>")
+      .replace(IP_RE, "<ip>")
+      .replace(IPV6_RE, "<ip>")
+      .replace(EMAIL_RE, "<email>")
+      .replace(QUOTED_RE, (_m, q) => `${q}<value>${q}`)
+      .replace(HEX_RE, "<hex>")
+      .replace(FILE_RE, "<file>");
+  }
+  s = s.replace(/\s+/g, " ").trim();
+  return s.length > MAX_LEN ? `${s.slice(0, MAX_LEN)}…` : s;
+}

packages/shared/src/index.ts

@@ -2,6 +2,7 @@ export * from "./analytics/baked.js";
 export * from "./analytics/error-sanitize.js";
 export * from "./analytics/events.js";
 export * from "./analytics/feedback.js";
+export { redactMessage } from "./analytics/redact-message.js";
 export * from "./analytics/types.js";
 export * from "./audit-events.js";
 export * from "./constants.js";