feat(telemetry): readable Sentry errors, Python tracebacks, and diagnostic mode

Keeps a real, redacted error message instead of "Error: Error", surfaces Python tracebacks in Sentry as a vetted context, and adds an opt-in SNAPOTTER_SENTRY_DIAGNOSTIC verbose mode plus SNAPOTTER_SENTRY_DSN_OVERRIDE. The default fleet path ships nothing on the never-collect list; raw detail is reachable only via the opt-in flag. Also classifies Redis OOM/READONLY replies as operational and removes a ReDoS in stack-frame extraction.

apps/api/src/instrument.ts

@@ -1,5 +1,10 @@
 import { ANALYTICS_BAKED } from "@snapotter/shared";
-import { analyticsEnabled, gatePrimed, telemetryEnvKilled } from "./lib/analytics-gate.js";
+import {
+  analyticsEnabled,
+  gatePrimed,
+  sentryDiagnostic,
+  telemetryEnvKilled,
+} from "./lib/analytics-gate.js";
 import { deployMode } from "./lib/deploy-mode.js";
 import { buildBeforeSend } from "./lib/sentry-scrub.js";
 import { buildTracesSampler } from "./lib/sentry-tracing.js";
@@ -9,7 +14,9 @@ import { buildTracesSampler } from "./lib/sentry-tracing.js";
 // so an opted-out instance never reports even a boot-window crash.
 const sentryActive = () => gatePrimed() && analyticsEnabled();
 
-if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
+const dsn = process.env.SNAPOTTER_SENTRY_DSN_OVERRIDE || ANALYTICS_BAKED.sentryDsn;
+
+if (dsn && !telemetryEnvKilled()) {
   try {
     const Sentry = await import("@sentry/node");
     const { APP_VERSION } = await import("@snapotter/shared");
@@ -31,7 +38,7 @@ if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
     const tracingEnabled = tracesSampleRate > 0 && tracesSampleRate <= 1;
 
     Sentry.init({
-      dsn: ANALYTICS_BAKED.sentryDsn,
+      dsn,
       release,
       environment: process.env.SNAPOTTER_ENV || "production",
       sendDefaultPii: false,
@@ -56,7 +63,10 @@ if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
       // Capture the breadcrumb trail (default 100). beforeSend (sentry-scrub.ts)
       // sanitizes each breadcrumb before send: urls/paths redacted, data dropped.
       initialScope: { tags: { deploy_mode: deployMode() } },
-      beforeSend: buildBeforeSend(sentryActive) as unknown as SentryOptions["beforeSend"],
+      beforeSend: buildBeforeSend(
+        sentryActive,
+        sentryDiagnostic(),
+      ) as unknown as SentryOptions["beforeSend"],
     });
 
     console.log(

apps/api/src/jobs/worker.ts

@@ -1119,6 +1119,10 @@ export function startWorkers(): void {
 
     worker.on("failed", (job, err) => {
       if (!job) return;
+      const pf = (err as { pythonFrames?: unknown }).pythonFrames;
+      if (Array.isArray(pf) && pf.length) {
+        logger.error({ pool, jobId: job.id, pythonFrames: pf }, "sidecar failure");
+      }
       void reportError(err, {
         source: "worker",
         pool,

apps/api/src/lib/analytics-gate.ts

@@ -38,6 +38,12 @@ export function telemetryEnvKilled(): boolean {
   return off(process.env.SNAPOTTER_TELEMETRY) || off(process.env.ANALYTICS_ENABLED);
 }
 
+/** Opt-in diagnostic verbosity for a single (usually self-hosted) instance. */
+export function sentryDiagnostic(): boolean {
+  const v = process.env.SNAPOTTER_SENTRY_DIAGNOSTIC;
+  return v === "1" || v === "true" || v === "on";
+}
+
 /** Compile-time bake, with a NON-PRODUCTION-only override so tests can force it on. */
 export function bakedEnabled(): boolean {
   if (telemetryEnvKilled()) return false;

apps/api/src/lib/error-report.ts

@@ -19,7 +19,7 @@ import {
   isSafeMessageError,
   isToolInputError,
 } from "@snapotter/shared";
-import { analyticsEnabled } from "./analytics-gate.js";
+import { analyticsEnabled, sentryDiagnostic } from "./analytics-gate.js";
 
 export type ErrorClass = "expected" | "operational" | "bug";
 
@@ -74,6 +74,13 @@ export function classifyError(err: unknown, source?: ReportContext["source"]): E
   if (connectivityClass(err)) return "operational";
   if (isEnvironmentalDbError(err)) return "operational";
   if (e?.code && OPERATIONAL_CODES.has(e.code)) return "operational";
+  if (
+    e?.name === "ReplyError" &&
+    typeof e.message === "string" &&
+    /^(OOM|READONLY|MISCONF|NOREPLICAS)\b/.test(e.message)
+  ) {
+    return "operational";
+  }
   return "bug";
 }
 
@@ -105,7 +112,7 @@ export function errorSignature(err: unknown): string {
   let frame = "-";
   if (typeof e?.stack === "string") {
     const line = e.stack.split("\n").find((l) => l.includes("/apps/") || l.includes("/packages/"));
-    const m = line?.match(/([^/\\]+\.[cm]?[jt]sx?):(\d+)/);
+    const m = line?.slice(0, 300).match(/([^/\\\s():]+):(\d+)/);
     if (m) frame = `${m[1]}:${m[2]}`;
   }
   return `${name}:${code}:${frame}`;
@@ -155,7 +162,7 @@ export async function reportError(err: unknown, ctx: ReportContext): Promise<voi
     if (!analyticsEnabled()) return;
     const cls = classifyError(err, ctx.source);
     if (cls === "expected") return;
-    if (!shouldReport(cls, errorSignature(err))) return;
+    if (!sentryDiagnostic() && !shouldReport(cls, errorSignature(err))) return;
 
     const Sentry = await import("@sentry/node");
     const net = connectivityClass(err);
@@ -189,6 +196,10 @@ export async function reportError(err: unknown, ctx: ReportContext): Promise<voi
         const vetted = vetSettings(ctx.settings);
         if (vetted) scope.setContext("tool", vetted);
       }
+      const py = err as { pythonType?: string; pythonFrames?: unknown };
+      if (Array.isArray(py.pythonFrames) && py.pythonFrames.length) {
+        scope.setContext("python", { type: py.pythonType, frames: py.pythonFrames });
+      }
       Sentry.captureException(err instanceof Error ? err : new Error(String(err)));
     });
   } catch {

apps/api/src/lib/sentry-scrub.ts

@@ -3,7 +3,7 @@
  * event ceiling. Kept pure (factory + injected gate) so it is unit-testable
  * without initializing the SDK. See the telemetry overhaul spec for the rules.
  */
-import { rebuildErrorValue } from "@snapotter/shared";
+import { rebuildErrorValue, redactMessage } from "@snapotter/shared";
 
 // Per-process runaway guard, not a quota lever: under the sponsored plan we want
 // real errors, but a single instance stuck in an error loop must not spam. Sentry
@@ -27,17 +27,6 @@ const TAG_ALLOWLIST = new Set([
   "instance_id",
 ]);
 
-const URL_RE = /https?:\/\/[^\s"')]+/g;
-const BLOB_RE = /blob:[^\s"')]+/g;
-// Absolute paths under roots that can hold user files (uploads live in /data,
-// /tmp) or dev machines (/Users, /home). Over-redaction of a benign path is fine.
-const PATH_RE = /(?:\/(?:Users|home|root|data|tmp|var|app|opt|mnt|srv)|[A-Za-z]:\\)[^\s"')]*/g;
-
-/** Redact urls, blob refs, and absolute paths from free text (breadcrumb messages). */
-function scrubText(s: string): string {
-  return s.replace(BLOB_RE, "<blob>").replace(URL_RE, "<url>").replace(PATH_RE, "<path>");
-}
-
 // Sentry event/hint are typed loosely on purpose: this module must not import
 // @sentry/node (instrument.ts loads the SDK lazily and passes events through).
 type AnyEvent = Record<string, unknown>;
@@ -60,7 +49,7 @@ function scrubBreadcrumb(entry: unknown): AnyEvent | null {
   for (const k of ["type", "category", "level", "timestamp"]) {
     if (b[k] !== undefined) out[k] = b[k];
   }
-  if (typeof b.message === "string") out.message = scrubText(b.message);
+  if (typeof b.message === "string") out.message = redactMessage(b.message);
   // For http breadcrumbs keep the non-PII status_code + method (the url is the
   // sensitive part, dropped with the rest of `data`): they answer "what request
   // failed right before the error".
@@ -135,7 +124,7 @@ function errorDigest(err: unknown): string {
   return (h >>> 0).toString(16);
 }
 
-export function buildBeforeSend(isActive: () => boolean) {
+export function buildBeforeSend(isActive: () => boolean, diagnostic = false) {
   let windowStart = 0;
   let sentInWindow = 0;
 
@@ -149,6 +138,20 @@ export function buildBeforeSend(isActive: () => boolean) {
     }
     if (++sentInWindow > CEILING_PER_HOUR) return null;
 
+    if (diagnostic) {
+      // A consenting instance: keep the raw message, request, and breadcrumb data.
+      // Still drop identity, and cap message length via redactMessage raw mode.
+      event.user = undefined;
+      const values = asObj(event.exception)?.values;
+      if (Array.isArray(values)) {
+        for (const entry of values) {
+          const ex = asObj(entry);
+          if (ex && typeof ex.value === "string") ex.value = redactMessage(ex.value, { raw: true });
+        }
+      }
+      return event;
+    }
+
     // Dropped: these surfaces can carry user data or PII.
     event.message = undefined;
     event.logentry = undefined;
@@ -176,6 +179,22 @@ export function buildBeforeSend(isActive: () => boolean) {
       }
       if (Object.keys(safe).length) keep.tool = safe;
     }
+    const py = asObj(ctx?.python);
+    if (py) {
+      const type = typeof py.type === "string" ? py.type.slice(0, 64) : undefined;
+      const frames = Array.isArray(py.frames)
+        ? py.frames
+            .map((f) => asObj(f))
+            .filter((f): f is AnyEvent => !!f)
+            .map((f) => ({
+              file: typeof f.file === "string" ? f.file.slice(0, 64) : "?",
+              line: typeof f.line === "number" ? f.line : 0,
+              func: typeof f.func === "string" ? f.func.slice(0, 64) : "?",
+            }))
+            .slice(0, 20)
+        : [];
+      if (frames.length) keep.python = { type, frames };
+    }
     event.contexts = Object.keys(keep).length ? keep : undefined;
 
     const tags = asObj(event.tags);