fix: error-only Sentry telemetry, storm-proof capture, and crash fixes (#476)

Removes Sentry tracing entirely (BullMQ idle polling burned 4.8M transactions in 2 days at the baked 0.1 rate), decouples PostHog sampling, and replaces the type-only error scrub with a vetted-field sanitizer plus SafeError/ToolInputError contracts. One classified capture path with per-signature throttles and a per-process ceiling makes storms impossible (NODE-1E was 4,541 events from one 30s loop). Browser errors move to a dedicated web Sentry project with their own source maps. Adds the SNAPOTTER_TELEMETRY runtime kill switch and silences test fleets.

Crash fixes: remote 204/304 SSRF process kill (NODE-20), conversion-preset boot crash loop (NODE-21), Redis version preflight + unhandled subscribe rejection (NODE-1T), Sign PDF on plain-http origins (NODE-1K/1M), wavesurfer/pdf.js teardown rejections (NODE-1P/1N), bundle-import ZlibError to 400 (NODE-1Z), chart-maker input errors declassified (NODE-1H/1J), asset requests skip the session DB lookup (NODE-1D).

apps/api/src/lib/analytics-gate.ts

@@ -25,8 +25,15 @@ async function defaultReader(): Promise<boolean | undefined> {
   return rows[0].value !== "false";
 }
 
+/** Runtime kill switch honored in ALL builds: SNAPOTTER_TELEMETRY=0|false|off. */
+export function telemetryEnvKilled(): boolean {
+  const v = process.env.SNAPOTTER_TELEMETRY;
+  return v === "0" || v === "false" || v === "off";
+}
+
 /** Compile-time bake, with a NON-PRODUCTION-only override so tests can force it on. */
 export function bakedEnabled(): boolean {
+  if (telemetryEnvKilled()) return false;
   if (process.env.NODE_ENV !== "production") {
     const o = process.env.ANALYTICS_BAKED_OVERRIDE;
     if (o === "on") return true;

apps/api/src/lib/analytics.ts

@@ -59,13 +59,10 @@ export async function initAnalytics(): Promise<void> {
 }
 
 export async function captureException(error: unknown): Promise<void> {
-  try {
-    if (!analyticsEnabled()) return;
-    const Sentry = await import("@sentry/node");
-    Sentry.captureException(error);
-  } catch {
-    // analytics must never throw
-  }
+  // Deprecated shim: route through the classified path. New code calls
+  // reportError directly with a source.
+  const { reportError } = await import("./error-report.js");
+  await reportError(error, { source: "boot" });
 }
 
 export async function shutdownAnalytics(): Promise<void> {
@@ -90,8 +87,13 @@ export async function trackEvent(
 ): Promise<void> {
   try {
     if (!analyticsEnabled() || !posthogClient) return;
-    if (ANALYTICS_BAKED.sampleRate < 1.0) {
-      if (ANALYTICS_BAKED.sampleRate <= 0.0 || Math.random() >= ANALYTICS_BAKED.sampleRate) return;
+    if (ANALYTICS_BAKED.posthogSampleRate < 1.0) {
+      if (
+        ANALYTICS_BAKED.posthogSampleRate <= 0.0 ||
+        Math.random() >= ANALYTICS_BAKED.posthogSampleRate
+      ) {
+        return;
+      }
     }
     posthogClient.capture({
       distinctId: distinctId ?? (await getInstanceId()),

apps/api/src/lib/error-report.ts

@@ -0,0 +1,123 @@
+/**
+ * The single deliberate Sentry capture path for the API.
+ *
+ * Classes:
+ *  - expected: user input / client aborts / cancels. Never sent.
+ *  - operational: someone's environment is broken (db down, disk full).
+ *    Sent once per signature per hour, level=warning, fingerprinted per class.
+ *  - bug: our fault. Sent up to 10 per signature per hour.
+ *
+ * State is per-process, so a crash-looping instance always reports its first
+ * event after each restart. The beforeSend ceiling (sentry-scrub.ts) is the
+ * final backstop and also covers SDK-captured uncaught exceptions.
+ */
+import {
+  connectivityClass,
+  isClientAbort,
+  isSafeMessageError,
+  isToolInputError,
+} from "@snapotter/shared";
+import { analyticsEnabled } from "./analytics-gate.js";
+
+export type ErrorClass = "expected" | "operational" | "bug";
+
+const HOUR_MS = 3600_000;
+const LIMITS: Record<Exclude<ErrorClass, "expected">, number> = { operational: 1, bug: 10 };
+const OPERATIONAL_CODES = new Set(["ENOSPC", "EACCES", "EROFS", "EMFILE", "ENFILE"]);
+
+export interface ReportContext {
+  source: "http" | "worker" | "cron" | "boot";
+  toolId?: string;
+  pool?: string;
+  route?: string;
+  method?: string;
+  statusCode?: number;
+  subsystem?: string;
+}
+
+export function classifyError(err: unknown, source?: ReportContext["source"]): ErrorClass {
+  if (isToolInputError(err)) return "expected";
+  const e = err as { name?: string; message?: string; code?: string } | null;
+  if (e && typeof e.message === "string" && /^(Canceled$|Timed out after )/.test(e.message)) {
+    return "expected";
+  }
+  // The next two shortcuts only make sense at the HTTP boundary (undefined
+  // keeps the http-ish default for direct calls). Off the request path a bare
+  // ECONNRESET is an upstream socket loss, not a client abort, and a ZodError
+  // means schema drift: settings were already validated at the boundary, so a
+  // worker-side parse failure is our bug.
+  if (source === "http" || source === undefined) {
+    if (isClientAbort(err)) return "expected";
+    // ZodError = settings validation; InputValidationError = upload validation
+    // (apps/api/src/modality/contract.ts). Both are user-input problems.
+    if (e?.name === "ZodError" || e?.name === "InputValidationError") return "expected";
+  }
+  if (isSafeMessageError(err)) return err.kind === "bug" ? "bug" : "operational";
+  if (connectivityClass(err)) return "operational";
+  if (e?.code && OPERATIONAL_CODES.has(e.code)) return "operational";
+  return "bug";
+}
+
+const seen = new Map<string, { count: number; windowStart: number }>();
+
+export function shouldReport(
+  cls: Exclude<ErrorClass, "expected">,
+  signature: string,
+  now = Date.now(),
+): boolean {
+  const key = `${cls}:${signature}`;
+  const entry = seen.get(key);
+  if (!entry || now - entry.windowStart > HOUR_MS) {
+    seen.set(key, { count: 1, windowStart: now });
+    return true;
+  }
+  entry.count++;
+  return entry.count <= LIMITS[cls];
+}
+
+export function resetThrottleForTests(): void {
+  seen.clear();
+}
+
+export function errorSignature(err: unknown): string {
+  const e = err as { name?: string; code?: string; stack?: string } | null;
+  const name = e?.name ?? "Unknown";
+  const code = e?.code ?? "-";
+  let frame = "-";
+  if (typeof e?.stack === "string") {
+    const line = e.stack.split("\n").find((l) => l.includes("/apps/") || l.includes("/packages/"));
+    const m = line?.match(/([^/\\]+\.[cm]?[jt]sx?):(\d+)/);
+    if (m) frame = `${m[1]}:${m[2]}`;
+  }
+  return `${name}:${code}:${frame}`;
+}
+
+/** Fire-and-forget; never throws, never blocks. */
+export async function reportError(err: unknown, ctx: ReportContext): Promise<void> {
+  try {
+    if (!analyticsEnabled()) return;
+    const cls = classifyError(err, ctx.source);
+    if (cls === "expected") return;
+    if (!shouldReport(cls, errorSignature(err))) return;
+
+    const Sentry = await import("@sentry/node");
+    const net = connectivityClass(err);
+    Sentry.withScope((scope) => {
+      scope.setLevel(cls === "operational" ? "warning" : "error");
+      scope.setTag("source", ctx.source);
+      scope.setTag("error_class", cls);
+      const code = (err as { code?: string } | null)?.code;
+      if (code) scope.setTag("error_code", code);
+      if (ctx.toolId) scope.setTag("tool_id", ctx.toolId);
+      if (ctx.pool) scope.setTag("pool", ctx.pool);
+      if (ctx.route) scope.setTag("route", ctx.route);
+      if (ctx.method) scope.setTag("method", ctx.method);
+      if (ctx.statusCode) scope.setTag("status_code", String(ctx.statusCode));
+      if (ctx.subsystem) scope.setTag("subsystem", ctx.subsystem);
+      if (net) scope.setFingerprint(["connectivity", net]);
+      Sentry.captureException(err instanceof Error ? err : new Error(String(err)));
+    });
+  } catch {
+    // telemetry must never throw
+  }
+}

apps/api/src/lib/feature-status.ts

@@ -741,6 +741,24 @@ export async function importBundleArchive(
       extractor.on("finish", () => res());
       extractor.on("error", rej);
       stream.on("error", rej);
+    }).catch((err: unknown) => {
+      // Malformed uploads (non-gzip data, corrupt/truncated gzip, garbage
+      // tar) surface as ZlibError or tar parse errors here. Map them to a
+      // 400-able validation error instead of letting them escape as a 500
+      // (Sentry NODE-1Z). Fatal node-tar parse errors always carry tarCode
+      // (TAR_ABORT, TAR_BAD_ARCHIVE); recoverable ones never reach "error".
+      if (err instanceof ImportValidationError) throw err;
+      const name = (err as Error | null)?.name ?? "";
+      const msg = String((err as Error | null)?.message ?? "");
+      const tarCode = (err as { tarCode?: unknown } | null)?.tarCode;
+      if (
+        name === "ZlibError" ||
+        typeof tarCode === "string" ||
+        /unexpected end of (file|data)|invalid tar|incorrect header check|zlib/i.test(msg)
+      ) {
+        throw new ImportValidationError("Not a valid bundle archive");
+      }
+      throw err;
     });
 
     // Read and validate bundle.json

apps/api/src/lib/file-storage.ts

@@ -4,6 +4,7 @@ import { mkdir, readFile, statfs, unlink, writeFile } from "node:fs/promises";
 import { extname, join } from "node:path";
 import type { Readable } from "node:stream";
 import type { S3StorageModule } from "@snapotter/enterprise";
+import { SafeError } from "@snapotter/shared";
 import { env } from "../config.js";
 
 const MIN_FREE_BYTES = 100 * 1024 * 1024;
@@ -13,9 +14,12 @@ async function assertDiskSpace(dir: string): Promise<void> {
     const stats = await statfs(dir);
     const freeBytes = stats.bfree * stats.bsize;
     if (freeBytes < MIN_FREE_BYTES) {
-      const err = new Error("Insufficient disk space") as Error & { statusCode: number };
-      err.statusCode = 507;
-      throw err;
+      // ENOSPC here is synthesized from the free-space floor check, not a syscall errno.
+      throw new SafeError("Insufficient disk space", {
+        kind: "operational",
+        code: "ENOSPC",
+        statusCode: 507,
+      });
     }
   } catch (e) {
     if (e instanceof Error && (e as Error & { statusCode?: number }).statusCode === 507) throw e;
@@ -102,11 +106,11 @@ export async function ensureStorageDir(): Promise<void> {
     await mkdir(env.FILES_STORAGE_PATH, { recursive: true });
   } catch (e) {
     if (e instanceof Error && (e as NodeJS.ErrnoException).code === "EACCES") {
-      const err = new Error("Storage directory is not writable") as Error & {
-        statusCode: number;
-      };
-      err.statusCode = 503;
-      throw err;
+      throw new SafeError("Storage directory is not writable", {
+        kind: "operational",
+        code: (e as NodeJS.ErrnoException).code,
+        statusCode: 503,
+      });
     }
     throw e;
   }
@@ -126,11 +130,11 @@ export async function saveFile(buffer: Buffer, originalName: string): Promise<st
     await writeFile(join(env.FILES_STORAGE_PATH, storedName), buffer);
   } catch (e) {
     if (e instanceof Error && (e as NodeJS.ErrnoException).code === "EACCES") {
-      const err = new Error("Storage directory is not writable") as Error & {
-        statusCode: number;
-      };
-      err.statusCode = 503;
-      throw err;
+      throw new SafeError("Storage directory is not writable", {
+        kind: "operational",
+        code: (e as NodeJS.ErrnoException).code,
+        statusCode: 503,
+      });
     }
     throw e;
   }
@@ -185,7 +189,11 @@ async function ensureThumbDir(): Promise<void> {
     await mkdir(join(env.FILES_STORAGE_PATH, THUMB_DIR), { recursive: true });
   } catch (err: unknown) {
     if ((err as NodeJS.ErrnoException).code === "EACCES") {
-      throw Object.assign(new Error("Storage directory is not writable"), { statusCode: 503 });
+      throw new SafeError("Storage directory is not writable", {
+        kind: "operational",
+        code: (err as NodeJS.ErrnoException).code,
+        statusCode: 503,
+      });
     }
     throw err;
   }

apps/api/src/lib/sentry-scrub.ts

@@ -0,0 +1,100 @@
+/**
+ * Sentry beforeSend for the API: allowlist-first scrubbing plus a per-process
+ * event ceiling. Kept pure (factory + injected gate) so it is unit-testable
+ * without initializing the SDK. See the telemetry overhaul spec for the rules.
+ */
+import { rebuildErrorValue } from "@snapotter/shared";
+
+const CEILING_PER_HOUR = 20;
+const HOUR_MS = 3600_000;
+
+const TAG_ALLOWLIST = new Set([
+  "source",
+  "tool_id",
+  "pool",
+  "route",
+  "method",
+  "error_class",
+  "error_code",
+  "deploy_mode",
+  "subsystem",
+  "status_code",
+]);
+
+function basename(p: string): string {
+  const i = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
+  return i >= 0 ? p.slice(i + 1) : p;
+}
+
+// Sentry event/hint are typed loosely on purpose: this module must not import
+// @sentry/node (instrument.ts loads the SDK lazily and passes events through).
+type AnyEvent = Record<string, unknown>;
+type AnyHint = { originalException?: unknown };
+
+/** Narrow to a plain mutable object, or null for anything else (fail-closed). */
+function asObj(value: unknown): AnyEvent | null {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as AnyEvent)
+    : null;
+}
+
+export function buildBeforeSend(isActive: () => boolean) {
+  let windowStart = 0;
+  let sentInWindow = 0;
+
+  return function beforeSend(event: AnyEvent, hint: AnyHint): AnyEvent | null {
+    if (!isActive()) return null;
+
+    const now = Date.now();
+    if (now - windowStart > HOUR_MS) {
+      windowStart = now;
+      sentInWindow = 0;
+    }
+    if (++sentInWindow > CEILING_PER_HOUR) return null;
+
+    event.message = undefined;
+    event.logentry = undefined;
+    event.server_name = undefined;
+    event.request = undefined;
+    event.extra = undefined;
+    event.breadcrumbs = undefined;
+    event.user = undefined;
+
+    const ctx = asObj(event.contexts);
+    const keep: AnyEvent = {};
+    const os = asObj(ctx?.os);
+    if (os?.name) keep.os = { name: os.name, version: os.version };
+    const runtime = asObj(ctx?.runtime);
+    if (runtime?.name) keep.runtime = { name: runtime.name, version: runtime.version };
+    event.contexts = Object.keys(keep).length ? keep : undefined;
+
+    const tags = asObj(event.tags);
+    if (tags) {
+      for (const key of Object.keys(tags)) {
+        if (!TAG_ALLOWLIST.has(key)) delete tags[key];
+      }
+    }
+
+    const rebuilt = rebuildErrorValue(hint?.originalException);
+    const values = asObj(event.exception)?.values;
+    if (Array.isArray(values)) {
+      for (let i = 0; i < values.length; i++) {
+        const ex = asObj(values[i]);
+        if (!ex) continue;
+        // The last entry is the original error; linked/outer wrappers get type-only.
+        ex.value = i === values.length - 1 && rebuilt ? rebuilt : ex.type;
+        const frames = asObj(ex.stacktrace)?.frames;
+        if (Array.isArray(frames)) {
+          for (const entry of frames) {
+            const frame = asObj(entry);
+            if (!frame) continue;
+            if (typeof frame.filename === "string") frame.filename = basename(frame.filename);
+            frame.abs_path = undefined;
+            frame.vars = undefined;
+          }
+        }
+      }
+    }
+    return event;
+  };
+}

apps/api/src/lib/ssrf.ts

@@ -172,6 +172,29 @@ function normalizeSafeFetchOptions(options?: AbortSignal | SafeFetchOptions): Sa
   return options;
 }
 
+// Statuses that undici's Response constructor rejects a body for. An empty
+// Buffer still counts as a body, so these must pass null explicitly. A remote
+// server controls this status; before this guard, a 204/304 reply crashed the
+// whole process from inside the 'end' event handler (Sentry NODE-20).
+const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]);
+
+export function toFetchResponse(
+  // ArrayBuffer-backed (what Buffer.concat/from/alloc return); the Response
+  // constructor's BodyInit does not accept SharedArrayBuffer-backed views.
+  body: Buffer<ArrayBuffer>,
+  statusCode: number | undefined,
+  statusText: string,
+  headers: Headers,
+): Response {
+  const status =
+    statusCode !== undefined && statusCode >= 200 && statusCode <= 599 ? statusCode : 502;
+  return new Response(NULL_BODY_STATUSES.has(status) ? null : body, {
+    status,
+    statusText,
+    headers,
+  });
+}
+
 function withResponseSizeLimit(response: Response, maxBytes?: number): Response {
   if (maxBytes === undefined || !response.body) return response;
 
@@ -275,13 +298,18 @@ export async function safeFetch(
                   for (const v of vals) headers.append(key, v);
                 }
               }
-              resolve(
-                new Response(body, {
-                  status: incomingMessage.statusCode ?? 500,
-                  statusText: incomingMessage.statusMessage ?? "",
-                  headers,
-                }),
-              );
+              try {
+                resolve(
+                  toFetchResponse(
+                    body,
+                    incomingMessage.statusCode,
+                    incomingMessage.statusMessage ?? "",
+                    headers,
+                  ),
+                );
+              } catch (err) {
+                reject(err);
+              }
             });
             incomingMessage.on("error", (err) => {
               if (settled) return;