feat(telemetry): Sentry + PostHog quality pass (#546)

Comprehensive telemetry quality improvements across Sentry and PostHog, grounded in an audit of the live data plus current best-practice research.

Sentry: job_id/instance_id tags, operational fingerprinting, PII-safe settings context on bug events, web tag population + extension-noise filtering, an early-crash buffer, http status/method kept on breadcrumbs, and a gated-off-by-default performance-tracing re-enable (tracesSampler that zeroes db/redis/queue-poll root spans + drops the Redis integration) with worker job spans and canonical-host cron monitors.

PostHog: history_change SPA pageviews, instance_id super property for fleet rollups, enriched tool_used (formats, byte sizes, is_batch, execution_hint, real error_kind taxonomy), the previously-dead result_saved/batch_processed/ai_bundle_prompted events fired, search click-through, editor + Automate authoring + auth instrumentation, a before_send PII boundary, and minimal opt-in landing-site pageviews.

apps/api/src/lib/analytics-allowlist.ts

@@ -2,7 +2,21 @@
 // primitives. Free-text fields (error_message, params, search query) are never
 // allow-listed, so tool settings and filenames cannot reach PostHog.
 const ALLOWED: Record<string, ReadonlySet<string>> = {
-  tool_used: new Set(["tool_id", "status", "duration_ms", "category", "is_ai_tool", "error_code"]),
+  tool_used: new Set([
+    "tool_id",
+    "status",
+    "duration_ms",
+    "category",
+    "is_ai_tool",
+    "is_batch",
+    "input_format",
+    "output_format",
+    "bytes_in",
+    "bytes_out",
+    "execution_hint",
+    "error_code",
+    "error_kind",
+  ]),
   pipeline_executed: new Set([
     "step_count",
     "tool_ids",
@@ -13,6 +27,8 @@ const ALLOWED: Record<string, ReadonlySet<string>> = {
   ]),
   ai_bundle_action: new Set(["bundle_id", "action", "duration_ms"]),
   instance_started: new Set(["arch", "os_platform", "deploy_mode", "gpu_present"]),
+  auth_login: new Set(["method"]),
+  auth_login_failed: new Set(["method"]),
 };
 
 function isAllowedValue(value: unknown): boolean {

apps/api/src/lib/error-report.ts

@@ -37,6 +37,10 @@ export interface ReportContext {
   subsystem?: string;
   /** Safe input format (file extension) for triage; never the filename. */
   inputFormat?: string;
+  /** BullMQ job id, so the event cross-references the jobs DB row and logs. */
+  jobId?: string;
+  /** Tool settings; a vetted (PII-safe) projection is attached for bug events. */
+  settings?: unknown;
 }
 
 export function classifyError(err: unknown, source?: ReportContext["source"]): ErrorClass {
@@ -112,6 +116,33 @@ export function safeFormatTag(filename?: string): string | undefined {
   return m ? m[1].toLowerCase() : undefined;
 }
 
+// Setting keys whose values can carry user data (filenames, free text, secrets)
+// even when short, and which must never reach the Sentry `tool` context.
+const UNSAFE_SETTING_KEY =
+  /pass|secret|token|credential|auth|file|name|path|url|email|user|title|text|label|query|prompt|message|caption|content|watermark/i;
+const SAFE_ENUM_VALUE = /^[A-Za-z0-9_.-]{1,32}$/;
+
+/**
+ * A privacy-safe projection of tool settings for the Sentry `tool` context, so
+ * a bug-class event carries the knobs that triggered it (format, quality, mode,
+ * ...) without ever including filenames, free text, or secrets. Keeps only
+ * numbers, booleans, and short enum-like strings under non-sensitive keys;
+ * drops objects, arrays, and long strings entirely. Returns undefined when
+ * nothing safe survives.
+ */
+export function vetSettings(
+  settings: unknown,
+): Record<string, string | number | boolean> | undefined {
+  if (!settings || typeof settings !== "object" || Array.isArray(settings)) return undefined;
+  const out: Record<string, string | number | boolean> = {};
+  for (const [k, v] of Object.entries(settings as Record<string, unknown>)) {
+    if (UNSAFE_SETTING_KEY.test(k)) continue;
+    if (typeof v === "number" || typeof v === "boolean") out[k] = v;
+    else if (typeof v === "string" && SAFE_ENUM_VALUE.test(v)) out[k] = v;
+  }
+  return Object.keys(out).length ? out : undefined;
+}
+
 /** Fire-and-forget; never throws, never blocks. */
 export async function reportError(err: unknown, ctx: ReportContext): Promise<void> {
   try {
@@ -135,10 +166,42 @@ export async function reportError(err: unknown, ctx: ReportContext): Promise<voi
       if (ctx.method) scope.setTag("method", ctx.method);
       if (ctx.statusCode) scope.setTag("status_code", String(ctx.statusCode));
       if (ctx.subsystem) scope.setTag("subsystem", ctx.subsystem);
-      if (net) scope.setFingerprint(["connectivity", net]);
+      if (ctx.jobId) scope.setTag("job_id", ctx.jobId);
+      if (net) {
+        scope.setFingerprint(["connectivity", net]);
+      } else if (cls === "operational") {
+        // Collapse an operational class (bad DB creds, full disk, ...) into a
+        // single issue keyed on its code, instead of fragmenting into a
+        // separate issue per call site/stack frame (the pg-auth flood showed up
+        // as 5 issues). bug-class errors keep default per-frame grouping, where
+        // distinct frames usually are distinct bugs.
+        scope.setFingerprint(["operational", code ?? (err as { name?: string }).name ?? "op"]);
+      }
+      if (cls === "bug" && ctx.settings !== undefined) {
+        // Attach the knobs that triggered a bug (format, quality, mode, ...) so
+        // it is reproducible, without leaking filenames, free text, or secrets.
+        const vetted = vetSettings(ctx.settings);
+        if (vetted) scope.setContext("tool", vetted);
+      }
       Sentry.captureException(err instanceof Error ? err : new Error(String(err)));
     });
   } catch {
     // telemetry must never throw
   }
 }
+
+/**
+ * Set the anonymized instance id as a GLOBAL Sentry tag so every event (errors
+ * and SDK-captured uncaught exceptions) carries it. Lets triage tell "one
+ * broken install" from "the whole fleet" and cross-references a Sentry event to
+ * that instance's PostHog stream (same instance_id). Called once at boot.
+ */
+export async function setSentryInstanceTag(instanceId: string): Promise<void> {
+  try {
+    if (!instanceId) return;
+    const Sentry = await import("@sentry/node");
+    Sentry.getGlobalScope().setTag("instance_id", instanceId);
+  } catch {
+    // telemetry must never throw
+  }
+}

apps/api/src/lib/sentry-scrub.ts

@@ -23,6 +23,8 @@ const TAG_ALLOWLIST = new Set([
   "subsystem",
   "status_code",
   "input_format",
+  "job_id",
+  "instance_id",
 ]);
 
 const URL_RE = /https?:\/\/[^\s"')]+/g;
@@ -59,6 +61,16 @@ function scrubBreadcrumb(entry: unknown): AnyEvent | null {
     if (b[k] !== undefined) out[k] = b[k];
   }
   if (typeof b.message === "string") out.message = scrubText(b.message);
+  // For http breadcrumbs keep the non-PII status_code + method (the url is the
+  // sensitive part, dropped with the rest of `data`): they answer "what request
+  // failed right before the error".
+  if (b.category === "http") {
+    const data = asObj(b.data);
+    const safe: AnyEvent = {};
+    if (data?.status_code !== undefined) safe.status_code = data.status_code;
+    if (typeof data?.method === "string") safe.method = data.method;
+    if (Object.keys(safe).length) out.data = safe;
+  }
   return out;
 }
 
@@ -102,6 +114,17 @@ export function buildBeforeSend(isActive: () => boolean) {
     if (os?.name) keep.os = { name: os.name, version: os.version };
     const runtime = asObj(ctx?.runtime);
     if (runtime?.name) keep.runtime = { name: runtime.name, version: runtime.version };
+    // The `tool` context is set only by reportError from an already-vetted
+    // settings projection; re-enforce primitives-only here as a final boundary.
+    const tool = asObj(ctx?.tool);
+    if (tool) {
+      const safe: AnyEvent = {};
+      for (const [k, v] of Object.entries(tool)) {
+        if (typeof v === "number" || typeof v === "boolean") safe[k] = v;
+        else if (typeof v === "string" && v.length <= 32) safe[k] = v;
+      }
+      if (Object.keys(safe).length) keep.tool = safe;
+    }
     event.contexts = Object.keys(keep).length ? keep : undefined;
 
     const tags = asObj(event.tags);

apps/api/src/lib/sentry-tracing.ts

@@ -0,0 +1,52 @@
+/**
+ * Sentry performance-tracing sampler for the API, kept pure (no @sentry/node
+ * import) so it is unit-testable without the SDK.
+ *
+ * Tracing is OFF unless SENTRY_TRACES_SAMPLE_RATE is a positive number (see
+ * instrument.ts). The July 2026 quota incident came from the default redis and
+ * postgres integrations turning BullMQ blocking-poll commands and pg idle
+ * connection pings into standalone root transactions the moment any sampler was
+ * enabled. This sampler returns 0 for exactly those, three independent ways
+ * (op prefix, db.system, messaging.system + name), so a poll storm can never
+ * count against quota again, while real HTTP requests and named job executions
+ * still sample.
+ */
+export interface SamplingContext {
+  name?: string;
+  attributes?: Record<string, unknown>;
+  /** Sentry's helper: returns the parent's decision when there is one, else the fallback. */
+  inheritOrSampleWith?: (fallback: number) => number;
+}
+
+export function buildTracesSampler(httpRate: number): (ctx: SamplingContext) => number {
+  return (ctx: SamplingContext): number => {
+    const attrs = ctx.attributes ?? {};
+    const name = ctx.name ?? "";
+    const inherit = (fallback: number): number =>
+      typeof ctx.inheritOrSampleWith === "function" ? ctx.inheritOrSampleWith(fallback) : fallback;
+
+    // Standalone db / redis root transactions (pg idle pings, BullMQ blocking
+    // polls). The exact incident source: zero them outright.
+    const op = String(attrs["sentry.op"] ?? "");
+    if (op.startsWith("db")) return 0;
+    if (attrs["db.system"] === "redis") return 0;
+
+    // Any queue root span: only sample a real job execution we explicitly named
+    // "job <name>" (see worker.ts); drop the continuous poll transactions.
+    if (attrs["messaging.system"] !== undefined) {
+      // Sample a real job execution (the worker's "job.process" span, or a
+      // "job <name>" span); drop the continuous BullMQ poll transactions.
+      const isJob = name === "job.process" || name.startsWith("job ");
+      return isJob ? Math.min(httpRate, 0.2) : 0;
+    }
+
+    // Never sample infra endpoints, even though they are real HTTP.
+    if (/\/(healthz|readyz|metrics)\b/.test(name)) return 0;
+
+    // Real inbound HTTP: sample at the configured rate, honoring parent traces.
+    if (attrs["http.request.method"] !== undefined) return inherit(httpRate);
+
+    // Anything else: follow the parent decision, else drop.
+    return inherit(0);
+  };
+}