feat(telemetry): Sentry + PostHog quality pass (#546)

Comprehensive telemetry quality improvements across Sentry and PostHog, grounded in an audit of the live data plus current best-practice research.

Sentry: job_id/instance_id tags, operational fingerprinting, PII-safe settings context on bug events, web tag population + extension-noise filtering, an early-crash buffer, http status/method kept on breadcrumbs, and a gated-off-by-default performance-tracing re-enable (tracesSampler that zeroes db/redis/queue-poll root spans + drops the Redis integration) with worker job spans and canonical-host cron monitors.

PostHog: history_change SPA pageviews, instance_id super property for fleet rollups, enriched tool_used (formats, byte sizes, is_batch, execution_hint, real error_kind taxonomy), the previously-dead result_saved/batch_processed/ai_bundle_prompted events fired, search click-through, editor + Automate authoring + auth instrumentation, a before_send PII boundary, and minimal opt-in landing-site pageviews.

tests/unit/api/analytics-allowlist.test.ts

@@ -2,22 +2,38 @@ import { describe, expect, it } from "vitest";
 import { sanitizeEventProperties } from "../../../apps/api/src/lib/analytics-allowlist.js";
 
 describe("sanitizeEventProperties", () => {
-  it("keeps only allow-listed keys for tool_used", () => {
+  it("keeps the enriched allow-listed keys for tool_used and drops free-text/PII", () => {
     const out = sanitizeEventProperties("tool_used", {
       tool_id: "resize",
-      status: "completed",
+      status: "failed",
       duration_ms: 12,
       category: "image",
       is_ai_tool: false,
+      is_batch: true,
+      input_format: "heic",
+      output_format: "png",
+      bytes_in: 4096,
+      bytes_out: 2048,
+      execution_hint: "fast",
+      error_kind: "input",
+      error_code: "corrupt-header",
       error_message: "stack with /uploads/secret.docx",
       params: { watermark_text: "CONFIDENTIAL" },
     });
     expect(out).toEqual({
       tool_id: "resize",
-      status: "completed",
+      status: "failed",
       duration_ms: 12,
       category: "image",
       is_ai_tool: false,
+      is_batch: true,
+      input_format: "heic",
+      output_format: "png",
+      bytes_in: 4096,
+      bytes_out: 2048,
+      execution_hint: "fast",
+      error_kind: "input",
+      error_code: "corrupt-header",
     });
     expect(out).not.toHaveProperty("error_message");
     expect(out).not.toHaveProperty("params");

tests/unit/api/capture-path.test.ts

@@ -8,20 +8,33 @@
  * and the throttle allows one capture per distinct signature.
  */
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { reportError, resetThrottleForTests } from "../../../apps/api/src/lib/error-report.js";
+import {
+  reportError,
+  resetThrottleForTests,
+  setSentryInstanceTag,
+} from "../../../apps/api/src/lib/error-report.js";
 
 const h = vi.hoisted(() => {
-  const scope = { setTag: vi.fn(), setLevel: vi.fn(), setFingerprint: vi.fn() };
+  const scope = {
+    setTag: vi.fn(),
+    setLevel: vi.fn(),
+    setFingerprint: vi.fn(),
+    setContext: vi.fn(),
+  };
+  const globalScope = { setTag: vi.fn() };
   return {
     scope,
+    globalScope,
     captureException: vi.fn(),
     withScope: vi.fn((cb: (s: typeof scope) => unknown) => cb(scope)),
+    getGlobalScope: vi.fn(() => globalScope),
   };
 });
 
 vi.mock("@sentry/node", () => ({
   captureException: h.captureException,
   withScope: h.withScope,
+  getGlobalScope: h.getGlobalScope,
 }));
 
 vi.mock("../../../apps/api/src/lib/analytics-gate.js", () => ({
@@ -71,4 +84,58 @@ describe("capture path", () => {
     await reportError(denied, { source: "worker", pool: "image" });
     expect(h.captureException).toHaveBeenCalledTimes(2);
   });
+
+  it("tags job_id so the event cross-references the DB row, logs, and PostHog stream", async () => {
+    await reportError(new Error("boom"), { source: "worker", pool: "image", jobId: "job-abc" });
+    expect(h.scope.setTag).toHaveBeenCalledWith("job_id", "job-abc");
+  });
+
+  it("collapses operational errors to one issue per code via fingerprint", async () => {
+    const full = Object.assign(new Error("disk full"), { code: "ENOSPC" });
+    await reportError(full, { source: "worker", pool: "image" });
+    expect(h.scope.setFingerprint).toHaveBeenCalledWith(["operational", "ENOSPC"]);
+  });
+
+  it("prefers the connectivity fingerprint for infra-connectivity operational errors", async () => {
+    const pg = Object.assign(new Error("Failed query: select 1"), {
+      cause: Object.assign(new Error("57P01"), { code: "57P01" }),
+    });
+    await reportError(pg, { source: "worker", pool: "docs" });
+    expect(h.scope.setFingerprint).toHaveBeenCalledWith(["connectivity", "pg-unavailable"]);
+  });
+
+  it("leaves bug-class errors on default per-frame grouping (no fingerprint)", async () => {
+    await reportError(new Error("undefined is not a function"), {
+      source: "worker",
+      pool: "image",
+    });
+    expect(h.scope.setFingerprint).not.toHaveBeenCalled();
+  });
+
+  it("attaches a vetted tool context for bug-class events to aid reproduction", async () => {
+    await reportError(new Error("boom"), {
+      source: "worker",
+      pool: "image",
+      settings: { format: "png", quality: 80, filename: "my secret vacation.png" },
+    });
+    expect(h.scope.setContext).toHaveBeenCalledWith("tool", { format: "png", quality: 80 });
+  });
+
+  it("does not attach a settings context for non-bug errors", async () => {
+    const full = Object.assign(new Error("disk full"), { code: "ENOSPC" });
+    await reportError(full, { source: "worker", pool: "image", settings: { format: "png" } });
+    expect(h.scope.setContext).not.toHaveBeenCalled();
+  });
+});
+
+describe("setSentryInstanceTag", () => {
+  it("sets instance_id on the global scope so every event carries it", async () => {
+    await setSentryInstanceTag("inst-xyz");
+    expect(h.globalScope.setTag).toHaveBeenCalledWith("instance_id", "inst-xyz");
+  });
+
+  it("is a no-op on a falsy id and never throws", async () => {
+    await expect(setSentryInstanceTag("")).resolves.toBeUndefined();
+    expect(h.globalScope.setTag).not.toHaveBeenCalled();
+  });
 });

tests/unit/api/error-report.test.ts

@@ -6,6 +6,7 @@ import {
   resetThrottleForTests,
   safeFormatTag,
   shouldReport,
+  vetSettings,
 } from "../../../apps/api/src/lib/error-report.js";
 
 describe("safeFormatTag", () => {
@@ -132,3 +133,37 @@ describe("errorSignature", () => {
     expect(errorSignature(null)).toBe("Unknown:-:-");
   });
 });
+
+describe("vetSettings", () => {
+  it("keeps numbers, booleans, and short enum-like string values", () => {
+    expect(vetSettings({ quality: 80, lossless: true, format: "png", fit: "cover" })).toEqual({
+      quality: 80,
+      lossless: true,
+      format: "png",
+      fit: "cover",
+    });
+  });
+  it("drops sensitive keys, free-text/PII-shaped values, objects and arrays", () => {
+    // A password, a filename, and watermark text must never reach Sentry; nested
+    // objects/arrays and long strings can carry user data, so drop them too.
+    expect(
+      vetSettings({
+        password: "hunter2",
+        filename: "IMG_1234.png",
+        watermarkText: "Property of Jane",
+        crop: { x: 1, y: 2 },
+        sizes: [1, 2, 3],
+        width: 1024,
+        format: "webp",
+      }),
+    ).toEqual({ width: 1024, format: "webp" });
+  });
+  it("returns undefined for non-objects and when nothing safe survives", () => {
+    expect(vetSettings(undefined)).toBeUndefined();
+    expect(vetSettings("nope")).toBeUndefined();
+    expect(vetSettings([1, 2])).toBeUndefined();
+    expect(
+      vetSettings({ note: "a long free-text field well beyond the safe length" }),
+    ).toBeUndefined();
+  });
+});

tests/unit/api/sentry-scrub.test.ts

@@ -59,18 +59,22 @@ describe("buildBeforeSend (api)", () => {
     );
     expect(out.exception.values[0].stacktrace.frames[0].abs_path).toBe("/app/x");
   });
-  it("keeps the breadcrumb trail, redacting paths/urls and dropping data payloads", () => {
+  it("keeps the breadcrumb trail, redacting urls but keeping safe http status/method", () => {
     const out = send(
       evt({
         breadcrumbs: [
-          { message: "GET https://host/u/photo.jpg 200", category: "http", data: { url: "x" } },
+          {
+            message: "GET https://host/u/photo.jpg 500",
+            category: "http",
+            data: { url: "https://host/u/photo.jpg", status_code: 500, method: "GET" },
+          },
           { message: "reading /Users/me/secret.txt", category: "console", level: "info" },
         ],
       }),
       {},
     )!;
     expect(out.breadcrumbs).toEqual([
-      { message: "GET <url> 200", category: "http" },
+      { message: "GET <url> 500", category: "http", data: { status_code: 500, method: "GET" } },
       { message: "reading <path>", category: "console", level: "info" },
     ]);
   });
@@ -102,6 +106,21 @@ describe("buildBeforeSend (api)", () => {
     expect(out.tags.input_format).toBe("webp");
     expect(out.tags.secret_tag).toBeUndefined();
   });
+  it("keeps job_id and instance_id tags for cross-referencing and blast-radius triage", () => {
+    const out = send(evt({ tags: { job_id: "j1", instance_id: "i1", secret_tag: "x" } }), {})!;
+    expect(out.tags.job_id).toBe("j1");
+    expect(out.tags.instance_id).toBe("i1");
+    expect(out.tags.secret_tag).toBeUndefined();
+  });
+  it("keeps a vetted tool context (primitives) and drops non-primitive fields", () => {
+    const out = send(
+      evt({
+        contexts: { tool: { format: "png", quality: 80, blob: { x: 1 }, long: "x".repeat(40) } },
+      }),
+      {},
+    )!;
+    expect(out.contexts.tool).toEqual({ format: "png", quality: 80 });
+  });
   it("drops contexts entirely when nothing allowlisted survives", () => {
     const out = send(evt({ contexts: { device: { hostname: "leak" } } }), {})!;
     expect(out.contexts).toBeUndefined();

tests/unit/api/sentry-tracing.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+import { buildTracesSampler } from "../../../apps/api/src/lib/sentry-tracing.js";
+
+const sampler = buildTracesSampler(0.05);
+
+describe("buildTracesSampler (the July-incident guard)", () => {
+  it("zeroes standalone redis root transactions (BullMQ blocking polls)", () => {
+    expect(sampler({ name: "BRPOPLPUSH", attributes: { "db.system": "redis" } })).toBe(0);
+  });
+
+  it("zeroes standalone db root transactions (pg idle pings) by op prefix", () => {
+    expect(sampler({ name: "SELECT 1", attributes: { "sentry.op": "db.query" } })).toBe(0);
+    expect(sampler({ name: "pg", attributes: { "sentry.op": "db.redis" } })).toBe(0);
+  });
+
+  it("drops queue poll transactions but samples real job executions", () => {
+    expect(sampler({ name: "queue.poll", attributes: { "messaging.system": "bullmq" } })).toBe(0);
+    expect(
+      sampler({ name: "job.process", attributes: { "messaging.system": "bullmq" } }),
+    ).toBeGreaterThan(0);
+    expect(
+      sampler({ name: "job resize", attributes: { "messaging.system": "bullmq" } }),
+    ).toBeGreaterThan(0);
+  });
+
+  it("never samples infra endpoints even as HTTP", () => {
+    expect(sampler({ name: "GET /healthz", attributes: { "http.request.method": "GET" } })).toBe(0);
+    expect(sampler({ name: "GET /readyz", attributes: { "http.request.method": "GET" } })).toBe(0);
+    expect(sampler({ name: "GET /metrics", attributes: { "http.request.method": "GET" } })).toBe(0);
+  });
+
+  it("samples real inbound HTTP at the configured rate", () => {
+    expect(
+      sampler({
+        name: "POST /api/v1/tools/image/resize",
+        attributes: { "http.request.method": "POST" },
+      }),
+    ).toBe(0.05);
+  });
+
+  it("respects an explicit parent sampling decision via inheritOrSampleWith", () => {
+    expect(
+      sampler({
+        name: "GET /api/v1/x",
+        attributes: { "http.request.method": "GET" },
+        inheritOrSampleWith: () => 1,
+      }),
+    ).toBe(1);
+  });
+
+  it("drops anything unrecognized when there is no parent", () => {
+    expect(sampler({ name: "mystery", attributes: {} })).toBe(0);
+    expect(sampler({})).toBe(0);
+  });
+});

tests/unit/shared/analytics-events.test.ts

@@ -2,8 +2,8 @@ import { ANALYTICS_EVENTS } from "@snapotter/shared";
 import { describe, expect, it } from "vitest";
 
 describe("ANALYTICS_EVENTS", () => {
-  it("has exactly 15 event keys", () => {
-    expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(15);
+  it("has exactly 24 event keys", () => {
+    expect(Object.keys(ANALYTICS_EVENTS)).toHaveLength(24);
   });
 
   it("contains the expected keys", () => {
@@ -22,6 +22,15 @@ describe("ANALYTICS_EVENTS", () => {
     expect(ANALYTICS_EVENTS).toHaveProperty("FEEDBACK_SUBMITTED");
     expect(ANALYTICS_EVENTS).toHaveProperty("SPONSOR_CLICKED");
     expect(ANALYTICS_EVENTS).toHaveProperty("INSTANCE_STARTED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_OPENED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_TOOL_USED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("EDITOR_EXPORTED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_OPENED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_STEP_ADDED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_SAVED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("PIPELINE_TEMPLATE_SELECTED");
+    expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN");
+    expect(ANALYTICS_EVENTS).toHaveProperty("AUTH_LOGIN_FAILED");
   });
 
   it("all event values are strings", () => {

tests/unit/web/early-errors.test.ts

@@ -0,0 +1,56 @@
+// @vitest-environment jsdom
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  flushEarlyErrors,
+  resetEarlyErrorsForTests,
+  startEarlyErrorCapture,
+} from "../../../apps/web/src/lib/early-errors.js";
+
+const h = vi.hoisted(() => ({ captureException: vi.fn(), client: {} as object | undefined }));
+vi.mock("@sentry/react", () => ({
+  captureException: h.captureException,
+  getClient: () => h.client,
+}));
+
+// Guard so jsdom does not surface these synthetic error events as "uncaught"
+// (production code intentionally never preventDefaults real errors).
+const guard = (e: Event) => e.preventDefault();
+beforeEach(() => {
+  resetEarlyErrorsForTests();
+  vi.clearAllMocks();
+  h.client = {};
+  window.addEventListener("error", guard);
+});
+afterEach(() => window.removeEventListener("error", guard));
+
+describe("early error buffer", () => {
+  it("buffers a pre-init error and replays it to Sentry on flush", async () => {
+    startEarlyErrorCapture();
+    const boom = new Error("early boom");
+    window.dispatchEvent(new ErrorEvent("error", { error: boom, cancelable: true }));
+
+    await flushEarlyErrors();
+
+    expect(h.captureException).toHaveBeenCalledTimes(1);
+    expect(h.captureException).toHaveBeenCalledWith(boom);
+  });
+
+  it("does not replay when Sentry never initialized (opt-out safe)", async () => {
+    startEarlyErrorCapture();
+    window.dispatchEvent(new ErrorEvent("error", { error: new Error("x"), cancelable: true }));
+    h.client = undefined; // no Sentry client -> analytics off
+
+    await flushEarlyErrors();
+
+    expect(h.captureException).not.toHaveBeenCalled();
+  });
+
+  it("stops buffering after flush so post-init errors are left to the SDK", async () => {
+    startEarlyErrorCapture();
+    await flushEarlyErrors();
+    window.dispatchEvent(new ErrorEvent("error", { error: new Error("late"), cancelable: true }));
+    await flushEarlyErrors();
+
+    expect(h.captureException).not.toHaveBeenCalled();
+  });
+});

tests/unit/web/sentry-scrub.test.ts

@@ -71,19 +71,23 @@ describe("buildWebBeforeSend", () => {
     expect(buildWebBeforeSend(() => false)(baseEvent(), {})).toBeNull();
   });
 
-  it("keeps the breadcrumb trail, redacting paths/urls and dropping data payloads", () => {
+  it("keeps the breadcrumb trail, redacting urls but keeping safe fetch status/method", () => {
     const send = buildWebBeforeSend(() => true);
     const out = send(
       baseEvent({
         breadcrumbs: [
-          { message: "fetch https://host/user.png", category: "fetch", data: { url: "x" } },
+          {
+            message: "fetch https://host/user.png",
+            category: "fetch",
+            data: { url: "https://host/user.png", status_code: 500, method: "POST" },
+          },
           { message: "open /Users/a/secret.pdf", category: "console", level: "warning" },
         ],
       }),
       {},
     )!;
     expect(out.breadcrumbs).toEqual([
-      { message: "fetch <url>", category: "fetch" },
+      { message: "fetch <url>", category: "fetch", data: { status_code: 500, method: "POST" } },
       { message: "open <path>", category: "console", level: "warning" },
     ]);
   });