feat(telemetry): Sentry + PostHog quality pass (#546)

Comprehensive telemetry quality improvements across Sentry and PostHog, grounded in an audit of the live data plus current best-practice research.

Sentry: job_id/instance_id tags, operational fingerprinting, PII-safe settings context on bug events, web tag population + extension-noise filtering, an early-crash buffer, http status/method kept on breadcrumbs, and a gated-off-by-default performance-tracing re-enable (tracesSampler that zeroes db/redis/queue-poll root spans + drops the Redis integration) with worker job spans and canonical-host cron monitors.

PostHog: history_change SPA pageviews, instance_id super property for fleet rollups, enriched tool_used (formats, byte sizes, is_batch, execution_hint, real error_kind taxonomy), the previously-dead result_saved/batch_processed/ai_bundle_prompted events fired, search click-through, editor + Automate authoring + auth instrumentation, a before_send PII boundary, and minimal opt-in landing-site pageviews.

apps/api/src/index.ts

@@ -20,7 +20,7 @@ import { closeWorkers, startWorkers } from "./jobs/worker.js";
 import { initAnalytics, shutdownAnalytics, trackEvent } from "./lib/analytics.js";
 import { shouldRunStartupCleanup } from "./lib/cleanup.js";
 import { buildCsp } from "./lib/csp.js";
-import { reportError } from "./lib/error-report.js";
+import { reportError, setSentryInstanceTag } from "./lib/error-report.js";
 import { stripInternalPaths } from "./lib/errors.js";
 import {
   acquireInstallLock,
@@ -180,17 +180,21 @@ if (env.AUTH_ENABLED) {
   await ensureAnonymousUser();
 }
 
-async function ensureInstanceId() {
+async function ensureInstanceId(): Promise<string> {
   const [existing] = await db
     .select()
     .from(schema.settings)
     .where(eq(schema.settings.key, "instance_id"));
-  if (!existing) {
-    await db.insert(schema.settings).values({ key: "instance_id", value: randomUUID() });
-  }
+  if (existing) return existing.value;
+  const value = randomUUID();
+  await db.insert(schema.settings).values({ key: "instance_id", value });
+  return value;
 }
 
-await ensureInstanceId();
+// Tag every Sentry event with the anonymized instance id so triage can tell one
+// broken install from the whole fleet, and cross-reference an event to the same
+// instance_id used by the PostHog server-side stream.
+await setSentryInstanceTag(await ensureInstanceId());
 
 async function ensureDefaultSettings() {
   const defaults: Record<string, string> = {

apps/api/src/instrument.ts

@@ -2,6 +2,7 @@ import { ANALYTICS_BAKED } from "@snapotter/shared";
 import { analyticsEnabled, gatePrimed, telemetryEnvKilled } from "./lib/analytics-gate.js";
 import { deployMode } from "./lib/deploy-mode.js";
 import { buildBeforeSend } from "./lib/sentry-scrub.js";
+import { buildTracesSampler } from "./lib/sentry-tracing.js";
 
 // Sentry inits at process load, before the gate cache is primed. Until the
 // first successful read, stay silent rather than emit on the default-ON cache,
@@ -20,15 +21,37 @@ if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
     // imports @sentry/node; cast at this one boundary to the SDK callback type.
     type SentryOptions = NonNullable<Parameters<typeof Sentry.init>[0]>;
 
+    // Performance tracing is OFF by default. The July 2026 quota incident was
+    // the default redis + postgres integrations turning BullMQ blocking polls
+    // and pg idle pings into transactions. Opt in with a positive
+    // SENTRY_TRACES_SAMPLE_RATE (e.g. 0.05): a tracesSampler then zeroes every
+    // standalone db/redis/queue-poll root span and the Redis integration is
+    // dropped, so that poll storm can never recur.
+    const tracesSampleRate = Number(process.env.SENTRY_TRACES_SAMPLE_RATE) || 0;
+    const tracingEnabled = tracesSampleRate > 0 && tracesSampleRate <= 1;
+
     Sentry.init({
       dsn: ANALYTICS_BAKED.sentryDsn,
       release,
       environment: process.env.SNAPOTTER_ENV || "production",
       sendDefaultPii: false,
-      // Errors only. No traces options are set at all, so the SDK never
-      // starts traces and BullMQ/pg idle polling can't become transactions
-      // again (the July 2026 quota incident).
-      integrations: [Sentry.httpIntegration({ trackIncomingRequestsAsSessions: false })],
+      // With tracing on, use the function form to DROP the default Redis
+      // integration (the array form is additive and would keep it). With
+      // tracing off, the array form is fine: no sampler means the defaults
+      // never start a transaction, so they stay inert.
+      integrations: tracingEnabled
+        ? (defaults) =>
+            defaults
+              .filter((i) => i.name !== "Redis")
+              .concat(Sentry.httpIntegration({ trackIncomingRequestsAsSessions: false }))
+        : [Sentry.httpIntegration({ trackIncomingRequestsAsSessions: false })],
+      ...(tracingEnabled
+        ? {
+            tracesSampler: buildTracesSampler(
+              tracesSampleRate,
+            ) as unknown as SentryOptions["tracesSampler"],
+          }
+        : {}),
       sendClientReports: false,
       // Capture the breadcrumb trail (default 100). beforeSend (sentry-scrub.ts)
       // sanitizes each breadcrumb before send: urls/paths redacted, data dropped.
@@ -36,7 +59,11 @@ if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
       beforeSend: buildBeforeSend(sentryActive) as unknown as SentryOptions["beforeSend"],
     });
 
-    console.log("[sentry] initialized (errors only), release:", release);
+    console.log(
+      tracingEnabled
+        ? `[sentry] initialized (errors + traces @ ${tracesSampleRate}), release: ${release}`
+        : `[sentry] initialized (errors only), release: ${release}`,
+    );
   } catch {
     // @sentry/node not available
   }

apps/api/src/jobs/system-jobs.ts

@@ -13,6 +13,7 @@ import type { Job } from "bullmq";
 import { and, eq, inArray, isNotNull, lt, sql } from "drizzle-orm";
 import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
+import { analyticsEnabled } from "../lib/analytics-gate.js";
 import { getMaxAgeMs } from "../lib/cleanup.js";
 import { deletePrefix, listJobDirs, type ObjectInfo } from "../lib/object-storage.js";
 import { getSettingNumber } from "../lib/settings-helpers.js";
@@ -67,9 +68,85 @@ export async function enqueueSystemJob(name: string): Promise<void> {
   await q.add(name, {} as never);
 }
 
+// -- Cron monitors (canonical host only) --------------------------------------
+
+// Sentry cron monitors for the repeatable sweeps, so a silently-dead scheduler
+// (disks fill, audit rows never prune) is caught. OFF by default: only the
+// maintainers' canonical hosted instance sets SENTRY_CRON_MONITORS=1, because
+// thousands of self-hosted installs checking into one org's monitors would make
+// a "missed" signal meaningless. High-frequency pollers (siemForward,
+// alertEvaluator) and on-demand jobs (gdprExport) are intentionally excluded.
+type MonitorSchedule =
+  | { type: "crontab"; value: string }
+  | { type: "interval"; value: number; unit: "minute" | "hour" | "day" };
+
+interface MonitorConfig {
+  schedule: MonitorSchedule;
+  checkinMargin: number;
+  maxRuntime: number;
+}
+
+const MONITOR_CONFIG: Record<string, MonitorConfig> = {
+  [SYSTEM_JOBS.storageTtl]: {
+    schedule: {
+      type: "interval",
+      value: Math.max(1, env.CLEANUP_INTERVAL_MINUTES),
+      unit: "minute",
+    },
+    checkinMargin: 5,
+    maxRuntime: 20,
+  },
+  [SYSTEM_JOBS.sessionPurge]: {
+    schedule: { type: "interval", value: 1, unit: "hour" },
+    checkinMargin: 10,
+    maxRuntime: 5,
+  },
+  [SYSTEM_JOBS.retention]: {
+    schedule: { type: "interval", value: 6, unit: "hour" },
+    checkinMargin: 15,
+    maxRuntime: 30,
+  },
+  [SYSTEM_JOBS.auditArchive]: {
+    schedule: { type: "crontab", value: "0 2 1 * *" },
+    checkinMargin: 30,
+    maxRuntime: 60,
+  },
+  [SYSTEM_JOBS.storageReconciliation]: {
+    schedule: { type: "crontab", value: "0 3 * * 0" },
+    checkinMargin: 30,
+    maxRuntime: 60,
+  },
+};
+
+function cronMonitorsEnabled(): boolean {
+  const v = process.env.SENTRY_CRON_MONITORS;
+  return (v === "1" || v === "true") && analyticsEnabled();
+}
+
+async function withCronMonitor(name: string, fn: () => Promise<unknown>): Promise<unknown> {
+  const cfg = MONITOR_CONFIG[name];
+  if (!cfg || !cronMonitorsEnabled()) return fn();
+  try {
+    const Sentry = await import("@sentry/node");
+    // Monitor slugs cannot contain ":".
+    return await Sentry.withMonitor(
+      name.replace(/:/g, "-"),
+      fn,
+      cfg as Parameters<typeof Sentry.withMonitor>[2],
+    );
+  } catch {
+    // Monitoring must never break the actual job.
+    return fn();
+  }
+}
+
 // -- Dispatcher ---------------------------------------------------------------
 
 export async function runSystemJob(job: Job): Promise<unknown> {
+  return withCronMonitor(job.name, () => dispatchSystemJob(job));
+}
+
+async function dispatchSystemJob(job: Job): Promise<unknown> {
   switch (job.name) {
     case SYSTEM_JOBS.storageTtl:
       return storageTtlSweep();

apps/api/src/jobs/worker.ts

@@ -27,6 +27,7 @@ import { join } from "node:path";
 import { context, propagation, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
 import {
   ANALYTICS_EVENTS,
+  extractErrorCode,
   getBundleForTool,
   getOptionalBundleForTool,
   isToolInputError,
@@ -39,7 +40,7 @@ import { db, schema } from "../db/index.js";
 import { trackEvent } from "../lib/analytics.js";
 import { analyticsEnabled } from "../lib/analytics-gate.js";
 import { resolveConcurrency } from "../lib/env.js";
-import { reportError, safeFormatTag } from "../lib/error-report.js";
+import { classifyError, reportError, safeFormatTag } from "../lib/error-report.js";
 import { friendlyError } from "../lib/errors.js";
 import { logger } from "../lib/logger.js";
 import { jobDuration, jobsTotal } from "../lib/metrics.js";
@@ -209,6 +210,27 @@ export function buildLegacyResultPayload(
 
 // ── Tool job processor ─────────────────────────────────────────
 
+/**
+ * Dimensions shared by both tool_used branches, kept in one place so the
+ * success and failure events never drift. input_format is the safe extension
+ * only (never the filename); is_batch flags a per-file batch child so batch
+ * volume is distinguishable from single runs; execution_hint records the tool's
+ * sync/async class.
+ */
+function toolUsedBaseProps(data: ToolJobData, durationMs: number): Record<string, unknown> {
+  const tool = TOOLS.find((t) => t.id === data.toolId);
+  return {
+    tool_id: data.toolId,
+    duration_ms: durationMs,
+    category: tool?.category ?? "unknown",
+    is_ai_tool:
+      getBundleForTool(data.toolId) !== null || getOptionalBundleForTool(data.toolId) !== null,
+    is_batch: data.kind === "batch-child",
+    input_format: safeFormatTag(data.filename) ?? "unknown",
+    execution_hint: tool?.executionHint ?? "fast",
+  };
+}
+
 async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
   const data = job.data;
   const { jobId } = data;
@@ -227,6 +249,11 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
             "snapotter.tool_id": data.toolId,
             "snapotter.pool": data.pool,
             "snapotter.attempt_number": job.attemptsMade + 1,
+            // Standard messaging semantics so Sentry (when tracing is enabled)
+            // labels this as a queue-consumer span and the tracesSampler
+            // recognizes a real job execution vs a poll.
+            "messaging.system": "bullmq",
+            "messaging.destination.name": queueName(data.pool),
           },
         },
         parentCtx,
@@ -451,17 +478,14 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
 
       // Analytics: emit tool_used on success
       if (analyticsEnabled()) {
-        const tool = TOOLS.find((t) => t.id === data.toolId);
         void trackEvent(
           ANALYTICS_EVENTS.TOOL_USED,
           {
-            tool_id: data.toolId,
+            ...toolUsedBaseProps(data, durationMs),
             status: "completed",
-            duration_ms: durationMs,
-            category: tool?.category ?? "unknown",
-            is_ai_tool:
-              getBundleForTool(data.toolId) !== null ||
-              getOptionalBundleForTool(data.toolId) !== null,
+            output_format: safeFormatTag(outName) ?? "unknown",
+            bytes_in: originalSize,
+            bytes_out: resultBuffer.length,
           },
           data.analyticsDistinctId,
         );
@@ -549,18 +573,25 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
 
       // Analytics: emit tool_used on failure
       if (analyticsEnabled()) {
-        const tool = TOOLS.find((t) => t.id === data.toolId);
+        const errorClass = isTimeout
+          ? "timeout"
+          : isCanceled
+            ? "cancelled"
+            : classifyError(err, "worker");
         void trackEvent(
           ANALYTICS_EVENTS.TOOL_USED,
           {
-            tool_id: data.toolId,
+            ...toolUsedBaseProps(data, durationMs),
             status: "failed",
-            duration_ms: durationMs,
-            category: tool?.category ?? "unknown",
-            is_ai_tool:
-              getBundleForTool(data.toolId) !== null ||
-              getOptionalBundleForTool(data.toolId) !== null,
-            error_code: isTimeout ? "timeout" : isCanceled ? "cancelled" : "processing",
+            error_code: isTimeout
+              ? "timeout"
+              : isCanceled
+                ? "cancelled"
+                : (extractErrorCode(err) ?? "processing"),
+            // A coarse, low-cardinality reason bucket so "why do tools fail" is
+            // answerable without leaking messages: bad input, environment, or
+            // our bug (classifyError's "expected" == bad user input).
+            error_kind: errorClass === "expected" ? "input" : errorClass,
           },
           data.analyticsDistinctId,
         );
@@ -1016,6 +1047,7 @@ export function startWorkers(): void {
           source: "worker",
           pool,
           toolId: (job.data as ToolJobData | undefined)?.toolId,
+          jobId: job.id,
         });
       });
 
@@ -1047,7 +1079,9 @@ export function startWorkers(): void {
         source: "worker",
         pool,
         toolId: (job.data as ToolJobData | undefined)?.toolId,
+        jobId: job.id,
         inputFormat: safeFormatTag((job.data as ToolJobData | undefined)?.filename),
+        settings: (job.data as ToolJobData | undefined)?.settings,
       });
     });
 

apps/api/src/lib/analytics-allowlist.ts

@@ -2,7 +2,21 @@
 // primitives. Free-text fields (error_message, params, search query) are never
 // allow-listed, so tool settings and filenames cannot reach PostHog.
 const ALLOWED: Record<string, ReadonlySet<string>> = {
-  tool_used: new Set(["tool_id", "status", "duration_ms", "category", "is_ai_tool", "error_code"]),
+  tool_used: new Set([
+    "tool_id",
+    "status",
+    "duration_ms",
+    "category",
+    "is_ai_tool",
+    "is_batch",
+    "input_format",
+    "output_format",
+    "bytes_in",
+    "bytes_out",
+    "execution_hint",
+    "error_code",
+    "error_kind",
+  ]),
   pipeline_executed: new Set([
     "step_count",
     "tool_ids",
@@ -13,6 +27,8 @@ const ALLOWED: Record<string, ReadonlySet<string>> = {
   ]),
   ai_bundle_action: new Set(["bundle_id", "action", "duration_ms"]),
   instance_started: new Set(["arch", "os_platform", "deploy_mode", "gpu_present"]),
+  auth_login: new Set(["method"]),
+  auth_login_failed: new Set(["method"]),
 };
 
 function isAllowedValue(value: unknown): boolean {

apps/api/src/lib/error-report.ts

@@ -37,6 +37,10 @@ export interface ReportContext {
   subsystem?: string;
   /** Safe input format (file extension) for triage; never the filename. */
   inputFormat?: string;
+  /** BullMQ job id, so the event cross-references the jobs DB row and logs. */
+  jobId?: string;
+  /** Tool settings; a vetted (PII-safe) projection is attached for bug events. */
+  settings?: unknown;
 }
 
 export function classifyError(err: unknown, source?: ReportContext["source"]): ErrorClass {
@@ -112,6 +116,33 @@ export function safeFormatTag(filename?: string): string | undefined {
   return m ? m[1].toLowerCase() : undefined;
 }
 
+// Setting keys whose values can carry user data (filenames, free text, secrets)
+// even when short, and which must never reach the Sentry `tool` context.
+const UNSAFE_SETTING_KEY =
+  /pass|secret|token|credential|auth|file|name|path|url|email|user|title|text|label|query|prompt|message|caption|content|watermark/i;
+const SAFE_ENUM_VALUE = /^[A-Za-z0-9_.-]{1,32}$/;
+
+/**
+ * A privacy-safe projection of tool settings for the Sentry `tool` context, so
+ * a bug-class event carries the knobs that triggered it (format, quality, mode,
+ * ...) without ever including filenames, free text, or secrets. Keeps only
+ * numbers, booleans, and short enum-like strings under non-sensitive keys;
+ * drops objects, arrays, and long strings entirely. Returns undefined when
+ * nothing safe survives.
+ */
+export function vetSettings(
+  settings: unknown,
+): Record<string, string | number | boolean> | undefined {
+  if (!settings || typeof settings !== "object" || Array.isArray(settings)) return undefined;
+  const out: Record<string, string | number | boolean> = {};
+  for (const [k, v] of Object.entries(settings as Record<string, unknown>)) {
+    if (UNSAFE_SETTING_KEY.test(k)) continue;
+    if (typeof v === "number" || typeof v === "boolean") out[k] = v;
+    else if (typeof v === "string" && SAFE_ENUM_VALUE.test(v)) out[k] = v;
+  }
+  return Object.keys(out).length ? out : undefined;
+}
+
 /** Fire-and-forget; never throws, never blocks. */
 export async function reportError(err: unknown, ctx: ReportContext): Promise<void> {
   try {
@@ -135,10 +166,42 @@ export async function reportError(err: unknown, ctx: ReportContext): Promise<voi
       if (ctx.method) scope.setTag("method", ctx.method);
       if (ctx.statusCode) scope.setTag("status_code", String(ctx.statusCode));
       if (ctx.subsystem) scope.setTag("subsystem", ctx.subsystem);
-      if (net) scope.setFingerprint(["connectivity", net]);
+      if (ctx.jobId) scope.setTag("job_id", ctx.jobId);
+      if (net) {
+        scope.setFingerprint(["connectivity", net]);
+      } else if (cls === "operational") {
+        // Collapse an operational class (bad DB creds, full disk, ...) into a
+        // single issue keyed on its code, instead of fragmenting into a
+        // separate issue per call site/stack frame (the pg-auth flood showed up
+        // as 5 issues). bug-class errors keep default per-frame grouping, where
+        // distinct frames usually are distinct bugs.
+        scope.setFingerprint(["operational", code ?? (err as { name?: string }).name ?? "op"]);
+      }
+      if (cls === "bug" && ctx.settings !== undefined) {
+        // Attach the knobs that triggered a bug (format, quality, mode, ...) so
+        // it is reproducible, without leaking filenames, free text, or secrets.
+        const vetted = vetSettings(ctx.settings);
+        if (vetted) scope.setContext("tool", vetted);
+      }
       Sentry.captureException(err instanceof Error ? err : new Error(String(err)));
     });
   } catch {
     // telemetry must never throw
   }
 }
+
+/**
+ * Set the anonymized instance id as a GLOBAL Sentry tag so every event (errors
+ * and SDK-captured uncaught exceptions) carries it. Lets triage tell "one
+ * broken install" from "the whole fleet" and cross-references a Sentry event to
+ * that instance's PostHog stream (same instance_id). Called once at boot.
+ */
+export async function setSentryInstanceTag(instanceId: string): Promise<void> {
+  try {
+    if (!instanceId) return;
+    const Sentry = await import("@sentry/node");
+    Sentry.getGlobalScope().setTag("instance_id", instanceId);
+  } catch {
+    // telemetry must never throw
+  }
+}

apps/api/src/lib/sentry-scrub.ts

@@ -23,6 +23,8 @@ const TAG_ALLOWLIST = new Set([
   "subsystem",
   "status_code",
   "input_format",
+  "job_id",
+  "instance_id",
 ]);
 
 const URL_RE = /https?:\/\/[^\s"')]+/g;
@@ -59,6 +61,16 @@ function scrubBreadcrumb(entry: unknown): AnyEvent | null {
     if (b[k] !== undefined) out[k] = b[k];
   }
   if (typeof b.message === "string") out.message = scrubText(b.message);
+  // For http breadcrumbs keep the non-PII status_code + method (the url is the
+  // sensitive part, dropped with the rest of `data`): they answer "what request
+  // failed right before the error".
+  if (b.category === "http") {
+    const data = asObj(b.data);
+    const safe: AnyEvent = {};
+    if (data?.status_code !== undefined) safe.status_code = data.status_code;
+    if (typeof data?.method === "string") safe.method = data.method;
+    if (Object.keys(safe).length) out.data = safe;
+  }
   return out;
 }
 
@@ -102,6 +114,17 @@ export function buildBeforeSend(isActive: () => boolean) {
     if (os?.name) keep.os = { name: os.name, version: os.version };
     const runtime = asObj(ctx?.runtime);
     if (runtime?.name) keep.runtime = { name: runtime.name, version: runtime.version };
+    // The `tool` context is set only by reportError from an already-vetted
+    // settings projection; re-enforce primitives-only here as a final boundary.
+    const tool = asObj(ctx?.tool);
+    if (tool) {
+      const safe: AnyEvent = {};
+      for (const [k, v] of Object.entries(tool)) {
+        if (typeof v === "number" || typeof v === "boolean") safe[k] = v;
+        else if (typeof v === "string" && v.length <= 32) safe[k] = v;
+      }
+      if (Object.keys(safe).length) keep.tool = safe;
+    }
     event.contexts = Object.keys(keep).length ? keep : undefined;
 
     const tags = asObj(event.tags);

apps/api/src/lib/sentry-tracing.ts

@@ -0,0 +1,52 @@
+/**
+ * Sentry performance-tracing sampler for the API, kept pure (no @sentry/node
+ * import) so it is unit-testable without the SDK.
+ *
+ * Tracing is OFF unless SENTRY_TRACES_SAMPLE_RATE is a positive number (see
+ * instrument.ts). The July 2026 quota incident came from the default redis and
+ * postgres integrations turning BullMQ blocking-poll commands and pg idle
+ * connection pings into standalone root transactions the moment any sampler was
+ * enabled. This sampler returns 0 for exactly those, three independent ways
+ * (op prefix, db.system, messaging.system + name), so a poll storm can never
+ * count against quota again, while real HTTP requests and named job executions
+ * still sample.
+ */
+export interface SamplingContext {
+  name?: string;
+  attributes?: Record<string, unknown>;
+  /** Sentry's helper: returns the parent's decision when there is one, else the fallback. */
+  inheritOrSampleWith?: (fallback: number) => number;
+}
+
+export function buildTracesSampler(httpRate: number): (ctx: SamplingContext) => number {
+  return (ctx: SamplingContext): number => {
+    const attrs = ctx.attributes ?? {};
+    const name = ctx.name ?? "";
+    const inherit = (fallback: number): number =>
+      typeof ctx.inheritOrSampleWith === "function" ? ctx.inheritOrSampleWith(fallback) : fallback;
+
+    // Standalone db / redis root transactions (pg idle pings, BullMQ blocking
+    // polls). The exact incident source: zero them outright.
+    const op = String(attrs["sentry.op"] ?? "");
+    if (op.startsWith("db")) return 0;
+    if (attrs["db.system"] === "redis") return 0;
+
+    // Any queue root span: only sample a real job execution we explicitly named
+    // "job <name>" (see worker.ts); drop the continuous poll transactions.
+    if (attrs["messaging.system"] !== undefined) {
+      // Sample a real job execution (the worker's "job.process" span, or a
+      // "job <name>" span); drop the continuous BullMQ poll transactions.
+      const isJob = name === "job.process" || name.startsWith("job ");
+      return isJob ? Math.min(httpRate, 0.2) : 0;
+    }
+
+    // Never sample infra endpoints, even though they are real HTTP.
+    if (/\/(healthz|readyz|metrics)\b/.test(name)) return 0;
+
+    // Real inbound HTTP: sample at the configured rate, honoring parent traces.
+    if (attrs["http.request.method"] !== undefined) return inherit(httpRate);
+
+    // Anything else: follow the parent decision, else drop.
+    return inherit(0);
+  };
+}

apps/api/src/plugins/auth.ts

@@ -1,11 +1,13 @@
 import { createHash, randomBytes, randomUUID, scrypt, timingSafeEqual } from "node:crypto";
 import { promisify } from "node:util";
+import { ANALYTICS_EVENTS } from "@snapotter/shared";
 import { and, asc, eq, ne, sql } from "drizzle-orm";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { z } from "zod";
 import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { sharedRedis } from "../jobs/connection.js";
+import { trackEvent } from "../lib/analytics.js";
 import { auditFromRequest, sanitizeAuditInput } from "../lib/audit.js";
 import { authAttempts } from "../lib/metrics.js";
 import { getSettingNumber, getSettingString } from "../lib/settings-helpers.js";
@@ -380,6 +382,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
         // response timing doesn't reveal whether the username exists.
         await verifyPassword(body.password, await getDummyHash());
         authAttempts.inc({ method: "password", result: "failure" });
+        void trackEvent(ANALYTICS_EVENTS.AUTH_LOGIN_FAILED, { method: "password" });
         await audit("LOGIN_FAILED", {
           username: sanitizeAuditInput(body.username),
           reason: "unknown_user",
@@ -390,6 +393,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
       const valid = await verifyPassword(body.password, user.passwordHash);
       if (!valid) {
         authAttempts.inc({ method: "password", result: "failure" });
+        void trackEvent(ANALYTICS_EVENTS.AUTH_LOGIN_FAILED, { method: "password" });
         await audit("LOGIN_FAILED", {
           username: sanitizeAuditInput(body.username),
           reason: "bad_password",
@@ -463,6 +467,7 @@ export async function authRoutes(app: FastifyInstance): Promise<void> {
       }
 
       authAttempts.inc({ method: "password", result: "success" });
+      void trackEvent(ANALYTICS_EVENTS.AUTH_LOGIN, { method: "password" });
       await audit("LOGIN_SUCCESS", { userId: user.id, username: user.username });
 
       const [teamRow] = await db.select().from(schema.teams).where(eq(schema.teams.id, user.team));

apps/api/src/plugins/oidc.ts

@@ -1,11 +1,13 @@
 import { randomUUID } from "node:crypto";
 import type {} from "@fastify/cookie";
+import { ANALYTICS_EVENTS } from "@snapotter/shared";
 import { eq } from "drizzle-orm";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import * as oidc from "openid-client";
 import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { sharedRedis } from "../jobs/connection.js";
+import { trackEvent } from "../lib/analytics.js";
 import { auditFromRequest, sanitizeAuditInput } from "../lib/audit.js";
 import { resolveExternalUser, sanitizeUsername } from "../lib/external-auth-resolver.js";
 import { authAttempts } from "../lib/metrics.js";
@@ -347,6 +349,7 @@ export async function oidcRoutes(app: FastifyInstance): Promise<void> {
       });
 
       authAttempts.inc({ method: "oidc", result: "success" });
+      void trackEvent(ANALYTICS_EVENTS.AUTH_LOGIN, { method: "oidc" });
       await audit("OIDC_LOGIN_SUCCESS", {
         userId: resolvedUser.id,
         username: resolvedUser.username,