feat(analytics): instance census, full capture, richer error context (#511)

Add a once-per-boot instance_started event (arch, os, deploy_mode,
gpu_present) so the fleet architecture mix is measurable. It reuses the
existing per-instance instance_id and is exempt from the volume sample
rate, since a census that fires once per boot must not be thinned.

Restore useful capture depth now that the sponsored plan removes the
quota pressure behind the earlier hardening:

- PostHog sample rate 0.1 to 1.0 (full analytics when enabled); the
  property allowlist still blocks file data.
- Sentry per-instance ceiling 20 to 500/hr, breadcrumb trail restored
  (sanitized: urls/paths redacted, data payloads dropped), full stack
  paths kept; local vars, request bodies, and PII still dropped. Both
  api and web.

Honor ANALYTICS_ENABLED=false as an opt-out alias: it was documented on
the Docker Hub README but never wired in 2.x, so anyone who set it was
still tracked.

All capture stays behind the analytics opt-out gate.

apps/api/src/index.ts

@@ -5,7 +5,7 @@ import cors from "@fastify/cors";
 import rateLimit from "@fastify/rate-limit";
 import { trace } from "@opentelemetry/api";
 import { getDispatcherStatus, initDispatcher, isGpuAvailable } from "@snapotter/ai";
-import { APP_VERSION, SafeError } from "@snapotter/shared";
+import { ANALYTICS_EVENTS, APP_VERSION, SafeError } from "@snapotter/shared";
 import { eq, sql } from "drizzle-orm";
 import Fastify from "fastify";
 import { env } from "./config.js";
@@ -17,7 +17,7 @@ import { closeFlowProducer, closeQueueEvents, warmQueueEvents } from "./jobs/enq
 import { closeQueues, perPoolHealth, queueCounts } from "./jobs/queues.js";
 import { enqueueSystemJob, SYSTEM_JOBS, scheduleSystemJobs } from "./jobs/system-jobs.js";
 import { closeWorkers, startWorkers } from "./jobs/worker.js";
-import { initAnalytics, shutdownAnalytics } from "./lib/analytics.js";
+import { initAnalytics, shutdownAnalytics, trackEvent } from "./lib/analytics.js";
 import { shouldRunStartupCleanup } from "./lib/cleanup.js";
 import { buildCsp } from "./lib/csp.js";
 import { reportError } from "./lib/error-report.js";
@@ -27,6 +27,7 @@ import { logger } from "./lib/logger.js";
 import { requestDuration } from "./lib/metrics.js";
 import { getSettingString } from "./lib/settings-helpers.js";
 import { assertStorageWritable } from "./lib/storage-writable.js";
+import { gatherSystemProperties } from "./lib/system-info.js";
 import { requirePermission } from "./permissions.js";
 import {
   authMiddleware,
@@ -183,6 +184,11 @@ if (!env.COOKIE_SECRET) {
 await initAnalytics();
 const { primeAnalyticsGate } = await import("./lib/analytics-gate.js");
 await primeAnalyticsGate();
+// ignoreSampleRate: this once-per-boot census must not be thinned by the
+// volume sample rate that exists to throttle high-frequency usage events.
+await trackEvent(ANALYTICS_EVENTS.INSTANCE_STARTED, { ...gatherSystemProperties() }, undefined, {
+  ignoreSampleRate: true,
+});
 
 // Enterprise features (license-gated)
 let enterpriseLicense: { org: string; plan: string } | null = null;

apps/api/src/instrument.ts

@@ -1,6 +1,6 @@
-import { existsSync } from "node:fs";
 import { ANALYTICS_BAKED } from "@snapotter/shared";
 import { analyticsEnabled, gatePrimed, telemetryEnvKilled } from "./lib/analytics-gate.js";
+import { deployMode } from "./lib/deploy-mode.js";
 import { buildBeforeSend } from "./lib/sentry-scrub.js";
 
 // Sentry inits at process load, before the gate cache is primed. Until the
@@ -8,17 +8,6 @@ import { buildBeforeSend } from "./lib/sentry-scrub.js";
 // so an opted-out instance never reports even a boot-window crash.
 const sentryActive = () => gatePrimed() && analyticsEnabled();
 
-// All-in-one detection: docker/entrypoint.sh exports EMBEDDED_MODE=1 before
-// exec'ing s6-overlay, and the snapotter service run script is with-contenv,
-// so the marker reaches this process. URL absence is not a usable signal:
-// embedded mode sets loopback DATABASE_URL/REDIS_URL before boot, and native
-// dev commonly leaves DATABASE_URL unset (config.ts defaults it).
-function deployMode(): string {
-  if (process.env.EMBEDDED_MODE) return "embedded";
-  if (existsSync("/.dockerenv")) return "external";
-  return "native";
-}
-
 if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
   try {
     const Sentry = await import("@sentry/node");
@@ -41,8 +30,8 @@ if (ANALYTICS_BAKED.sentryDsn && !telemetryEnvKilled()) {
       // again (the July 2026 quota incident).
       integrations: [Sentry.httpIntegration({ trackIncomingRequestsAsSessions: false })],
       sendClientReports: false,
-      maxBreadcrumbs: 0,
-      beforeBreadcrumb: () => null,
+      // Capture the breadcrumb trail (default 100). beforeSend (sentry-scrub.ts)
+      // sanitizes each breadcrumb before send: urls/paths redacted, data dropped.
       initialScope: { tags: { deploy_mode: deployMode() } },
       beforeSend: buildBeforeSend(sentryActive) as unknown as SentryOptions["beforeSend"],
     });

apps/api/src/lib/analytics-allowlist.ts

@@ -12,6 +12,7 @@ const ALLOWED: Record<string, ReadonlySet<string>> = {
     "status",
   ]),
   ai_bundle_action: new Set(["bundle_id", "action", "duration_ms"]),
+  instance_started: new Set(["arch", "os_platform", "deploy_mode", "gpu_present"]),
 };
 
 function isAllowedValue(value: unknown): boolean {

apps/api/src/lib/analytics-gate.ts

@@ -25,10 +25,17 @@ async function defaultReader(): Promise<boolean | undefined> {
   return rows[0].value !== "false";
 }
 
-/** Runtime kill switch honored in ALL builds: SNAPOTTER_TELEMETRY=0|false|off. */
+/**
+ * Runtime kill switch honored in ALL builds: SNAPOTTER_TELEMETRY=0|false|off.
+ *
+ * ANALYTICS_ENABLED=false|0|off is honored as an alias. It is the opt-out
+ * variable historically documented on the Docker Hub README but never wired in
+ * 2.x, so a user who set it was still tracked. Treating it as a kill signal
+ * makes that documented opt-out genuinely stop egress.
+ */
 export function telemetryEnvKilled(): boolean {
-  const v = process.env.SNAPOTTER_TELEMETRY;
-  return v === "0" || v === "false" || v === "off";
+  const off = (v: string | undefined) => v === "0" || v === "false" || v === "off";
+  return off(process.env.SNAPOTTER_TELEMETRY) || off(process.env.ANALYTICS_ENABLED);
 }
 
 /** Compile-time bake, with a NON-PRODUCTION-only override so tests can force it on. */

apps/api/src/lib/analytics.ts

@@ -84,10 +84,14 @@ export async function trackEvent(
   event: string,
   properties: Record<string, unknown>,
   distinctId?: string,
+  // ignoreSampleRate bypasses the volume sample for low-frequency, high-value
+  // events (e.g. the once-per-boot instance_started census). It does NOT bypass
+  // the opt-out gate or the property allowlist below.
+  options?: { ignoreSampleRate?: boolean },
 ): Promise<void> {
   try {
     if (!analyticsEnabled() || !posthogClient) return;
-    if (ANALYTICS_BAKED.posthogSampleRate < 1.0) {
+    if (!options?.ignoreSampleRate && ANALYTICS_BAKED.posthogSampleRate < 1.0) {
       if (
         ANALYTICS_BAKED.posthogSampleRate <= 0.0 ||
         Math.random() >= ANALYTICS_BAKED.posthogSampleRate

apps/api/src/lib/deploy-mode.ts

@@ -0,0 +1,14 @@
+import { existsSync } from "node:fs";
+
+export type DeployMode = "embedded" | "external" | "native";
+
+// All-in-one detection: docker/entrypoint.sh exports EMBEDDED_MODE=1 before
+// exec'ing s6-overlay, and the snapotter service run script is with-contenv,
+// so the marker reaches this process. URL absence is not a usable signal:
+// embedded mode sets loopback DATABASE_URL/REDIS_URL before boot, and native
+// dev commonly leaves DATABASE_URL unset (config.ts defaults it).
+export function deployMode(): DeployMode {
+  if (process.env.EMBEDDED_MODE) return "embedded";
+  if (existsSync("/.dockerenv")) return "external";
+  return "native";
+}

apps/api/src/lib/sentry-scrub.ts

@@ -5,7 +5,10 @@
  */
 import { rebuildErrorValue } from "@snapotter/shared";
 
-const CEILING_PER_HOUR = 20;
+// Per-process runaway guard, not a quota lever: under the sponsored plan we want
+// real errors, but a single instance stuck in an error loop must not spam. Sentry
+// de-dupes by fingerprint server-side, so 500 distinct events/hour is ample.
+const CEILING_PER_HOUR = 500;
 const HOUR_MS = 3600_000;
 
 const TAG_ALLOWLIST = new Set([
@@ -21,9 +24,15 @@ const TAG_ALLOWLIST = new Set([
   "status_code",
 ]);
 
-function basename(p: string): string {
-  const i = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
-  return i >= 0 ? p.slice(i + 1) : p;
+const URL_RE = /https?:\/\/[^\s"')]+/g;
+const BLOB_RE = /blob:[^\s"')]+/g;
+// Absolute paths under roots that can hold user files (uploads live in /data,
+// /tmp) or dev machines (/Users, /home). Over-redaction of a benign path is fine.
+const PATH_RE = /(?:\/(?:Users|home|root|data|tmp|var|app|opt|mnt|srv)|[A-Za-z]:\\)[^\s"')]*/g;
+
+/** Redact urls, blob refs, and absolute paths from free text (breadcrumb messages). */
+function scrubText(s: string): string {
+  return s.replace(BLOB_RE, "<blob>").replace(URL_RE, "<url>").replace(PATH_RE, "<path>");
 }
 
 // Sentry event/hint are typed loosely on purpose: this module must not import
@@ -38,6 +47,30 @@ function asObj(value: unknown): AnyEvent | null {
     : null;
 }
 
+// Keep the breadcrumb trail (the sequence of operations before the error) but
+// strip content that can carry user data: redact urls/paths from the message and
+// drop the structured `data` payload (http urls, query params) entirely.
+function scrubBreadcrumb(entry: unknown): AnyEvent | null {
+  const b = asObj(entry);
+  if (!b) return null;
+  const out: AnyEvent = {};
+  for (const k of ["type", "category", "level", "timestamp"]) {
+    if (b[k] !== undefined) out[k] = b[k];
+  }
+  if (typeof b.message === "string") out.message = scrubText(b.message);
+  return out;
+}
+
+/** Sanitize the breadcrumb list, tolerating both the array and {values} shapes. */
+function scrubBreadcrumbs(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(scrubBreadcrumb).filter(Boolean);
+  const wrapped = asObj(value);
+  if (Array.isArray(wrapped?.values)) {
+    return { values: wrapped.values.map(scrubBreadcrumb).filter(Boolean) };
+  }
+  return undefined;
+}
+
 export function buildBeforeSend(isActive: () => boolean) {
   let windowStart = 0;
   let sentInWindow = 0;
@@ -52,13 +85,15 @@ export function buildBeforeSend(isActive: () => boolean) {
     }
     if (++sentInWindow > CEILING_PER_HOUR) return null;
 
+    // Dropped: these surfaces can carry user data or PII.
     event.message = undefined;
     event.logentry = undefined;
     event.server_name = undefined;
     event.request = undefined;
     event.extra = undefined;
-    event.breadcrumbs = undefined;
     event.user = undefined;
+    // Kept, sanitized: the operation trail leading up to the error.
+    event.breadcrumbs = scrubBreadcrumbs(event.breadcrumbs);
 
     const ctx = asObj(event.contexts);
     const keep: AnyEvent = {};
@@ -88,8 +123,8 @@ export function buildBeforeSend(isActive: () => boolean) {
           for (const entry of frames) {
             const frame = asObj(entry);
             if (!frame) continue;
-            if (typeof frame.filename === "string") frame.filename = basename(frame.filename);
-            frame.abs_path = undefined;
+            // Keep filename + abs_path (open-source code paths, not user data);
+            // drop only vars, which can hold user file contents or secrets.
             frame.vars = undefined;
           }
         }

apps/api/src/lib/system-info.ts

@@ -0,0 +1,23 @@
+import { existsSync } from "node:fs";
+import os from "node:os";
+import type { InstanceStartedProperties } from "@snapotter/shared";
+import { deployMode } from "./deploy-mode.js";
+
+// Facts about the running instance, shipped once at boot as the
+// `instance_started` analytics event. Returning the shared event-property type
+// keeps this gatherer in lockstep with the event contract (single source of
+// truth) rather than duplicating the shape here.
+//
+// The NVIDIA Container Toolkit exposes GPU device nodes when a GPU is passed
+// through (docker run --gpus / compose deploy.resources.reservations), so a
+// filesystem check is enough; no nvidia-smi spawn or AI dispatcher needed.
+// This is deliberately independent of packages/ai's isGpuAvailable(), which
+// only reflects hardware in use by an already-started dispatcher.
+export function gatherSystemProperties(): InstanceStartedProperties {
+  return {
+    arch: process.arch === "arm64" ? "arm64" : "amd64",
+    os_platform: os.platform(),
+    deploy_mode: deployMode(),
+    gpu_present: existsSync("/dev/nvidia0"),
+  };
+}