feat: pipeline templates, analytics opt-out, 83 conversion presets, positioning + e2e modernization

Lands five integrated branches: pipeline templates (#355), analytics opt-out (#354), 83 conversion presets bringing the catalog to 240 tools (#356), self-hosted positioning (#353), and e2e modernization (#351).

Integration fixes: aligned stale web analytics tests with the opt-out/allow-list model, closed 3 CodeQL incomplete-sanitization alerts in the i18n generator, resolved settings/index/docs/format-matrix conflicts, and corrected tool counts to 240.

apps/api/src/lib/analytics-allowlist.ts

@@ -0,0 +1,36 @@
+// Defense in depth: only these keys may leave the server per event, and only as
+// primitives. Free-text fields (error_message, params, search query) are never
+// allow-listed, so tool settings and filenames cannot reach PostHog.
+const ALLOWED: Record<string, ReadonlySet<string>> = {
+  tool_used: new Set(["tool_id", "status", "duration_ms", "category", "is_ai_tool", "error_code"]),
+  pipeline_executed: new Set([
+    "step_count",
+    "tool_ids",
+    "is_batch",
+    "file_count",
+    "duration_ms",
+    "status",
+  ]),
+  ai_bundle_action: new Set(["bundle_id", "action", "duration_ms"]),
+};
+
+function isAllowedValue(value: unknown): boolean {
+  if (value === null) return false;
+  const t = typeof value;
+  if (t === "string" || t === "number" || t === "boolean") return true;
+  // tool_ids is an array of strings (low-cardinality ids); allow that one shape.
+  return Array.isArray(value) && value.every((v) => typeof v === "string");
+}
+
+export function sanitizeEventProperties(
+  event: string,
+  properties: Record<string, unknown>,
+): Record<string, unknown> {
+  const allow = ALLOWED[event];
+  if (!allow) return {};
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(properties)) {
+    if (allow.has(key) && isAllowedValue(value)) out[key] = value;
+  }
+  return out;
+}

apps/api/src/lib/analytics-gate.ts

@@ -0,0 +1,118 @@
+import { ANALYTICS_BAKED } from "@snapotter/shared";
+import type Redis from "ioredis";
+
+const TTL_MS = 30_000;
+const SETTING_KEY = "analyticsEnabled";
+
+// `undefined` from a reader means the key is absent (default ON).
+type GateReader = () => Promise<boolean | undefined>;
+
+let cachedEnabled = true; // last known toggle value (default ON)
+let knownDisabled = false; // have we positively read "disabled"? fail-closed anchor
+let primed = false; // has a successful read happened yet? gates Sentry at cold start
+let fetchedAt = 0; // Date.now() of the last read attempt
+let reader: GateReader = defaultReader;
+
+async function defaultReader(): Promise<boolean | undefined> {
+  const { db, schema } = await import("../db/index.js");
+  const { eq } = await import("drizzle-orm");
+  const rows = await db
+    .select({ value: schema.settings.value })
+    .from(schema.settings)
+    .where(eq(schema.settings.key, SETTING_KEY))
+    .limit(1);
+  if (rows.length === 0) return undefined;
+  return rows[0].value !== "false";
+}
+
+/** Compile-time bake, with a NON-PRODUCTION-only override so tests can force it on. */
+export function bakedEnabled(): boolean {
+  if (process.env.NODE_ENV !== "production") {
+    const o = process.env.ANALYTICS_BAKED_OVERRIDE;
+    if (o === "on") return true;
+    if (o === "off") return false;
+  }
+  return ANALYTICS_BAKED.enabled;
+}
+
+/** Synchronous effective gate. Safe to call from Sentry beforeSend. Never blocks. */
+export function analyticsEnabled(): boolean {
+  if (!bakedEnabled()) return false;
+  if (Date.now() - fetchedAt > TTL_MS) {
+    void refreshAnalyticsGate(); // background refresh; serve the cached value now
+  }
+  return cachedEnabled;
+}
+
+/** Read the toggle and update the cache. Fails closed on read error. */
+export async function refreshAnalyticsGate(): Promise<void> {
+  try {
+    const v = await reader();
+    const on = v === undefined ? true : v;
+    cachedEnabled = on;
+    knownDisabled = !on;
+    primed = true;
+    fetchedAt = Date.now();
+  } catch {
+    // DB read failed. If we ever positively saw "disabled", keep serving disabled
+    // rather than reverting to the ON default. Otherwise keep the last value.
+    if (knownDisabled) cachedEnabled = false;
+    fetchedAt = Date.now(); // do not hammer the DB on repeated errors
+  }
+}
+
+/** Warm the cache at boot before traffic is served. */
+export async function primeAnalyticsGate(): Promise<void> {
+  await refreshAnalyticsGate();
+}
+
+/**
+ * True once a successful read has populated the cache. Backend Sentry inits at
+ * process load (before the cache is primed), so its hooks check this to stay
+ * silent during the boot window rather than emitting on the default-ON cache.
+ */
+export function gatePrimed(): boolean {
+  return primed;
+}
+
+// Test seams (no-ops in production paths).
+export function __setReaderForTests(r: GateReader | null): void {
+  reader = r ?? defaultReader;
+}
+export function __resetGateForTests(): void {
+  cachedEnabled = true;
+  knownDisabled = false;
+  primed = false;
+  fetchedAt = 0;
+  reader = defaultReader;
+}
+
+let gateSubscriber: Redis | null = null;
+const CHANNEL = async () => {
+  const { bullPrefix } = await import("../jobs/types.js");
+  return `${bullPrefix()}:analytics-gate`;
+};
+
+/** Subscribe so a setting change on any replica refreshes this process's cache. */
+export async function startAnalyticsGateListener(): Promise<void> {
+  const { createRedisConnection } = await import("../jobs/connection.js");
+  gateSubscriber = createRedisConnection();
+  gateSubscriber.on("error", (err) => console.error("Analytics gate subscriber error", err));
+  await gateSubscriber.subscribe(await CHANNEL());
+  gateSubscriber.on("message", () => {
+    void refreshAnalyticsGate();
+  });
+}
+
+/** Publish so every replica drops its cache after a toggle write. */
+export async function publishAnalyticsGateInvalidation(): Promise<void> {
+  const { sharedRedis } = await import("../jobs/connection.js");
+  await sharedRedis().publish(await CHANNEL(), "1");
+}
+
+export async function stopAnalyticsGateListener(): Promise<void> {
+  if (gateSubscriber) {
+    await gateSubscriber.quit();
+    gateSubscriber = null;
+  }
+}

apps/api/src/lib/analytics.ts

@@ -2,11 +2,13 @@ import { ANALYTICS_BAKED } from "@snapotter/shared";
 import { eq } from "drizzle-orm";
 import type { PostHog } from "posthog-node";
 import { db, schema } from "../db/index.js";
+import { sanitizeEventProperties } from "./analytics-allowlist.js";
+import { analyticsEnabled, bakedEnabled } from "./analytics-gate.js";
 
 let posthogClient: PostHog | null = null;
 
 export async function initAnalytics(): Promise<void> {
-  if (!ANALYTICS_BAKED.enabled) return;
+  if (!bakedEnabled()) return;
 
   if (ANALYTICS_BAKED.posthogApiKey) {
     try {
@@ -24,7 +26,7 @@ export async function initAnalytics(): Promise<void> {
 
 export async function captureException(error: unknown): Promise<void> {
   try {
-    if (!ANALYTICS_BAKED.enabled) return;
+    if (!analyticsEnabled()) return;
     const Sentry = await import("@sentry/node");
     Sentry.captureException(error);
   } catch {
@@ -53,14 +55,14 @@ export async function trackEvent(
   distinctId?: string,
 ): Promise<void> {
   try {
-    if (!ANALYTICS_BAKED.enabled || !posthogClient) return;
+    if (!analyticsEnabled() || !posthogClient) return;
     if (ANALYTICS_BAKED.sampleRate < 1.0) {
       if (ANALYTICS_BAKED.sampleRate <= 0.0 || Math.random() >= ANALYTICS_BAKED.sampleRate) return;
     }
     posthogClient.capture({
       distinctId: distinctId ?? (await getInstanceId()),
       event,
-      properties,
+      properties: sanitizeEventProperties(event, properties),
     });
   } catch {
     // analytics must never throw

apps/api/src/lib/env.ts

@@ -33,6 +33,7 @@ const envSchema = z
     CONCURRENT_JOBS: z.coerce.number().default(0),
     MAX_MEGAPIXELS: z.coerce.number().default(0),
     RATE_LIMIT_PER_MIN: z.coerce.number().default(300),
+    API_KEYS_RATE_LIMIT_PER_MIN: z.coerce.number().default(30),
     DATABASE_URL: z.string().default("postgres://snapotter:snapotter@localhost:5432/snapotter"),
     SQLITE_MIGRATE_PATH: z.string().default(""),
     FILES_STORAGE_PATH: z.string().default("./data/files"),

apps/api/src/lib/heic-converter.ts

@@ -52,7 +52,9 @@ async function findDecodeCmd(): Promise<string> {
  */
 export async function decodeHeic(buffer: Buffer): Promise<Buffer> {
   const cmd = await findDecodeCmd();
-  const id = randomUUID();
+  // Include the PID so concurrent processes (and test workers) write to
+  // distinct, attributable temp paths in the shared tmpdir.
+  const id = `${process.pid}-${randomUUID()}`;
   const inputPath = join(tmpdir(), `heic-in-${id}.heic`);
   const outputPath = join(tmpdir(), `heic-out-${id}.png`);
   const suffixedPath = outputPath.replace(/\.png$/, "-1.png");
@@ -101,7 +103,7 @@ export async function ensureSharpCompat(buffer: Buffer): Promise<Buffer> {
 }
 
 export async function encodeHeic(buffer: Buffer, quality = 80): Promise<Buffer> {
-  const id = randomUUID();
+  const id = `${process.pid}-${randomUUID()}`;
   const inputPath = join(tmpdir(), `heic-in-${id}.png`);
   const outputPath = join(tmpdir(), `heic-out-${id}.heic`);