feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.

apps/api/src/index.ts

@@ -1,9 +1,9 @@
 import { randomUUID } from "node:crypto";
 import { statfs } from "node:fs/promises";
-import { join } from "node:path";
 import cookie from "@fastify/cookie";
 import cors from "@fastify/cors";
 import rateLimit from "@fastify/rate-limit";
+import { trace } from "@opentelemetry/api";
 import { getDispatcherStatus, initDispatcher, isGpuAvailable } from "@snapotter/ai";
 import { APP_VERSION } from "@snapotter/shared";
 import { eq, sql } from "drizzle-orm";
@@ -21,7 +21,7 @@ import { captureException, initAnalytics, shutdownAnalytics } from "./lib/analyt
 import { shouldRunStartupCleanup } from "./lib/cleanup.js";
 import { buildCsp } from "./lib/csp.js";
 import { ensureAiDirs, recoverInterruptedInstalls } from "./lib/feature-status.js";
-
+import { logger } from "./lib/logger.js";
 import { requestDuration } from "./lib/metrics.js";
 import { getSettingString } from "./lib/settings-helpers.js";
 import { requirePermission } from "./permissions.js";
@@ -31,6 +31,7 @@ import {
   ensureAnonymousUser,
   ensureBuiltinRoles,
   ensureDefaultAdmin,
+  getAuthUser,
 } from "./plugins/auth.js";
 import { registerMfa } from "./plugins/mfa.js";
 import { oidcRoutes } from "./plugins/oidc.js";
@@ -56,6 +57,7 @@ import { settingsRoutes } from "./routes/settings.js";
 import { teamsRoutes } from "./routes/teams.js";
 import { registerToolRoutes } from "./routes/tools/index.js";
 import { userFileRoutes } from "./routes/user-files.js";
+import { shutdownTracing } from "./tracing.js";
 
 // Run before anything else
 try {
@@ -188,26 +190,7 @@ function parseTrustProxy(value: string): boolean | number | string {
 
 const app = Fastify({
   genReqId: (req) => (req.headers["x-request-id"] as string) ?? randomUUID(),
-  logger: {
-    level: env.LOG_LEVEL,
-    transport: {
-      targets: [
-        { target: "pino/file", options: { destination: 1 } },
-        {
-          // Rotate at 10 MB, keep 5 files
-          target: "pino-roll",
-          options: {
-            file: join(env.LOG_DIR, "snapotter"),
-            extension: ".log",
-            size: "10m",
-            limit: { count: 5 },
-            mkdir: true,
-          },
-        },
-      ],
-    },
-    redact: ["req.headers.authorization", "req.headers.cookie"],
-  },
+  loggerInstance: logger,
   bodyLimit: env.MAX_UPLOAD_SIZE_MB > 0 ? env.MAX_UPLOAD_SIZE_MB * 1024 * 1024 : 1073741824,
   trustProxy: parseTrustProxy(env.TRUST_PROXY),
   routerOptions: { maxParamLength: 500 },
@@ -330,6 +313,18 @@ await authMiddleware(app);
 // Per-user rate limiting (after auth so request.user is populated)
 await registerPerUserRateLimit(app);
 
+// Enrich active OTel span with tool_id and user_id when available
+app.addHook("preHandler", (request, _reply, done) => {
+  const span = trace.getActiveSpan();
+  if (span) {
+    const params = request.params as Record<string, string> | undefined;
+    if (params?.toolId) span.setAttribute("snapotter.tool_id", params.toolId);
+    const user = getAuthUser(request);
+    if (user) span.setAttribute("snapotter.user_id", user.id);
+  }
+  done();
+});
+
 // Auth routes
 await authRoutes(app);
 
@@ -668,6 +663,17 @@ async function shutdown(signal: string) {
   // Close BullMQ resources before database (workers first so no new jobs start)
   try {
     await closeWorkers();
+  } catch (err) {
+    console.error("Error closing workers:", err);
+  }
+
+  try {
+    await shutdownTracing();
+  } catch {
+    // tracing shutdown is best-effort
+  }
+
+  try {
     await closeFlowProducer();
     await closeQueueEvents();
     await closeQueues();