feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.

apps/api/src/routes/batch.ts

@@ -18,7 +18,7 @@ import sharp from "sharp";
 import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { recordChildOutcome } from "../jobs/batch-progress.js";
-import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
+import { getFlowProducer, injectTraceContext, waitForJob } from "../jobs/enqueue.js";
 import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
 import { autoOrient } from "../lib/auto-orient.js";
 import { getSecurityHeaders } from "../lib/csp.js";
@@ -39,6 +39,16 @@ interface ParsedFile {
   filename: string;
 }
 
+/** Recursively inject OTel trace context into every node of a FlowJob tree. */
+function injectTraceContextIntoFlow(node: FlowJob): void {
+  injectTraceContext(node.data as ToolJobData);
+  if (node.children) {
+    for (const child of node.children) {
+      injectTraceContextIntoFlow(child);
+    }
+  }
+}
+
 export async function registerBatchRoutes(app: FastifyInstance): Promise<void> {
   app.post(
     "/api/v1/tools/:toolId/batch",
@@ -297,6 +307,9 @@ export async function registerBatchRoutes(app: FastifyInstance): Promise<void> {
         .set({ settings: { flowChildCount: flowChildren.length } })
         .where(eq(schema.jobs.id, parentId));
 
+      // Inject OTel trace context into every node of the batch flow tree
+      injectTraceContextIntoFlow(batchTree);
+
       await getFlowProducer().add(batchTree);
 
       // ── Wait for completion and stream ZIP ─────────────────────────

apps/api/src/routes/pipeline.ts

@@ -17,7 +17,7 @@ import { z } from "zod";
 import { env } from "../config.js";
 import { db, schema } from "../db/index.js";
 import { recordChildOutcome } from "../jobs/batch-progress.js";
-import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
+import { getFlowProducer, injectTraceContext, waitForJob } from "../jobs/enqueue.js";
 import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
 import { trackEvent } from "../lib/analytics.js";
 import { autoOrient } from "../lib/auto-orient.js";
@@ -78,6 +78,16 @@ interface ParsedStep {
  */
 const PASSWORD_TOOLS = new Set(["protect-pdf", "unlock-pdf"]);
 
+/** Recursively inject OTel trace context into every node of a FlowJob tree. */
+function injectTraceContextIntoFlow(node: FlowJob): void {
+  injectTraceContext(node.data as ToolJobData);
+  if (node.children) {
+    for (const child of node.children) {
+      injectTraceContextIntoFlow(child);
+    }
+  }
+}
+
 /**
  * Build a FlowJob tree for a single-file pipeline.
  *
@@ -405,6 +415,9 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
       settings: {},
     });
 
+    // Inject OTel trace context into every node of the flow tree
+    injectTraceContextIntoFlow(tree);
+
     // Add the flow to BullMQ
     await getFlowProducer().add(tree);
 
@@ -913,6 +926,9 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
       .set({ settings: { flowChildCount: perFileChildren.length } })
       .where(eq(schema.jobs.id, parentId));
 
+    // Inject OTel trace context into every node of the batch flow tree
+    injectTraceContextIntoFlow(batchTree);
+
     await getFlowProducer().add(batchTree);
 
     // Wait for batch completion