feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.

tests/unit/api/bullmq-context-propagation.test.ts

@@ -0,0 +1,145 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import {
+  type Context,
+  type ContextManager,
+  context,
+  propagation,
+  ROOT_CONTEXT,
+  trace,
+} from "@opentelemetry/api";
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { afterEach, describe, expect, it } from "vitest";
+
+/**
+ * Minimal AsyncLocalStorage-based context manager for tests.
+ * OTel v2 BasicTracerProvider no longer registers one automatically.
+ */
+class TestContextManager implements ContextManager {
+  private _als = new AsyncLocalStorage<Context>();
+
+  active(): Context {
+    return this._als.getStore() ?? ROOT_CONTEXT;
+  }
+
+  with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(
+    ctx: Context,
+    fn: F,
+    thisArg?: ThisParameterType<F>,
+    ...args: A
+  ): ReturnType<F> {
+    return this._als.run(ctx, () => fn.call(thisArg, ...args));
+  }
+
+  bind<T>(_ctx: Context, target: T): T {
+    return target;
+  }
+
+  enable(): this {
+    return this;
+  }
+
+  disable(): this {
+    this._als.disable();
+    return this;
+  }
+}
+
+describe("BullMQ trace context injection", () => {
+  afterEach(() => {
+    propagation.disable();
+    context.disable();
+    trace.disable();
+  });
+
+  it("injects _otel with traceparent when a span is active", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    context.setGlobalContextManager(new TestContextManager());
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    const tracer = trace.getTracer("test");
+    const span = tracer.startSpan("http-request");
+    const ctx = trace.setSpan(ROOT_CONTEXT, span);
+
+    const carrier: Record<string, string> = {};
+    context.with(ctx, () => {
+      propagation.inject(context.active(), carrier);
+    });
+
+    expect(carrier.traceparent).toBeDefined();
+    expect(carrier.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$/);
+
+    // Verify traceparent contains the correct traceId and spanId
+    const parts = carrier.traceparent!.split("-");
+    expect(parts[1]).toBe(span.spanContext().traceId);
+    expect(parts[2]).toBe(span.spanContext().spanId);
+
+    span.end();
+    await provider.shutdown();
+  });
+
+  it("injects nothing when no SDK is registered", () => {
+    const carrier: Record<string, string> = {};
+    propagation.inject(context.active(), carrier);
+    expect(carrier.traceparent).toBeUndefined();
+  });
+});
+
+describe("BullMQ trace context extraction", () => {
+  afterEach(() => {
+    propagation.disable();
+    context.disable();
+    trace.disable();
+  });
+
+  it("extracts parent context from _otel carrier", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    context.setGlobalContextManager(new TestContextManager());
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    // Create a parent span and inject its context
+    const tracer = trace.getTracer("test");
+    const parentSpan = tracer.startSpan("parent-request");
+    const parentCtx = trace.setSpan(ROOT_CONTEXT, parentSpan);
+
+    const carrier: Record<string, string> = {};
+    context.with(parentCtx, () => {
+      propagation.inject(context.active(), carrier);
+    });
+
+    // Extract context from the carrier (simulating worker side)
+    const extractedCtx = propagation.extract(ROOT_CONTEXT, carrier);
+    const childSpan = tracer.startSpan("worker-process", undefined, extractedCtx);
+
+    // Child should share traceId but have a different spanId
+    expect(childSpan.spanContext().traceId).toBe(parentSpan.spanContext().traceId);
+    expect(childSpan.spanContext().spanId).not.toBe(parentSpan.spanContext().spanId);
+
+    parentSpan.end();
+    childSpan.end();
+    await provider.shutdown();
+  });
+
+  it("creates root span when _otel is absent", () => {
+    const extractedCtx = propagation.extract(ROOT_CONTEXT, {});
+    const span = trace.getSpan(extractedCtx);
+    expect(span).toBeUndefined();
+  });
+});

tests/unit/api/enterprise-flags.test.ts

@@ -32,7 +32,13 @@ describe("enterprise feature flags", () => {
     }
   });
 
-  it("has exactly 18 features total", () => {
-    expect(ENTERPRISE_FEATURES).toHaveLength(18);
+  it("has exactly 19 features total", () => {
+    expect(ENTERPRISE_FEATURES).toHaveLength(19);
+  });
+
+  it("includes distributed_tracing in enterprise plan only", () => {
+    expect(ENTERPRISE_FEATURES).toContain("distributed_tracing");
+    expect(PLAN_FEATURES.enterprise).toContain("distributed_tracing");
+    expect(PLAN_FEATURES.team).not.toContain("distributed_tracing");
   });
 });

tests/unit/api/log-trace-mixin.test.ts

@@ -0,0 +1,84 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import {
+  type Context,
+  type ContextManager,
+  context,
+  ROOT_CONTEXT,
+  trace,
+} from "@opentelemetry/api";
+import { afterEach, describe, expect, it } from "vitest";
+import { traceMixin } from "../../../apps/api/src/lib/log-trace-mixin.js";
+
+/**
+ * Minimal AsyncLocalStorage-based context manager for tests.
+ * OTel v2 BasicTracerProvider no longer registers one automatically.
+ */
+class TestContextManager implements ContextManager {
+  private _als = new AsyncLocalStorage<Context>();
+
+  active(): Context {
+    return this._als.getStore() ?? ROOT_CONTEXT;
+  }
+
+  with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(
+    ctx: Context,
+    fn: F,
+    thisArg?: ThisParameterType<F>,
+    ...args: A
+  ): ReturnType<F> {
+    return this._als.run(ctx, () => fn.call(thisArg, ...args));
+  }
+
+  bind<T>(_ctx: Context, target: T): T {
+    return target;
+  }
+
+  enable(): this {
+    return this;
+  }
+
+  disable(): this {
+    this._als.disable();
+    return this;
+  }
+}
+
+describe("traceMixin", () => {
+  afterEach(() => {
+    context.disable();
+    trace.disable();
+  });
+
+  it("returns empty object when no span is active", () => {
+    const result = traceMixin();
+    expect(result).toEqual({});
+  });
+
+  it("returns traceId, spanId, traceFlags when span is active", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    // Register a real context manager so context.with() propagates
+    context.setGlobalContextManager(new TestContextManager());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    const tracer = trace.getTracer("test");
+    const span = tracer.startSpan("test-span");
+    const ctx = trace.setSpan(ROOT_CONTEXT, span);
+
+    const result = context.with(ctx, () => traceMixin());
+
+    expect(result.traceId).toBe(span.spanContext().traceId);
+    expect(result.spanId).toBe(span.spanContext().spanId);
+    expect(result.traceFlags).toBe(span.spanContext().traceFlags);
+
+    span.end();
+    await provider.shutdown();
+  });
+});

tests/unit/api/sidecar-context-propagation.test.ts

@@ -0,0 +1,121 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import {
+  type Context,
+  type ContextManager,
+  context,
+  propagation,
+  ROOT_CONTEXT,
+  trace,
+} from "@opentelemetry/api";
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { afterEach, describe, expect, it } from "vitest";
+
+/**
+ * Minimal AsyncLocalStorage-based context manager for tests.
+ * OTel v2 BasicTracerProvider no longer registers one automatically.
+ */
+class TestContextManager implements ContextManager {
+  private _als = new AsyncLocalStorage<Context>();
+
+  active(): Context {
+    return this._als.getStore() ?? ROOT_CONTEXT;
+  }
+
+  with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(
+    ctx: Context,
+    fn: F,
+    thisArg?: ThisParameterType<F>,
+    ...args: A
+  ): ReturnType<F> {
+    return this._als.run(ctx, () => fn.call(thisArg, ...args));
+  }
+
+  bind<T>(_ctx: Context, target: T): T {
+    return target;
+  }
+
+  enable(): this {
+    return this;
+  }
+
+  disable(): this {
+    this._als.disable();
+    return this;
+  }
+}
+
+describe("sidecar trace context injection", () => {
+  afterEach(() => {
+    propagation.disable();
+    context.disable();
+    trace.disable();
+  });
+
+  it("produces _otel field when span is active", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    context.setGlobalContextManager(new TestContextManager());
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    const tracer = trace.getTracer("test");
+    const span = tracer.startSpan("sidecar-call");
+    const ctx = trace.setSpan(ROOT_CONTEXT, span);
+
+    const message: Record<string, unknown> = context.with(ctx, () => {
+      const carrier: Record<string, string> = {};
+      propagation.inject(context.active(), carrier);
+
+      const msg: Record<string, unknown> = {
+        id: "test-123",
+        script: "remove_bg",
+        args: ["input.png"],
+      };
+      if (carrier.traceparent) {
+        msg._otel = {
+          traceparent: carrier.traceparent,
+          tracestate: carrier.tracestate,
+        };
+      }
+      return msg;
+    });
+
+    // _otel should be present with a valid traceparent
+    expect(message._otel).toBeDefined();
+    const otel = message._otel as { traceparent: string; tracestate?: string };
+    expect(otel.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$/);
+
+    // traceparent should contain the correct traceId and spanId
+    const parts = otel.traceparent.split("-");
+    expect(parts[1]).toBe(span.spanContext().traceId);
+    expect(parts[2]).toBe(span.spanContext().spanId);
+
+    // args array is untouched
+    expect(message.args).toEqual(["input.png"]);
+
+    span.end();
+    await provider.shutdown();
+  });
+
+  it("omits _otel when no SDK is registered", () => {
+    const carrier: Record<string, string> = {};
+    propagation.inject(context.active(), carrier);
+
+    const message: Record<string, unknown> = {
+      id: "test",
+      script: "remove_bg",
+      args: ["input.png"],
+    };
+    if (carrier.traceparent) {
+      message._otel = { traceparent: carrier.traceparent };
+    }
+    expect(message._otel).toBeUndefined();
+  });
+});

tests/unit/api/tracing-bootstrap.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+describe("tracing bootstrap", () => {
+  beforeEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  afterEach(async () => {
+    const { shutdownTracing } = await import("../../../apps/api/src/tracing.js");
+    await shutdownTracing();
+    vi.resetModules();
+  });
+
+  it("is inactive when OTEL_EXPORTER_OTLP_ENDPOINT is unset", async () => {
+    vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
+    const { isTracingActive } = await import("../../../apps/api/src/tracing.js");
+    expect(isTracingActive()).toBe(false);
+  });
+
+  it("is inactive when enterprise package is unavailable", async () => {
+    vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318");
+    vi.doMock("@snapotter/enterprise", () => {
+      throw new Error("Cannot find module '@snapotter/enterprise'");
+    });
+    const { isTracingActive } = await import("../../../apps/api/src/tracing.js");
+    expect(isTracingActive()).toBe(false);
+  });
+
+  it("initializes with valid config and enterprise license", async () => {
+    vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
+    vi.doMock("@snapotter/enterprise", () => ({
+      isFeatureEnabled: (f: string) => f === "distributed_tracing",
+      initEnterprise: () => true,
+      getActiveLicense: () => ({
+        plan: "enterprise",
+        features: ["distributed_tracing"],
+      }),
+    }));
+
+    const { InMemorySpanExporter } = await import("@opentelemetry/sdk-trace-base");
+    const exporter = new InMemorySpanExporter();
+    const { initTracing, isTracingActive } = await import("../../../apps/api/src/tracing.js");
+    await initTracing({ exporter });
+    expect(isTracingActive()).toBe(true);
+  });
+});