feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag * feat(enterprise): add distributed_tracing feature gate * feat(tracing): add SDK bootstrap with enterprise gating * fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init Test 2 now mocks @snapotter/enterprise to throw an import error, exercising the catch block in the preload. Test 3 imports with no endpoint so the preload is a no-op, avoiding leaked SDK from double-initialization. Added idempotency guard to initTracing() as a safety net. * feat(tracing): add Pino trace mixin and shared logger When OTel tracing is active, every Pino log line now includes traceId, spanId, and traceFlags fields for log-to-trace correlation. The mixin is a no-op when no SDK is registered (community users). * feat(tracing): add _otel to ToolJobData and inject trace context at enqueue Add optional _otel carrier field to ToolJobData for W3C trace context propagation across BullMQ job boundaries. When an active OTel span exists, propagation.inject() writes traceparent/tracestate into the job data before queue.add(). When no SDK is registered (community edition), the carrier stays empty and _otel remains undefined -- zero overhead. * feat(tracing): extract trace context and create spans in BullMQ worker * feat(tracing): inject trace context into Python sidecar calls * feat(tracing): add trace context extraction to Python sidecar * feat(tracing): add shutdownTracing to graceful shutdown sequence * feat(tracing): enrich HTTP spans with tool_id and user_id attributes * docs: add OpenTelemetry env var documentation to .env.example * test(tracing): add lifecycle integration tests for trace propagation * fix(tracing): inject trace context into pipeline and batch flow jobs * fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment Wraps PythonDispatcher.run() with a sidecar.execute span on the Node side so traces show the full round-trip (Node span -> Python span). Also removes an obvious comment from logger.ts.
2026-08-03 07:46:42 +02:00 · 2026-06-15 12:53:06 +08:00
parent 5af1ac4dcf
commit 3fb8164fa5
25 changed files with 1968 additions and 320 deletions
@@ -0,0 +1,145 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import {
+  type Context,
+  type ContextManager,
+  context,
+  propagation,
+  ROOT_CONTEXT,
+  trace,
+} from "@opentelemetry/api";
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { afterEach, describe, expect, it } from "vitest";
+
+/**
+ * Minimal AsyncLocalStorage-based context manager for tests.
+ * OTel v2 BasicTracerProvider no longer registers one automatically.
+ */
+class TestContextManager implements ContextManager {
+  private _als = new AsyncLocalStorage<Context>();
+
+  active(): Context {
+    return this._als.getStore() ?? ROOT_CONTEXT;
+  }
+
+  with<A extends unknown[], F extends (...args: A) => ReturnType<F>>(
+    ctx: Context,
+    fn: F,
+    thisArg?: ThisParameterType<F>,
+    ...args: A
+  ): ReturnType<F> {
+    return this._als.run(ctx, () => fn.call(thisArg, ...args));
+  }
+
+  bind<T>(_ctx: Context, target: T): T {
+    return target;
+  }
+
+  enable(): this {
+    return this;
+  }
+
+  disable(): this {
+    this._als.disable();
+    return this;
+  }
+}
+
+describe("BullMQ trace context injection", () => {
+  afterEach(() => {
+    propagation.disable();
+    context.disable();
+    trace.disable();
+  });
+
+  it("injects _otel with traceparent when a span is active", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    context.setGlobalContextManager(new TestContextManager());
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    const tracer = trace.getTracer("test");
+    const span = tracer.startSpan("http-request");
+    const ctx = trace.setSpan(ROOT_CONTEXT, span);
+
+    const carrier: Record<string, string> = {};
+    context.with(ctx, () => {
+      propagation.inject(context.active(), carrier);
+    });
+
+    expect(carrier.traceparent).toBeDefined();
+    expect(carrier.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$/);
+
+    // Verify traceparent contains the correct traceId and spanId
+    const parts = carrier.traceparent!.split("-");
+    expect(parts[1]).toBe(span.spanContext().traceId);
+    expect(parts[2]).toBe(span.spanContext().spanId);
+
+    span.end();
+    await provider.shutdown();
+  });
+
+  it("injects nothing when no SDK is registered", () => {
+    const carrier: Record<string, string> = {};
+    propagation.inject(context.active(), carrier);
+    expect(carrier.traceparent).toBeUndefined();
+  });
+});
+
+describe("BullMQ trace context extraction", () => {
+  afterEach(() => {
+    propagation.disable();
+    context.disable();
+    trace.disable();
+  });
+
+  it("extracts parent context from _otel carrier", async () => {
+    const { InMemorySpanExporter, SimpleSpanProcessor, BasicTracerProvider } = await import(
+      "@opentelemetry/sdk-trace-base"
+    );
+
+    context.setGlobalContextManager(new TestContextManager());
+    propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+
+    const exporter = new InMemorySpanExporter();
+    const provider = new BasicTracerProvider({
+      spanProcessors: [new SimpleSpanProcessor(exporter)],
+    });
+    trace.setGlobalTracerProvider(provider);
+
+    // Create a parent span and inject its context
+    const tracer = trace.getTracer("test");
+    const parentSpan = tracer.startSpan("parent-request");
+    const parentCtx = trace.setSpan(ROOT_CONTEXT, parentSpan);
+
+    const carrier: Record<string, string> = {};
+    context.with(parentCtx, () => {
+      propagation.inject(context.active(), carrier);
+    });
+
+    // Extract context from the carrier (simulating worker side)
+    const extractedCtx = propagation.extract(ROOT_CONTEXT, carrier);
+    const childSpan = tracer.startSpan("worker-process", undefined, extractedCtx);
+
+    // Child should share traceId but have a different spanId
+    expect(childSpan.spanContext().traceId).toBe(parentSpan.spanContext().traceId);
+    expect(childSpan.spanContext().spanId).not.toBe(parentSpan.spanContext().spanId);
+
+    parentSpan.end();
+    childSpan.end();
+    await provider.shutdown();
+  });
+
+  it("creates root span when _otel is absent", () => {
+    const extractedCtx = propagation.extract(ROOT_CONTEXT, {});
+    const span = trace.getSpan(extractedCtx);
+    expect(span).toBeUndefined();
+  });
+});