feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.

packages/ai/package.json

@@ -10,6 +10,7 @@
     "clean": "rm -rf dist"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.1",
     "@snapotter/shared": "workspace:*",
     "sharp": "^0.34.5"
   },

packages/ai/python/dispatcher.py

@@ -28,6 +28,32 @@ import os
 import traceback
 
 
+# ── Optional OpenTelemetry tracing (enterprise only) ─────────────
+_tracer = None
+_tracer_provider = None
+
+def _init_tracing():
+    """Initialize OTel tracing if OTEL_EXPORTER_OTLP_ENDPOINT is set."""
+    global _tracer, _tracer_provider
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if not endpoint:
+        return
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+        from opentelemetry.sdk.resources import Resource
+
+        resource = Resource.create({"service.name": "snapotter-sidecar"})
+        _tracer_provider = TracerProvider(resource=resource)
+        _tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+        otel_trace.set_tracer_provider(_tracer_provider)
+        _tracer = otel_trace.get_tracer("snapotter-sidecar")
+    except ImportError:
+        pass
+
+
 # ── Script allowlist ───────────────────────────────────────────────────
 # Only these script names (without .py) may be dispatched. This is the
 # primary security gate -- no path traversal, no arbitrary file execution.
@@ -319,6 +345,8 @@ def main():
     print(json.dumps({"ready": True, "gpu": gpu}), file=sys.stderr, flush=True)
     print(f"[dispatcher] Ready. GPU: {gpu}. Max requests: {MAX_REQUESTS}. Modules: {list(available_modules.keys())}", file=sys.stderr, flush=True)
 
+    _init_tracing()
+
     request_count = 0
 
     for line in sys.stdin:
@@ -335,8 +363,38 @@ def main():
         script_name = request.get("script", "")
         args = request.get("args", [])
 
+        otel_data = request.pop("_otel", None)
+        otel_ctx = None
+        if otel_data and _tracer:
+            from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+            from opentelemetry import context as otel_context
+            propagator = TraceContextTextMapPropagator()
+            otel_ctx = propagator.extract(carrier=otel_data)
+
         try:
-            stdout_output, exit_code = _run_script_main(script_name, args)
+            if otel_ctx and _tracer:
+                from opentelemetry import trace as otel_trace
+                from opentelemetry.trace import StatusCode
+                from opentelemetry import context as otel_context
+                token = otel_context.attach(otel_ctx)
+                span = _tracer.start_span(f"sidecar.{script_name}", context=otel_ctx)
+                try:
+                    stdout_output, exit_code = _run_script_main(script_name, args)
+                    if exit_code != 0:
+                        span.set_status(StatusCode.ERROR, f"exit code {exit_code}")
+                except Exception as exc:
+                    span.set_status(StatusCode.ERROR, str(exc))
+                    span.record_exception(exc)
+                    raise
+                finally:
+                    span.end()
+                    otel_context.detach(token)
+                    try:
+                        _tracer_provider.force_flush()
+                    except Exception:
+                        pass
+            else:
+                stdout_output, exit_code = _run_script_main(script_name, args)
             response = {
                 "id": request_id,
                 "stdout": stdout_output,
@@ -361,6 +419,12 @@ def main():
                   file=sys.stderr, flush=True)
             break
 
+    if _tracer_provider:
+        try:
+            _tracer_provider.shutdown()
+        except Exception:
+            pass
+
 
 if __name__ == "__main__":
     main()

packages/ai/python/tests/test_tracing.py

@@ -0,0 +1,43 @@
+import json
+import os
+import pytest
+
+
+def test_otel_stripped_from_request():
+    """_otel should be popped from the request before reaching scripts."""
+    request = {
+        "id": "test-1",
+        "script": "remove_bg",
+        "args": ["input.png"],
+        "_otel": {"traceparent": "00-abc123-def456-01"},
+    }
+    otel_data = request.pop("_otel", None)
+    assert otel_data is not None
+    assert otel_data["traceparent"] == "00-abc123-def456-01"
+    assert "_otel" not in request
+    assert request["args"] == ["input.png"]
+
+
+def test_no_otel_in_request():
+    """Requests without _otel should work normally."""
+    request = {
+        "id": "test-2",
+        "script": "remove_bg",
+        "args": ["input.png"],
+    }
+    otel_data = request.pop("_otel", None)
+    assert otel_data is None
+    assert request["args"] == ["input.png"]
+
+
+def test_tracing_init_without_endpoint(monkeypatch):
+    """_init_tracing should be a no-op without OTEL_EXPORTER_OTLP_ENDPOINT."""
+    monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False)
+    import sys
+    dispatcher_dir = os.path.join(os.path.dirname(__file__), "..")
+    if dispatcher_dir not in sys.path:
+        sys.path.insert(0, dispatcher_dir)
+    from dispatcher import _init_tracing, _tracer
+    _init_tracing()
+    from dispatcher import _tracer as tracer_after
+    assert tracer_after is None

packages/ai/src/bridge.ts

@@ -2,6 +2,7 @@ import { type ChildProcess, spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { context, propagation, SpanStatusCode, trace } from "@opentelemetry/api";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PYTHON_DIR = resolve(__dirname, "../python");
@@ -32,6 +33,9 @@ function buildMinimalEnv(): Record<string, string> {
     "DISPATCHER_MAX_REQUESTS",
     "PYTHON_VENV_PATH",
     "SNAPOTTER_GPU",
+    "OTEL_EXPORTER_OTLP_ENDPOINT",
+    "OTEL_EXPORTER_OTLP_PROTOCOL",
+    "OTEL_EXPORTER_OTLP_HEADERS",
   ];
   for (const key of passthrough) {
     if (process.env[key] !== undefined) {
@@ -364,7 +368,16 @@ export class PythonDispatcher {
         stderrLines: [],
       });
 
-      const request = JSON.stringify({ id, script: scriptName.replace(".py", ""), args });
+      const msg: Record<string, unknown> = { id, script: scriptName.replace(".py", ""), args };
+      const otelCarrier: Record<string, string> = {};
+      propagation.inject(context.active(), otelCarrier);
+      if (otelCarrier.traceparent) {
+        msg._otel = {
+          traceparent: otelCarrier.traceparent,
+          tracestate: otelCarrier.tracestate,
+        };
+      }
+      const request = JSON.stringify(msg);
       try {
         proc.stdin!.write(request + "\n");
       } catch {
@@ -567,28 +580,58 @@ export class PythonDispatcher {
       timeout?: number;
     } = {},
   ): Promise<{ stdout: string; stderr: string }> {
-    // Try persistent dispatcher first
-    const dispatcherPromise = this.dispatcherRun(scriptName, args, options);
-    if (dispatcherPromise) {
-      return dispatcherPromise.catch((err: Error) => {
-        if (
-          err.message === "Python dispatcher exited unexpectedly" ||
-          err.message === "Python dispatcher stdin closed unexpectedly"
-        ) {
-          console.warn(
-            `[bridge] Dispatcher crashed during ${scriptName}, retrying with per-request process`,
-          );
-          return this.runPerRequest(scriptName, args, options).then((result) => ({
-            ...result,
-            stderr: `${result.stderr}\n[bridge] retried after dispatcher crash`,
-          }));
-        }
-        throw err;
-      });
-    }
+    const tracer = trace.getTracer("snapotter-sidecar");
+    const span = trace.getActiveSpan()
+      ? tracer.startSpan("sidecar.execute", {
+          attributes: {
+            "sidecar.script": scriptName.replace(".py", ""),
+            "sidecar.profile": this.profile,
+          },
+        })
+      : null;
 
-    // Fall back to per-request spawning
-    return this.runPerRequest(scriptName, args, options);
+    const doRun = (): Promise<{ stdout: string; stderr: string }> => {
+      // Try persistent dispatcher first
+      const dispatcherPromise = this.dispatcherRun(scriptName, args, options);
+      if (dispatcherPromise) {
+        return dispatcherPromise.catch((err: Error) => {
+          if (
+            err.message === "Python dispatcher exited unexpectedly" ||
+            err.message === "Python dispatcher stdin closed unexpectedly"
+          ) {
+            console.warn(
+              `[bridge] Dispatcher crashed during ${scriptName}, retrying with per-request process`,
+            );
+            return this.runPerRequest(scriptName, args, options).then((result) => ({
+              ...result,
+              stderr: `${result.stderr}\n[bridge] retried after dispatcher crash`,
+            }));
+          }
+          throw err;
+        });
+      }
+
+      // Fall back to per-request spawning
+      return this.runPerRequest(scriptName, args, options);
+    };
+
+    if (!span) return doRun();
+
+    return doRun().then(
+      (result) => {
+        span.end();
+        return result;
+      },
+      (err) => {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: err instanceof Error ? err.message : String(err),
+        });
+        span.recordException(err instanceof Error ? err : new Error(String(err)));
+        span.end();
+        throw err;
+      },
+    );
   }
 }
 

packages/enterprise/src/license.ts

@@ -23,6 +23,7 @@ export const ENTERPRISE_FEATURES = [
   "config_export_import",
   "upgrade_management",
   "admin_alerts",
+  "distributed_tracing",
 ] as const;
 
 export type EnterpriseFeature = (typeof ENTERPRISE_FEATURES)[number];