feat: add OpenTelemetry distributed tracing (enterprise) (#232)

* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.

packages/ai/python/dispatcher.py

@@ -28,6 +28,32 @@ import os
 import traceback
 
 
+# ── Optional OpenTelemetry tracing (enterprise only) ─────────────
+_tracer = None
+_tracer_provider = None
+
+def _init_tracing():
+    """Initialize OTel tracing if OTEL_EXPORTER_OTLP_ENDPOINT is set."""
+    global _tracer, _tracer_provider
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+    if not endpoint:
+        return
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+        from opentelemetry.sdk.resources import Resource
+
+        resource = Resource.create({"service.name": "snapotter-sidecar"})
+        _tracer_provider = TracerProvider(resource=resource)
+        _tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+        otel_trace.set_tracer_provider(_tracer_provider)
+        _tracer = otel_trace.get_tracer("snapotter-sidecar")
+    except ImportError:
+        pass
+
+
 # ── Script allowlist ───────────────────────────────────────────────────
 # Only these script names (without .py) may be dispatched. This is the
 # primary security gate -- no path traversal, no arbitrary file execution.
@@ -319,6 +345,8 @@ def main():
     print(json.dumps({"ready": True, "gpu": gpu}), file=sys.stderr, flush=True)
     print(f"[dispatcher] Ready. GPU: {gpu}. Max requests: {MAX_REQUESTS}. Modules: {list(available_modules.keys())}", file=sys.stderr, flush=True)
 
+    _init_tracing()
+
     request_count = 0
 
     for line in sys.stdin:
@@ -335,8 +363,38 @@ def main():
         script_name = request.get("script", "")
         args = request.get("args", [])
 
+        otel_data = request.pop("_otel", None)
+        otel_ctx = None
+        if otel_data and _tracer:
+            from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+            from opentelemetry import context as otel_context
+            propagator = TraceContextTextMapPropagator()
+            otel_ctx = propagator.extract(carrier=otel_data)
+
         try:
-            stdout_output, exit_code = _run_script_main(script_name, args)
+            if otel_ctx and _tracer:
+                from opentelemetry import trace as otel_trace
+                from opentelemetry.trace import StatusCode
+                from opentelemetry import context as otel_context
+                token = otel_context.attach(otel_ctx)
+                span = _tracer.start_span(f"sidecar.{script_name}", context=otel_ctx)
+                try:
+                    stdout_output, exit_code = _run_script_main(script_name, args)
+                    if exit_code != 0:
+                        span.set_status(StatusCode.ERROR, f"exit code {exit_code}")
+                except Exception as exc:
+                    span.set_status(StatusCode.ERROR, str(exc))
+                    span.record_exception(exc)
+                    raise
+                finally:
+                    span.end()
+                    otel_context.detach(token)
+                    try:
+                        _tracer_provider.force_flush()
+                    except Exception:
+                        pass
+            else:
+                stdout_output, exit_code = _run_script_main(script_name, args)
             response = {
                 "id": request_id,
                 "stdout": stdout_output,
@@ -361,6 +419,12 @@ def main():
                   file=sys.stderr, flush=True)
             break
 
+    if _tracer_provider:
+        try:
+            _tracer_provider.shutdown()
+        except Exception:
+            pass
+
 
 if __name__ == "__main__":
     main()