External/SnapOtter
1
0
Fork 0
mirror of https://github.com/snapotter-hq/SnapOtter.git synced 2026-08-03 07:46:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add OpenTelemetry distributed tracing (enterprise) (#232)

Browse Source 
* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.
This commit is contained in:
SnapOtter
2026-06-15 12:53:06 +08:00
committed by GitHub
parent 5af1ac4dcf
commit 3fb8164fa5
25 changed files with 1968 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/bridge.ts
+65 -22
View File
@@ -2,6 +2,7 @@ import { type ChildProcess, spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { context, propagation, SpanStatusCode, trace } from "@opentelemetry/api";
const __dirname = dirname(fileURLToPath(import.meta.url));
const PYTHON_DIR = resolve(__dirname, "../python");
@@ -32,6 +33,9 @@ function buildMinimalEnv(): Record<string, string> {
"DISPATCHER_MAX_REQUESTS",
"PYTHON_VENV_PATH",
"SNAPOTTER_GPU",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_EXPORTER_OTLP_HEADERS",
];
for (const key of passthrough) {
if (process.env[key] !== undefined) {
@@ -364,7 +368,16 @@ export class PythonDispatcher {
stderrLines: [],
});
const request = JSON.stringify({ id, script: scriptName.replace(".py", ""), args });
const msg: Record<string, unknown> = { id, script: scriptName.replace(".py", ""), args };
const otelCarrier: Record<string, string> = {};
propagation.inject(context.active(), otelCarrier);
if (otelCarrier.traceparent) {
msg._otel = {
traceparent: otelCarrier.traceparent,
tracestate: otelCarrier.tracestate,
};
}
const request = JSON.stringify(msg);
try {
proc.stdin!.write(request + "\n");
} catch {
@@ -567,28 +580,58 @@ export class PythonDispatcher {
timeout?: number;
} = {},
): Promise<{ stdout: string; stderr: string }> {
// Try persistent dispatcher first
const dispatcherPromise = this.dispatcherRun(scriptName, args, options);
if (dispatcherPromise) {
return dispatcherPromise.catch((err: Error) => {
if (
err.message === "Python dispatcher exited unexpectedly" ||
err.message === "Python dispatcher stdin closed unexpectedly"
) {
console.warn(
`[bridge] Dispatcher crashed during ${scriptName}, retrying with per-request process`,
);
return this.runPerRequest(scriptName, args, options).then((result) => ({
...result,
stderr: `${result.stderr}\n[bridge] retried after dispatcher crash`,
}));
}
throw err;
});
}
const tracer = trace.getTracer("snapotter-sidecar");
const span = trace.getActiveSpan()
? tracer.startSpan("sidecar.execute", {
attributes: {
"sidecar.script": scriptName.replace(".py", ""),
"sidecar.profile": this.profile,
},
})
: null;
// Fall back to per-request spawning
return this.runPerRequest(scriptName, args, options);
const doRun = (): Promise<{ stdout: string; stderr: string }> => {
// Try persistent dispatcher first
const dispatcherPromise = this.dispatcherRun(scriptName, args, options);
if (dispatcherPromise) {
return dispatcherPromise.catch((err: Error) => {
if (
err.message === "Python dispatcher exited unexpectedly" ||
err.message === "Python dispatcher stdin closed unexpectedly"
) {
console.warn(
`[bridge] Dispatcher crashed during ${scriptName}, retrying with per-request process`,
);
return this.runPerRequest(scriptName, args, options).then((result) => ({
...result,
stderr: `${result.stderr}\n[bridge] retried after dispatcher crash`,
}));
}
throw err;
});
}
// Fall back to per-request spawning
return this.runPerRequest(scriptName, args, options);
};
if (!span) return doRun();
return doRun().then(
(result) => {
span.end();
return result;
},
(err) => {
span.setStatus({
code: SpanStatusCode.ERROR,
message: err instanceof Error ? err.message : String(err),
});
span.recordException(err instanceof Error ? err : new Error(String(err)));
span.end();
throw err;
},
);
}
}