External/SnapOtter
1
0
Fork 0
mirror of https://github.com/snapotter-hq/SnapOtter.git synced 2026-08-03 07:46:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add OpenTelemetry distributed tracing (enterprise) (#232)

Browse Source 
* feat(tracing): add OpenTelemetry dependencies and --import preload flag

* feat(enterprise): add distributed_tracing feature gate

* feat(tracing): add SDK bootstrap with enterprise gating

* fix(tracing): correct test coverage for enterprise-unavailable path and prevent double-init

Test 2 now mocks @snapotter/enterprise to throw an import error, exercising
the catch block in the preload. Test 3 imports with no endpoint so the preload
is a no-op, avoiding leaked SDK from double-initialization. Added idempotency
guard to initTracing() as a safety net.

* feat(tracing): add Pino trace mixin and shared logger

When OTel tracing is active, every Pino log line now includes traceId,
spanId, and traceFlags fields for log-to-trace correlation. The mixin
is a no-op when no SDK is registered (community users).

* feat(tracing): add _otel to ToolJobData and inject trace context at enqueue

Add optional _otel carrier field to ToolJobData for W3C trace context
propagation across BullMQ job boundaries. When an active OTel span exists,
propagation.inject() writes traceparent/tracestate into the job data before
queue.add(). When no SDK is registered (community edition), the carrier
stays empty and _otel remains undefined -- zero overhead.

* feat(tracing): extract trace context and create spans in BullMQ worker

* feat(tracing): inject trace context into Python sidecar calls

* feat(tracing): add trace context extraction to Python sidecar

* feat(tracing): add shutdownTracing to graceful shutdown sequence

* feat(tracing): enrich HTTP spans with tool_id and user_id attributes

* docs: add OpenTelemetry env var documentation to .env.example

* test(tracing): add lifecycle integration tests for trace propagation

* fix(tracing): inject trace context into pipeline and batch flow jobs

* fix(tracing): add sidecar.execute Node-side span and remove unnecessary comment

Wraps PythonDispatcher.run() with a sidecar.execute span on the Node
side so traces show the full round-trip (Node span -> Python span).
Also removes an obvious comment from logger.ts.
This commit is contained in:
SnapOtter
2026-06-15 12:53:06 +08:00
committed by GitHub
parent 5af1ac4dcf
commit 3fb8164fa5
25 changed files with 1968 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/index.ts
+28 -22
View File
@@ -1,9 +1,9 @@
import { randomUUID } from "node:crypto";
import { statfs } from "node:fs/promises";
import { join } from "node:path";
import cookie from "@fastify/cookie";
import cors from "@fastify/cors";
import rateLimit from "@fastify/rate-limit";
import { trace } from "@opentelemetry/api";
import { getDispatcherStatus, initDispatcher, isGpuAvailable } from "@snapotter/ai";
import { APP_VERSION } from "@snapotter/shared";
import { eq, sql } from "drizzle-orm";
@@ -21,7 +21,7 @@ import { captureException, initAnalytics, shutdownAnalytics } from "./lib/analyt
import { shouldRunStartupCleanup } from "./lib/cleanup.js";
import { buildCsp } from "./lib/csp.js";
import { ensureAiDirs, recoverInterruptedInstalls } from "./lib/feature-status.js";
import { logger } from "./lib/logger.js";
import { requestDuration } from "./lib/metrics.js";
import { getSettingString } from "./lib/settings-helpers.js";
import { requirePermission } from "./permissions.js";
@@ -31,6 +31,7 @@ import {
ensureAnonymousUser,
ensureBuiltinRoles,
ensureDefaultAdmin,
getAuthUser,
} from "./plugins/auth.js";
import { registerMfa } from "./plugins/mfa.js";
import { oidcRoutes } from "./plugins/oidc.js";
@@ -56,6 +57,7 @@ import { settingsRoutes } from "./routes/settings.js";
import { teamsRoutes } from "./routes/teams.js";
import { registerToolRoutes } from "./routes/tools/index.js";
import { userFileRoutes } from "./routes/user-files.js";
import { shutdownTracing } from "./tracing.js";
// Run before anything else
try {
@@ -188,26 +190,7 @@ function parseTrustProxy(value: string): boolean | number | string {
const app = Fastify({
genReqId: (req) => (req.headers["x-request-id"] as string) ?? randomUUID(),
logger: {
level: env.LOG_LEVEL,
transport: {
targets: [
{ target: "pino/file", options: { destination: 1 } },
{
// Rotate at 10 MB, keep 5 files
target: "pino-roll",
options: {
file: join(env.LOG_DIR, "snapotter"),
extension: ".log",
size: "10m",
limit: { count: 5 },
mkdir: true,
},
},
],
},
redact: ["req.headers.authorization", "req.headers.cookie"],
},
loggerInstance: logger,
bodyLimit: env.MAX_UPLOAD_SIZE_MB > 0 ? env.MAX_UPLOAD_SIZE_MB * 1024 * 1024 : 1073741824,
trustProxy: parseTrustProxy(env.TRUST_PROXY),
routerOptions: { maxParamLength: 500 },
@@ -330,6 +313,18 @@ await authMiddleware(app);
// Per-user rate limiting (after auth so request.user is populated)
await registerPerUserRateLimit(app);
// Enrich active OTel span with tool_id and user_id when available
app.addHook("preHandler", (request, _reply, done) => {
const span = trace.getActiveSpan();
if (span) {
const params = request.params as Record<string, string> | undefined;
if (params?.toolId) span.setAttribute("snapotter.tool_id", params.toolId);
const user = getAuthUser(request);
if (user) span.setAttribute("snapotter.user_id", user.id);
}
done();
});
// Auth routes
await authRoutes(app);
@@ -668,6 +663,17 @@ async function shutdown(signal: string) {
// Close BullMQ resources before database (workers first so no new jobs start)
try {
await closeWorkers();
} catch (err) {
console.error("Error closing workers:", err);
}
try {
await shutdownTracing();
} catch {
// tracing shutdown is best-effort
}
try {
await closeFlowProducer();
await closeQueueEvents();
await closeQueues();
apps/api/src/jobs/enqueue.ts
+21
View File
@@ -5,6 +5,7 @@
* the appropriate BullMQ queue. waitForJob() blocks the HTTP request
* until the worker produces a result or the sync-wait window expires.
*/
import { context, propagation } from "@opentelemetry/api";
import { FlowProducer, type Job, QueueEvents } from "bullmq";
import { eq } from "drizzle-orm";
import { env } from "../config.js";
@@ -54,6 +55,24 @@ export async function closeFlowProducer(): Promise<void> {
}
}
// ── Trace context injection ─────────────────────────────────────
/**
* Inject the active OpenTelemetry trace context into a ToolJobData object.
* Called from enqueueToolJob (single jobs) and from pipeline/batch routes
* that build FlowProducer trees bypassing enqueueToolJob.
*/
export function injectTraceContext(data: ToolJobData): void {
const carrier: Record<string, string> = {};
propagation.inject(context.active(), carrier);
if (carrier.traceparent) {
data._otel = {
traceparent: carrier.traceparent,
tracestate: carrier.tracestate,
};
}
}
// ── Enqueue + wait ──────────────────────────────────────────────
/**
@@ -82,6 +101,8 @@ export async function enqueueToolJob(data: ToolJobData): Promise<Job<ToolJobData
void computeDeleteAfter(data.jobId, data.userId).catch(() => {});
}
injectTraceContext(data);
const queue = getQueue(data.pool);
const job = await queue.add(data.toolId, { ...data, jobId: data.jobId }, { jobId: data.jobId });
return job;
apps/api/src/jobs/types.ts
+1
View File
@@ -46,6 +46,7 @@ export interface ToolJobData {
parentId?: string;
totalFiles?: number;
fileIndex?: number;
_otel?: { traceparent: string; tracestate?: string };
}
/** Result returned by a completed BullMQ job. */
apps/api/src/jobs/worker.ts
+289 -238
View File
@@ -24,12 +24,14 @@
import { mkdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { context, propagation, ROOT_CONTEXT, SpanStatusCode, trace } from "@opentelemetry/api";
import { type Job, UnrecoverableError, Worker } from "bullmq";
import { eq } from "drizzle-orm";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { resolveConcurrency } from "../lib/env.js";
import { stripInternalPaths } from "../lib/errors.js";
import { logger } from "../lib/logger.js";
import { jobDuration, jobsTotal } from "../lib/metrics.js";
import { getObjectBuffer, putObject } from "../lib/object-storage.js";
import { publishEphemeral, updateSingleFileProgress } from "../routes/progress.js";
@@ -104,258 +106,307 @@ async function processToolJob(job: Job<ToolJobData>): Promise<ToolJobResult> {
const { jobId } = data;
const startTime = Date.now();
// Register for cooperative cancellation
const ac = registerCancelable(jobId);
const signal = ac.signal;
// Extract OTel trace context if present (no-op without SDK)
const otel = data._otel;
const parentCtx = otel?.traceparent ? propagation.extract(ROOT_CONTEXT, otel) : ROOT_CONTEXT;
const tracer = trace.getTracer("snapotter-worker");
const span = otel?.traceparent
? tracer.startSpan(
"job.process",
{
attributes: {
"snapotter.job_id": jobId,
"snapotter.tool_id": data.toolId,
"snapotter.pool": data.pool,
"snapotter.attempt_number": job.attemptsMade + 1,
},
},
parentCtx,
)
: null;
// Timeout guard (0 means unlimited; only arm when positive)
const timeoutMs = timeoutMsFor(data.pool);
const timeoutHandle =
timeoutMs > 0 ? setTimeout(() => ac.abort("timeout"), timeoutMs) : undefined;
const runBody = async (): Promise<ToolJobResult> => {
if (span) span.addEvent("job.active");
// Per-job scratch directory
const scratchDir = join(scratchRoot(), jobId);
// Register for cooperative cancellation
const ac = registerCancelable(jobId);
const signal = ac.signal;
try {
await mkdir(scratchDir, { recursive: true });
// Timeout guard (0 means unlimited; only arm when positive)
const timeoutMs = timeoutMsFor(data.pool);
const timeoutHandle =
timeoutMs > 0 ? setTimeout(() => ac.abort("timeout"), timeoutMs) : undefined;
// Mark job as processing in the durable row
await db
.update(schema.jobs)
.set({
status: "processing",
startedAt: new Date(),
attempts: job.attemptsMade + 1,
})
.where(eq(schema.jobs.id, jobId));
// Per-job scratch directory
const scratchDir = join(scratchRoot(), jobId);
// Load all input refs from object storage. The primary input keeps
// the client-facing filename; secondary inputs derive filenames from
// their ref basenames.
const inputs: ToolProcessInputV2[] = await Promise.all(
data.inputRefs.map(async (ref) => ({
ref,
buffer: await getObjectBuffer(ref),
filename: ref.split("/").slice(2).join("/") || data.filename,
})),
);
inputs[0].filename = data.filename; // primary keeps the client-facing name
const inputBuffer = inputs[0].buffer; // existing metrics/size/preview paths
// Progress reporter: emits both Redis pub/sub and BullMQ job progress
const progressJobId = data.clientJobId ?? jobId;
const report = (percent: number, stage?: string) => {
updateSingleFileProgress({
jobId: progressJobId,
phase: "processing",
percent,
stage,
});
void job.updateProgress({ percent, stage });
};
// Check for cancellation before dispatching
if (signal.aborted) throw new Error("Canceled");
// Build the process context
const ctx: ToolProcessCtx = { signal, scratchDir, report };
// Dispatch: AI handler or standard tool registry
let resultBuffer: Buffer;
let resultFilename: string;
let resultContentType: string;
let resultPayload: Record<string, unknown> | undefined;
let extraOutputs: Array<{ name: string; buffer: Buffer; contentType: string }> | undefined;
if (hasAiJobHandler(data.toolId)) {
const aiResult = await runAiToolJob(data, inputBuffer, ctx);
resultBuffer = aiResult.buffer;
resultFilename = aiResult.filename;
resultContentType = aiResult.contentType;
resultPayload = aiResult.resultPayload;
extraOutputs = aiResult.extraOutputs;
} else {
const config = getToolConfig(data.toolId);
if (!config) throw new Error(`No tool config for ${data.toolId}`);
// Use the resolved v2 process function (adapter or native)
if (!config.processV2) throw new Error(`No processV2 for ${data.toolId}`);
const result = await config.processV2({
inputs,
settings: data.settings,
scratchDir,
signal,
report,
});
// Resolve buffer OR scratchPath for the primary output
if (result.buffer) {
resultBuffer = result.buffer;
} else if (result.scratchPath) {
resultBuffer = await readFile(result.scratchPath);
} else {
throw new Error(`Tool ${data.toolId} returned neither buffer nor scratchPath`);
}
resultFilename = result.filename;
resultContentType = result.contentType;
resultPayload = result.resultPayload;
// Resolve extra outputs with the same buffer/scratchPath duality
if (result.extraOutputs) {
extraOutputs = await Promise.all(
result.extraOutputs.map(async (extra) => {
let buf: Buffer;
if (extra.buffer) {
buf = extra.buffer;
} else if (extra.scratchPath) {
buf = await readFile(extra.scratchPath);
} else {
throw new Error(`Extra output "${extra.name}" has neither buffer nor scratchPath`);
}
return { name: extra.name, buffer: buf, contentType: extra.contentType };
}),
);
}
}
// Build output name with tool suffix and extension fixup
const outName = buildOutputName(resultFilename, data.filename, data.toolId, resultContentType);
// Write primary output to object storage
const primaryKey = `outputs/${jobId}/${outName}`;
await putObject(primaryKey, resultBuffer);
const outputRefs: string[] = [primaryKey];
// Write extra outputs (AI tools may produce multiple files)
if (extraOutputs) {
for (const extra of extraOutputs) {
const extraKey = `outputs/${jobId}/${extra.name}`;
await putObject(extraKey, extra.buffer);
outputRefs.push(extraKey);
}
}
// Generate preview for non-browser-previewable formats
const previewRef = await generatePreview(resultBuffer, resultContentType, jobId, inputBuffer);
// Auto-save to user file library
const savedFileId = await autoSaveToLibrary({
fileId: data.fileId,
userId: data.userId,
buffer: resultBuffer,
outName,
contentType: resultContentType,
toolId: data.toolId,
});
const durationMs = Date.now() - startTime;
// Build the result
const jobResult: ToolJobResult = {
outputRefs,
filename: outName,
contentType: resultContentType,
originalSize: inputBuffer.length,
processedSize: resultBuffer.length,
previewRef,
savedFileId,
resultPayload,
};
// Update durable row to completed
await db
.update(schema.jobs)
.set({
status: "completed",
completedAt: new Date(),
durationMs,
bytesIn: inputBuffer.length,
bytesOut: resultBuffer.length,
outputRefs,
progress: { percent: 100, stage: "complete" },
})
.where(eq(schema.jobs.id, jobId));
// Record Prometheus metrics
jobsTotal.inc({ pool: data.pool, status: "completed" });
jobDuration.observe({ pool: data.pool }, durationMs / 1000);
// Emit terminal progress event with legacy result payload
const legacyResult = buildLegacyResultPayload(jobResult, jobId);
updateSingleFileProgress({
jobId: progressJobId,
phase: "complete",
percent: 100,
stage: "complete",
result: legacyResult,
});
return jobResult;
} catch (err) {
const durationMs = Date.now() - startTime;
const isTimeout = signal.aborted && signal.reason === "timeout";
const isCanceled = signal.aborted && !isTimeout;
const errorMessage = err instanceof Error ? err.message : String(err);
const finalError = isCanceled
? "Canceled"
: isTimeout
? `Timed out after ${Math.round(timeoutMs / 1000)}s`
: errorMessage;
const maxAttempts = job.opts.attempts ?? 1;
const willRetry = !isCanceled && job.attemptsMade + 1 < maxAttempts;
const progressJobId = data.clientJobId ?? jobId;
// When the job will be retried, do NOT write a terminal DB row or
// emit a terminal SSE frame. The row stays "processing" and the
// next attempt overwrites startedAt/attempts as usual.
if (!willRetry) {
// Record Prometheus metrics on final attempt only
jobsTotal.inc({ pool: data.pool, status: isCanceled ? "canceled" : "failed" });
jobDuration.observe({ pool: data.pool }, durationMs / 1000);
try {
await mkdir(scratchDir, { recursive: true });
// Mark job as processing in the durable row
await db
.update(schema.jobs)
.set({
status: isCanceled ? "canceled" : "failed",
completedAt: new Date(),
durationMs,
error: { message: finalError },
status: "processing",
startedAt: new Date(),
attempts: job.attemptsMade + 1,
})
.where(eq(schema.jobs.id, jobId))
.catch(() => {});
.where(eq(schema.jobs.id, jobId));
if (isCanceled) {
// Ephemeral terminal event for live SSE clients. Uses
// publishEphemeral so the replay key is set without
// overwriting the DB row (which stays "canceled").
publishEphemeral({
jobId: progressJobId,
type: "single",
phase: "failed",
percent: 0,
error: "Canceled",
});
} else {
// Load all input refs from object storage. The primary input keeps
// the client-facing filename; secondary inputs derive filenames from
// their ref basenames.
const inputs: ToolProcessInputV2[] = await Promise.all(
data.inputRefs.map(async (ref) => ({
ref,
buffer: await getObjectBuffer(ref),
filename: ref.split("/").slice(2).join("/") || data.filename,
})),
);
inputs[0].filename = data.filename; // primary keeps the client-facing name
const inputBuffer = inputs[0].buffer; // existing metrics/size/preview paths
// Progress reporter: emits both Redis pub/sub and BullMQ job progress
const progressJobId = data.clientJobId ?? jobId;
const report = (percent: number, stage?: string) => {
updateSingleFileProgress({
jobId: progressJobId,
phase: "failed",
percent: 0,
error: stripInternalPaths(finalError),
phase: "processing",
percent,
stage,
});
}
}
void job.updateProgress({ percent, stage });
};
if (isCanceled) throw new UnrecoverableError("Canceled");
if (isTimeout) throw new Error(finalError);
throw err;
} finally {
clearTimeout(timeoutHandle);
unregisterCancelable(jobId);
// Clean up scratch directory
await rm(scratchDir, { recursive: true, force: true }).catch(() => {});
// Check for cancellation before dispatching
if (signal.aborted) throw new Error("Canceled");
// Build the process context
const ctx: ToolProcessCtx = { signal, scratchDir, report };
// Dispatch: AI handler or standard tool registry
let resultBuffer: Buffer;
let resultFilename: string;
let resultContentType: string;
let resultPayload: Record<string, unknown> | undefined;
let extraOutputs: Array<{ name: string; buffer: Buffer; contentType: string }> | undefined;
if (hasAiJobHandler(data.toolId)) {
const aiResult = await runAiToolJob(data, inputBuffer, ctx);
resultBuffer = aiResult.buffer;
resultFilename = aiResult.filename;
resultContentType = aiResult.contentType;
resultPayload = aiResult.resultPayload;
extraOutputs = aiResult.extraOutputs;
} else {
const config = getToolConfig(data.toolId);
if (!config) throw new Error(`No tool config for ${data.toolId}`);
// Use the resolved v2 process function (adapter or native)
if (!config.processV2) throw new Error(`No processV2 for ${data.toolId}`);
const result = await config.processV2({
inputs,
settings: data.settings,
scratchDir,
signal,
report,
});
// Resolve buffer OR scratchPath for the primary output
if (result.buffer) {
resultBuffer = result.buffer;
} else if (result.scratchPath) {
resultBuffer = await readFile(result.scratchPath);
} else {
throw new Error(`Tool ${data.toolId} returned neither buffer nor scratchPath`);
}
resultFilename = result.filename;
resultContentType = result.contentType;
resultPayload = result.resultPayload;
// Resolve extra outputs with the same buffer/scratchPath duality
if (result.extraOutputs) {
extraOutputs = await Promise.all(
result.extraOutputs.map(async (extra) => {
let buf: Buffer;
if (extra.buffer) {
buf = extra.buffer;
} else if (extra.scratchPath) {
buf = await readFile(extra.scratchPath);
} else {
throw new Error(`Extra output "${extra.name}" has neither buffer nor scratchPath`);
}
return { name: extra.name, buffer: buf, contentType: extra.contentType };
}),
);
}
}
// Build output name with tool suffix and extension fixup
const outName = buildOutputName(
resultFilename,
data.filename,
data.toolId,
resultContentType,
);
// Write primary output to object storage
const primaryKey = `outputs/${jobId}/${outName}`;
await putObject(primaryKey, resultBuffer);
const outputRefs: string[] = [primaryKey];
// Write extra outputs (AI tools may produce multiple files)
if (extraOutputs) {
for (const extra of extraOutputs) {
const extraKey = `outputs/${jobId}/${extra.name}`;
await putObject(extraKey, extra.buffer);
outputRefs.push(extraKey);
}
}
// Generate preview for non-browser-previewable formats
const previewRef = await generatePreview(resultBuffer, resultContentType, jobId, inputBuffer);
// Auto-save to user file library
const savedFileId = await autoSaveToLibrary({
fileId: data.fileId,
userId: data.userId,
buffer: resultBuffer,
outName,
contentType: resultContentType,
toolId: data.toolId,
});
const durationMs = Date.now() - startTime;
// Build the result
const jobResult: ToolJobResult = {
outputRefs,
filename: outName,
contentType: resultContentType,
originalSize: inputBuffer.length,
processedSize: resultBuffer.length,
previewRef,
savedFileId,
resultPayload,
};
// Update durable row to completed
await db
.update(schema.jobs)
.set({
status: "completed",
completedAt: new Date(),
durationMs,
bytesIn: inputBuffer.length,
bytesOut: resultBuffer.length,
outputRefs,
progress: { percent: 100, stage: "complete" },
})
.where(eq(schema.jobs.id, jobId));
// Record Prometheus metrics
jobsTotal.inc({ pool: data.pool, status: "completed" });
jobDuration.observe({ pool: data.pool }, durationMs / 1000);
// Emit terminal progress event with legacy result payload
const legacyResult = buildLegacyResultPayload(jobResult, jobId);
updateSingleFileProgress({
jobId: progressJobId,
phase: "complete",
percent: 100,
stage: "complete",
result: legacyResult,
});
// Record queue wait time and completion on the OTel span
if (span && job.processedOn) {
span.setAttribute("snapotter.queue.wait_ms", job.processedOn - job.timestamp);
}
if (span) span.addEvent("job.completed");
return jobResult;
} catch (err) {
const durationMs = Date.now() - startTime;
const isTimeout = signal.aborted && signal.reason === "timeout";
const isCanceled = signal.aborted && !isTimeout;
const errorMessage = err instanceof Error ? err.message : String(err);
const finalError = isCanceled
? "Canceled"
: isTimeout
? `Timed out after ${Math.round(timeoutMs / 1000)}s`
: errorMessage;
// Record error on the OTel span
if (span) {
span.setStatus({ code: SpanStatusCode.ERROR, message: finalError });
span.recordException(err instanceof Error ? err : new Error(String(err)));
span.addEvent("job.failed");
}
const maxAttempts = job.opts.attempts ?? 1;
const willRetry = !isCanceled && job.attemptsMade + 1 < maxAttempts;
const progressJobId = data.clientJobId ?? jobId;
// When the job will be retried, do NOT write a terminal DB row or
// emit a terminal SSE frame. The row stays "processing" and the
// next attempt overwrites startedAt/attempts as usual.
if (!willRetry) {
// Record Prometheus metrics on final attempt only
jobsTotal.inc({ pool: data.pool, status: isCanceled ? "canceled" : "failed" });
jobDuration.observe({ pool: data.pool }, durationMs / 1000);
await db
.update(schema.jobs)
.set({
status: isCanceled ? "canceled" : "failed",
completedAt: new Date(),
durationMs,
error: { message: finalError },
})
.where(eq(schema.jobs.id, jobId))
.catch(() => {});
if (isCanceled) {
// Ephemeral terminal event for live SSE clients. Uses
// publishEphemeral so the replay key is set without
// overwriting the DB row (which stays "canceled").
publishEphemeral({
jobId: progressJobId,
type: "single",
phase: "failed",
percent: 0,
error: "Canceled",
});
} else {
updateSingleFileProgress({
jobId: progressJobId,
phase: "failed",
percent: 0,
error: stripInternalPaths(finalError),
});
}
}
if (isCanceled) throw new UnrecoverableError("Canceled");
if (isTimeout) throw new Error(finalError);
throw err;
} finally {
if (span) span.end();
clearTimeout(timeoutHandle);
unregisterCancelable(jobId);
// Clean up scratch directory
await rm(scratchDir, { recursive: true, force: true }).catch(() => {});
}
};
// Execute with or without active span context
if (span) {
const activeCtx = trace.setSpan(parentCtx, span);
return context.with(activeCtx, runBody);
}
return runBody();
}
// ── Pipeline step handler ─────────────────────────────────────
@@ -694,7 +745,7 @@ export function startWorkers(): void {
});
worker.on("error", (err) => {
console.error(`Worker error [${pool}]:`, err);
logger.error({ err, pool }, "Worker error");
});
workers.push(worker);
@@ -722,7 +773,7 @@ export function startWorkers(): void {
workers.push(worker);
}
console.log(
logger.info(
`Workers started: ${POOLS.map((p) => `${p}(${p === "system" || p === "ai" ? 1 : concurrency})`).join(", ")}`,
);
}
apps/api/src/lib/log-trace-mixin.ts
+8
View File
@@ -0,0 +1,8 @@
import { trace } from "@opentelemetry/api";
export function traceMixin(): Record<string, unknown> {
const span = trace.getActiveSpan();
if (!span) return {};
const { traceId, spanId, traceFlags } = span.spanContext();
return { traceId, spanId, traceFlags };
}
apps/api/src/lib/logger.ts
+26
View File
@@ -0,0 +1,26 @@
import { join } from "node:path";
import type { FastifyBaseLogger } from "fastify";
import pino from "pino";
import { env } from "../config.js";
import { traceMixin } from "./log-trace-mixin.js";
export const logger: FastifyBaseLogger = pino({
level: env.LOG_LEVEL,
mixin: traceMixin,
transport: {
targets: [
{ target: "pino/file", options: { destination: 1 } },
{
target: "pino-roll",
options: {
file: join(env.LOG_DIR, "snapotter"),
extension: ".log",
size: "10m",
limit: { count: 5 },
mkdir: true,
},
},
],
},
redact: ["req.headers.authorization", "req.headers.cookie"],
});
apps/api/src/routes/batch.ts
+14 -1
View File
@@ -18,7 +18,7 @@ import sharp from "sharp";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { recordChildOutcome } from "../jobs/batch-progress.js";
import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
import { getFlowProducer, injectTraceContext, waitForJob } from "../jobs/enqueue.js";
import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
import { autoOrient } from "../lib/auto-orient.js";
import { getSecurityHeaders } from "../lib/csp.js";
@@ -39,6 +39,16 @@ interface ParsedFile {
filename: string;
}
/** Recursively inject OTel trace context into every node of a FlowJob tree. */
function injectTraceContextIntoFlow(node: FlowJob): void {
injectTraceContext(node.data as ToolJobData);
if (node.children) {
for (const child of node.children) {
injectTraceContextIntoFlow(child);
}
}
}
export async function registerBatchRoutes(app: FastifyInstance): Promise<void> {
app.post(
"/api/v1/tools/:toolId/batch",
@@ -297,6 +307,9 @@ export async function registerBatchRoutes(app: FastifyInstance): Promise<void> {
.set({ settings: { flowChildCount: flowChildren.length } })
.where(eq(schema.jobs.id, parentId));
// Inject OTel trace context into every node of the batch flow tree
injectTraceContextIntoFlow(batchTree);
await getFlowProducer().add(batchTree);
// ── Wait for completion and stream ZIP ─────────────────────────
apps/api/src/routes/pipeline.ts
+17 -1
View File
@@ -17,7 +17,7 @@ import { z } from "zod";
import { env } from "../config.js";
import { db, schema } from "../db/index.js";
import { recordChildOutcome } from "../jobs/batch-progress.js";
import { getFlowProducer, waitForJob } from "../jobs/enqueue.js";
import { getFlowProducer, injectTraceContext, waitForJob } from "../jobs/enqueue.js";
import { type Pool, queueName, type ToolJobData } from "../jobs/types.js";
import { trackEvent } from "../lib/analytics.js";
import { autoOrient } from "../lib/auto-orient.js";
@@ -78,6 +78,16 @@ interface ParsedStep {
*/
const PASSWORD_TOOLS = new Set(["protect-pdf", "unlock-pdf"]);
/** Recursively inject OTel trace context into every node of a FlowJob tree. */
function injectTraceContextIntoFlow(node: FlowJob): void {
injectTraceContext(node.data as ToolJobData);
if (node.children) {
for (const child of node.children) {
injectTraceContextIntoFlow(child);
}
}
}
/**
* Build a FlowJob tree for a single-file pipeline.
*
@@ -405,6 +415,9 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
settings: {},
});
// Inject OTel trace context into every node of the flow tree
injectTraceContextIntoFlow(tree);
// Add the flow to BullMQ
await getFlowProducer().add(tree);
@@ -913,6 +926,9 @@ export async function registerPipelineRoutes(app: FastifyInstance): Promise<void
.set({ settings: { flowChildCount: perFileChildren.length } })
.where(eq(schema.jobs.id, parentId));
// Inject OTel trace context into every node of the batch flow tree
injectTraceContextIntoFlow(batchTree);
await getFlowProducer().add(batchTree);
// Wait for batch completion
apps/api/src/tracing.ts
+93
View File
@@ -0,0 +1,93 @@
import { createRequire } from "node:module";
import type { SpanExporter } from "@opentelemetry/sdk-trace-base";
let _active = false;
let _sdk: { shutdown(): Promise<void> } | null = null;
export function isTracingActive(): boolean {
return _active;
}
export async function initTracing(options: { exporter?: SpanExporter } = {}): Promise<void> {
if (_active) return;
const { NodeSDK } = await import("@opentelemetry/sdk-node");
const { BatchSpanProcessor } = await import("@opentelemetry/sdk-trace-base");
const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-http");
const { defaultResource, resourceFromAttributes } = await import("@opentelemetry/resources");
const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = await import(
"@opentelemetry/semantic-conventions"
);
const { HttpInstrumentation } = await import("@opentelemetry/instrumentation-http");
const { FastifyInstrumentation } = await import("@opentelemetry/instrumentation-fastify");
const { PgInstrumentation } = await import("@opentelemetry/instrumentation-pg");
const { IORedisInstrumentation } = await import("@opentelemetry/instrumentation-ioredis");
const { AwsInstrumentation } = await import("@opentelemetry/instrumentation-aws-sdk");
const require = createRequire(import.meta.url);
const { version } = require("../package.json");
const exporter = options.exporter ?? new OTLPTraceExporter();
const resource = defaultResource().merge(
resourceFromAttributes({
[ATTR_SERVICE_NAME]: "snapotter-api",
[ATTR_SERVICE_VERSION]: version,
}),
);
const sdk = new NodeSDK({
resource,
spanProcessors: [new BatchSpanProcessor(exporter)],
instrumentations: [
new HttpInstrumentation({
ignoreOutgoingRequestHook: (req) => {
const host = req.hostname || req.host || "";
return host.includes("posthog") || host.includes("sentry");
},
}),
new FastifyInstrumentation(),
new PgInstrumentation(),
new IORedisInstrumentation({
requireParentSpan: true,
dbStatementSerializer: (cmd, args) => {
if (cmd === "evalsha" || cmd === "eval") return `${cmd} <lua>`;
return `${cmd} ${(args ?? []).slice(0, 2).join(" ")}`;
},
}),
new AwsInstrumentation(),
],
});
sdk.start();
_sdk = sdk;
_active = true;
}
export async function shutdownTracing(): Promise<void> {
if (_sdk) {
await _sdk.shutdown().catch(() => {});
_sdk = null;
_active = false;
}
}
// -- Preload entry point --
// When loaded via --import, this top-level await runs before the app.
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
if (endpoint) {
try {
const enterprise = await import("@snapotter/enterprise");
const licenseKey = process.env.LICENSE_KEY ?? "";
if (licenseKey) {
enterprise.initEnterprise(licenseKey);
}
if (enterprise.isFeatureEnabled("distributed_tracing")) {
await initTracing();
console.log("[tracing] OpenTelemetry initialized, exporting to", endpoint);
}
} catch {
// Enterprise package not available or license invalid -- tracing stays off
}
}