External/SnapOtter
1
0
Fork 0
mirror of https://github.com/snapotter-hq/SnapOtter.git synced 2026-08-03 07:46:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai-bridge): surface sidecar exit reasons in Sentry via SafeError (#538)

Browse Source 
Route both Python exit paths through pythonExitError so the reason survives the scrubber; OOM/segfault stay operational and keep "out of memory" for the lighter-model fallback.
This commit is contained in:
SnapOtter
2026-07-16 19:26:15 +08:00
committed by GitHub
parent 9cccbc9576
commit 55e1e95f20
2 changed files with 97 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/bridge.ts
+37 -32
View File
@@ -80,6 +80,39 @@ function getPythonPath(): string {
/**
* Extract a user-friendly error from a Python process error.
*/
const OOM_EXIT_TEXT =
/out of memory|failed to allocate|cudaerrormemoryallocation|cublas_status_alloc_failed|bad_alloc/i;
/**
* Build the error to reject a non-zero Python exit with. The extracted reason is
* kept as the message (it is already surfaced to callers) but wrapped in a
* SafeError so it survives the API's Sentry scrubber, which otherwise reduces a
* plain Error to "Error: Error" (NODE-24). Classification is deliberate: a kill
* signal / OS OOM (SIGKILL, exit 137) and a segfault (SIGSEGV, exit 139) are
* operational, and their constant messages keep the "out of memory" text so
* memory-aware callers still retry on a lighter model. Any other non-zero exit
* is a bug, with the extracted reason (or an OOM reported in-band) preserved.
*/
function pythonExitError(code: number | null, signal: string | null, extracted: string): SafeError {
if (signal === "SIGSEGV" || code === 139) {
return new SafeError("Process crashed (segmentation fault)", {
kind: "operational",
code: `exit-${code ?? signal ?? "unknown"}`,
});
}
if (signal === "SIGKILL" || code === 137) {
return new SafeError("Process killed (out of memory) -- try a lighter model or smaller image", {
kind: "operational",
code: `exit-${code ?? signal ?? "unknown"}`,
});
}
const message = extracted || `Python script exited with code ${code}`;
return new SafeError(message, {
kind: OOM_EXIT_TEXT.test(message) ? "operational" : "bug",
code: `exit-${code ?? "unknown"}`,
});
}
function extractPythonError(error: unknown): string {
if (error && typeof error === "object") {
const pErr = error as {
@@ -331,20 +364,7 @@ export class PythonDispatcher {
stdout: response.stdout,
stderr: req.stderrLines.join("\n"),
});
if (!extracted && (response.exitCode === 137 || response.exitCode === 139)) {
req.reject(
new SafeError(
response.exitCode === 137
? "Process killed (out of memory) -- try a lighter model or smaller image"
: "Process crashed (segmentation fault)",
{ kind: "operational", code: `exit-${response.exitCode}` },
),
);
} else {
req.reject(
new Error(extracted || `Python script exited with code ${response.exitCode}`),
);
}
req.reject(pythonExitError(response.exitCode, null, extracted));
} else {
req.resolve({
stdout: response.stdout || "",
@@ -567,24 +587,9 @@ export class PythonDispatcher {
const stderr = stderrLines.join("\n");
if (code !== 0) {
// When the process was killed by a signal, use a clear message
// instead of surfacing unrelated stderr (e.g. CUDA warnings).
const signalMsg =
signal === "SIGKILL" || code === 137
? "Process killed (out of memory) -- try a lighter model or smaller image"
: signal === "SIGSEGV" || code === 139
? "Process crashed (segmentation fault)"
: null;
if (signalMsg) {
rejectPromise(
new SafeError(signalMsg, { kind: "operational", code: `exit-${code ?? signal}` }),
);
return;
}
const errorText =
extractPythonError({ stdout: stdout.trim(), stderr }) ||
`Python script exited with code ${code}`;
rejectPromise(new Error(errorText));
rejectPromise(
pythonExitError(code, signal, extractPythonError({ stdout: stdout.trim(), stderr })),
);
return;
}
tests/unit/ai/bridge.test.ts
+60 -9
View File
@@ -203,11 +203,16 @@ describe("bridge - runPythonWithProgress (per-request fallback)", () => {
mock.stderr.emit("data", Buffer.from("RuntimeError: model not found\n"));
mock.emitEvent("close", 1, null);
const err = (await promise.catch((e: unknown) => e)) as Error & { isSafeMessage?: unknown };
const err = (await promise.catch((e: unknown) => e)) as Error & {
isSafeMessage?: unknown;
kind?: string;
};
expect(err).toBeInstanceOf(Error);
expect(err.message).toContain("RuntimeError: model not found");
// Variable python-derived text must stay a plain Error, never a SafeError
expect(err.isSafeMessage).toBeUndefined();
// The sidecar reason is wrapped in a SafeError so it survives the Sentry
// scrubber instead of showing as "Error: Error" (NODE-24).
expect(err.isSafeMessage).toBe(true);
expect(err.kind).toBe("bug");
});
it("rejects with OOM message on exit code 137 (SIGKILL)", async () => {
@@ -1585,7 +1590,7 @@ describe("bridge - dispatcher stdin JSON-RPC protocol", () => {
expect(err.code).toBe("exit-139");
});
it("keeps extracted python text as a plain Error even on dispatcher exitCode 137", async () => {
it("classifies dispatcher exitCode 137 as an operational OOM SafeError", async () => {
const mock = await setupReadyDispatcher();
const promise = runPythonWithProgress("heavy.py", []);
@@ -1594,7 +1599,8 @@ describe("bridge - dispatcher stdin JSON-RPC protocol", () => {
const line = mock.stdinWrites.join("").split("\n").filter(Boolean)[0];
const id = JSON.parse(line).id;
// Extractable error text takes precedence over the constant signal message
// 137 is an OS OOM kill: the operational message keeps "out of memory" so
// memory-aware callers retry, and it does not surface possibly-unrelated stderr.
mock.stdout.emit(
"data",
Buffer.from(
@@ -1602,10 +1608,13 @@ describe("bridge - dispatcher stdin JSON-RPC protocol", () => {
),
);
const err = (await promise.catch((e: unknown) => e)) as Error & { isSafeMessage?: unknown };
expect(err).toBeInstanceOf(Error);
expect(err.message).toBe("CUDA out of memory");
expect(err.isSafeMessage).toBeUndefined();
const err = (await promise.catch((e: unknown) => e)) as Error & {
isSafeMessage?: unknown;
kind?: string;
};
expect(err.message).toContain("out of memory");
expect(err.isSafeMessage).toBe(true);
expect(err.kind).toBe("operational");
});
it("ignores stdout lines that are not valid JSON", async () => {
@@ -2313,6 +2322,48 @@ describe("bridge - extractPythonError via dispatcher responses", () => {
await expect(promise).rejects.toThrow("exited with code 42");
});
it("rejects a non-zero exit with a bug-classed SafeError so the reason survives Sentry scrubbing", async () => {
const mock = await setupReadyDispatcher();
const promise = runPythonWithProgress("test.py", []);
await new Promise((r) => setTimeout(r, 10));
const line = mock.stdinWrites.join("").split("\n").filter(Boolean)[0];
const id = JSON.parse(line).id;
mock.stdout.emit(
"data",
Buffer.from(
`${JSON.stringify({ id, exitCode: 1, stdout: '{"error": "model not found"}' })}\n`,
),
);
const err = (await promise.catch((e: unknown) => e)) as Error & {
isSafeMessage?: unknown;
kind?: string;
};
expect(err.message).toBe("model not found");
expect(err.isSafeMessage).toBe(true);
expect(err.kind).toBe("bug");
});
it("classifies an out-of-memory exit reported in-band as operational", async () => {
const mock = await setupReadyDispatcher();
const promise = runPythonWithProgress("test.py", []);
await new Promise((r) => setTimeout(r, 10));
const line = mock.stdinWrites.join("").split("\n").filter(Boolean)[0];
const id = JSON.parse(line).id;
mock.stdout.emit(
"data",
Buffer.from(
`${JSON.stringify({ id, exitCode: 1, stdout: '{"error": "CUDA out of memory"}' })}\n`,
),
);
const err = (await promise.catch((e: unknown) => e)) as Error & {
isSafeMessage?: unknown;
kind?: string;
};
expect(err.isSafeMessage).toBe(true);
expect(err.kind).toBe("operational");
expect(err.message).toContain("out of memory");
});
});
// ── Dispatcher partial stderr buffering ──────────────────────────────