fix: format preservation, dispatcher stability, and health reporting

Closes #17, #18, #19, #31, #32, #33, #34

Format preservation (#17, #18, #19):
- Add resolveOutputFormat to rotate, resize, text-overlay, watermark-text,
  border, replace-color, blur-faces, upscale, erase-object, restore-photo
- Alpha-aware fallback: border with corner radius/shadow and replace-color
  with makeTransparent fall back to PNG for non-alpha formats (JPEG)
- Python sidecar tools (blur-faces, upscale, erase-object) now convert
  PNG output back to input format, matching restore-photo/colorize pattern
- Upscale and erase-object default to "auto" format detection instead of PNG

Dispatcher stability (#31, #32):
- Add gc.collect() and torch.cuda.empty_cache() after each dispatcher request
- Add configurable max_requests (default 50) for periodic dispatcher restart
- Add exponential backoff to dispatcher crash recovery in bridge.ts
- Circuit breaker: 5 crashes within 60s permanently disables dispatcher
- Reset crash counter on successful dispatcher startup

Health & security (#33, #34):
- Export getDispatcherStatus() from @snapotter/ai with running/ready/failed/
  gpu/pid/consecutiveCrashes fields
- Admin health endpoint now includes full dispatcher status
- Add pip-audit job to CI workflow for Python dependency scanning
This commit is contained in:
SnapOtter
2026-04-26 03:22:26 +08:00
parent 86db131198
commit dee9452c48
15 changed files with 227 additions and 34 deletions
+60 -3
View File
@@ -74,6 +74,41 @@ let dispatcherGpuAvailable = false;
const pendingRequests = new Map<string, PendingRequest>();
let stdoutBuffer = "";
// Crash recovery with exponential backoff
// biome-ignore lint/style/useConst: reassigned on crash events
let consecutiveCrashes = 0;
// biome-ignore lint/style/useConst: reassigned on crash events
let lastCrashTime = 0;
// biome-ignore lint/style/useConst: reassigned on crash events
let backoffUntil = 0;
const CRASH_WINDOW_MS = 60_000;
const MAX_CONSECUTIVE_CRASHES = 5;
const BASE_BACKOFF_MS = 1_000;
function recordCrash(): void {
const now = Date.now();
if (now - lastCrashTime > CRASH_WINDOW_MS) {
consecutiveCrashes = 1;
} else {
consecutiveCrashes++;
}
lastCrashTime = now;
if (consecutiveCrashes >= MAX_CONSECUTIVE_CRASHES) {
console.error(
`[bridge] Dispatcher crashed ${consecutiveCrashes} times in ${CRASH_WINDOW_MS / 1000}s, disabling permanently`,
);
dispatcherFailed = true;
return;
}
const delay = BASE_BACKOFF_MS * 2 ** (consecutiveCrashes - 1);
backoffUntil = now + delay;
console.warn(
`[bridge] Dispatcher crash #${consecutiveCrashes}, backing off ${delay}ms before restart`,
);
}
function startDispatcher(): ChildProcess | null {
if (dispatcherFailed) return null;
@@ -100,6 +135,7 @@ function startDispatcher(): ChildProcess | null {
if (parsed.ready === true) {
dispatcherReady = true;
dispatcherGpuAvailable = parsed.gpu === true;
consecutiveCrashes = 0;
console.log(`[bridge] Python dispatcher ready (GPU: ${parsed.gpu === true})`);
continue;
}
@@ -168,10 +204,10 @@ function startDispatcher(): ChildProcess | null {
child.on("error", (err: NodeJS.ErrnoException) => {
console.error(`[bridge] Dispatcher error: ${err.message} (code: ${err.code})`);
if (err.code === "ENOENT") {
// Venv python not found - mark as failed, will fall back to per-request
dispatcherFailed = true;
} else {
recordCrash();
}
// Reject all pending requests
for (const [id, req] of pendingRequests.entries()) {
req.reject(new Error(extractPythonError(err)));
pendingRequests.delete(id);
@@ -181,11 +217,11 @@ function startDispatcher(): ChildProcess | null {
});
child.on("close", () => {
// Reject all pending requests
for (const [id, req] of pendingRequests.entries()) {
req.reject(new Error("Python dispatcher exited unexpectedly"));
pendingRequests.delete(id);
}
recordCrash();
dispatcher = null;
dispatcherReady = false;
});
@@ -200,6 +236,7 @@ function startDispatcher(): ChildProcess | null {
function getDispatcher(): ChildProcess | null {
if (dispatcherFailed) return null;
if (!dispatcher || dispatcher.killed) {
if (Date.now() < backoffUntil) return null;
dispatcher = startDispatcher();
}
return dispatcher;
@@ -259,6 +296,26 @@ export function isGpuAvailable(): boolean {
return dispatcherGpuAvailable;
}
export interface DispatcherStatus {
running: boolean;
ready: boolean;
failed: boolean;
gpu: boolean;
pid: number | null;
consecutiveCrashes: number;
}
export function getDispatcherStatus(): DispatcherStatus {
return {
running: dispatcher !== null && !dispatcher.killed,
ready: dispatcherReady,
failed: dispatcherFailed,
gpu: dispatcherGpuAvailable,
pid: dispatcher?.pid ?? null,
consecutiveCrashes,
};
}
/**
* Shut down the persistent dispatcher process.
*/
+2 -1
View File
@@ -1,5 +1,6 @@
export { removeBackground } from "./background-removal.js";
export { isGpuAvailable, shutdownDispatcher } from "./bridge.js";
export type { DispatcherStatus } from "./bridge.js";
export { getDispatcherStatus, isGpuAvailable, shutdownDispatcher } from "./bridge.js";
export { colorize } from "./colorization.js";
export type { DetectFacesResult, FaceRegion } from "./face-detection.js";
export { blurFaces, detectFaces } from "./face-detection.js";