fix(api): refuse HQ inpainting on CPU hosts below the measured memory floor (#685)

SD1.5 inpainting is OOM-killed at the stock 6g compose limit on CPU hosts and completes at 8g. Read the cgroup limit and refuse hq up front with an actionable message; GPU hosts and unlimited containers are untouched. Fixes #670.

apps/api/src/lib/hq-memory-gate.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from "node:fs";
+
+// Measured on an amd64 CPU host (#670): SD1.5 inpainting is OOM-killed at the
+// stock 6g compose limit and completes at 8g (503s). GPU hosts run it in VRAM,
+// so only CPU inference needs this floor.
+const HQ_CPU_MIN_MEMORY_BYTES = 7.5 * 1024 ** 3;
+
+/** Effective cgroup memory limit, or null when unlimited or unknown (bare metal). */
+export function containerMemoryLimitBytes(): number | null {
+  for (const path of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
+    let raw: string;
+    try {
+      raw = readFileSync(path, "utf8").trim();
+    } catch {
+      continue;
+    }
+    if (raw === "max") return null;
+    const value = Number(raw);
+    if (!Number.isFinite(value)) return null;
+    // cgroup v1 reports "unlimited" as a page-rounded near-2^63 sentinel.
+    if (value >= 2 ** 60) return null;
+    return value;
+  }
+  return null;
+}
+
+/** Refusal message when HQ inpainting cannot fit this container, else null. */
+export function hqCpuMemoryRefusal(
+  gpuAvailable: boolean,
+  limitBytes: number | null,
+): string | null {
+  if (gpuAvailable || limitBytes === null || limitBytes >= HQ_CPU_MIN_MEMORY_BYTES) {
+    return null;
+  }
+  const gib = (limitBytes / 1024 ** 3).toFixed(1);
+  return (
+    `HQ inpainting needs at least 8g of container memory on CPU hosts ` +
+    `(this container is limited to ${gib}g). Raise mem_limit or use the fast quality mode.`
+  );
+}

apps/api/src/routes/tools/erase-object.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { inpaint } from "@snapotter/ai";
+import { inpaint, isGpuAvailable } from "@snapotter/ai";
 import { FEATURE_BUNDLES, getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import sharp from "sharp";
@@ -13,6 +13,7 @@ import { validateImageBuffer } from "../../lib/file-validation.js";
 import { decodeToSharpCompat, needsCliDecode } from "../../lib/format-decoders.js";
 import { encodeJxl } from "../../lib/format-encoders.js";
 import { decodeHeic, encodeHeic } from "../../lib/heic-converter.js";
+import { containerMemoryLimitBytes, hqCpuMemoryRefusal } from "../../lib/hq-memory-gate.js";
 import { getObjectBuffer, putObject } from "../../lib/object-storage.js";
 import { resolveOutputFormat } from "../../lib/output-format.js";
 import { receiveUpload } from "../../lib/upload-stream.js";
@@ -158,6 +159,18 @@ export function registerEraseObject(app: FastifyInstance) {
         });
       }
 
+      // Refuse HQ up front on CPU hosts whose cgroup limit cannot fit SD1.5
+      // inference; the alternative is an OOM-killed dispatcher ~40s in and a
+      // "try a smaller image" hint that cannot help (#670).
+      if (qualityMode === "hq") {
+        const refusal = hqCpuMemoryRefusal(isGpuAvailable(), containerMemoryLimitBytes());
+        if (refusal) {
+          return reply
+            .status(422)
+            .send({ error: "Not enough memory for HQ mode", details: refusal });
+        }
+      }
+
       if (format === "auto") {
         const detected = await resolveOutputFormat(imageBuffer, filename);
         format = detected.format === "jpeg" ? "jpg" : detected.format;

docker/docker-compose.yml

@@ -75,6 +75,9 @@ services:
       redis:
         condition: service_healthy
     # --- Security hardening ---
+    # erase-object's HQ quality mode (SD1.5 inpainting) needs mem_limit >= 8g
+    # when running on CPU; at 6g the API refuses HQ with a clear error. Fast
+    # mode and every other tool fit in 6g.
     mem_limit: 6g
     memswap_limit: 6g
     cpus: 4

tests/unit/api/hq-memory-gate.test.ts

@@ -0,0 +1,73 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const mockReadFileSync = vi.hoisted(() => vi.fn());
+
+vi.mock("node:fs", () => ({
+  readFileSync: mockReadFileSync,
+}));
+
+import {
+  containerMemoryLimitBytes,
+  hqCpuMemoryRefusal,
+} from "../../../apps/api/src/lib/hq-memory-gate.js";
+
+const GIB = 1024 ** 3;
+
+function cgroupFiles(files: Record<string, string>) {
+  mockReadFileSync.mockImplementation((path: string) => {
+    if (path in files) return files[path];
+    throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
+  });
+}
+
+afterEach(() => {
+  mockReadFileSync.mockReset();
+});
+
+describe("containerMemoryLimitBytes", () => {
+  it("reads a numeric cgroup v2 limit", () => {
+    cgroupFiles({ "/sys/fs/cgroup/memory.max": `${6 * GIB}\n` });
+    expect(containerMemoryLimitBytes()).toBe(6 * GIB);
+  });
+
+  it("treats cgroup v2 'max' as unlimited", () => {
+    cgroupFiles({ "/sys/fs/cgroup/memory.max": "max\n" });
+    expect(containerMemoryLimitBytes()).toBeNull();
+  });
+
+  it("falls back to the cgroup v1 file and ignores the unlimited sentinel", () => {
+    cgroupFiles({ "/sys/fs/cgroup/memory/memory.limit_in_bytes": "9223372036854771712\n" });
+    expect(containerMemoryLimitBytes()).toBeNull();
+  });
+
+  it("reads a numeric cgroup v1 limit", () => {
+    cgroupFiles({ "/sys/fs/cgroup/memory/memory.limit_in_bytes": `${8 * GIB}\n` });
+    expect(containerMemoryLimitBytes()).toBe(8 * GIB);
+  });
+
+  it("returns null when no cgroup file exists (bare metal)", () => {
+    cgroupFiles({});
+    expect(containerMemoryLimitBytes()).toBeNull();
+  });
+});
+
+describe("hqCpuMemoryRefusal", () => {
+  it("refuses a 6g CPU container with an actionable message", () => {
+    const msg = hqCpuMemoryRefusal(false, 6 * GIB);
+    expect(msg).toContain("8g");
+    expect(msg).toContain("fast quality mode");
+    expect(msg).toContain("6.0g");
+  });
+
+  it("allows 8g on CPU (the measured floor)", () => {
+    expect(hqCpuMemoryRefusal(false, 8 * GIB)).toBeNull();
+  });
+
+  it("never refuses when a GPU is available", () => {
+    expect(hqCpuMemoryRefusal(true, 6 * GIB)).toBeNull();
+  });
+
+  it("never refuses when the limit is unknown", () => {
+    expect(hqCpuMemoryRefusal(false, null)).toBeNull();
+  });
+});