fix: improve AI feature install error handling and resource limits

Handle OOM kills (exit code 137) with actionable memory guidance,
filter ANSI/progress noise from error output, add --no-cache-dir to
pip installs, reduce download concurrency to 2, and bump default
container memory from 4g to 6g.
This commit is contained in:
SnapOtter
2026-05-18 10:19:43 +08:00
parent d978ae211b
commit 36431bce48
3 changed files with 22 additions and 11 deletions
+18 -7
View File
@@ -225,13 +225,24 @@ export async function registerFeatureRoutes(app: FastifyInstance): Promise<void>
} }
} }
if (!errorMsg) { if (!errorMsg) {
const meaningful = lastStderrLines.filter( if (code === 137) {
(l) => !l.startsWith("{") && !l.includes("pthread_setaffinity_np"), errorMsg =
); "Installation was killed due to insufficient memory. " +
errorMsg = "Try increasing the container's memory limit (e.g. mem_limit: 6g in docker-compose.yml) and retry.";
meaningful.join("\n") || } else {
stdoutBuffer.trim() || const meaningful = lastStderrLines.filter(
`Install failed with exit code ${code}`; (l) =>
!l.startsWith("{") &&
!l.includes("pthread_setaffinity_np") &&
!l.includes("\x1b[") &&
!l.includes("━") &&
!/^\s*\d+%\|/.test(l),
);
errorMsg =
meaningful.join("\n") ||
stdoutBuffer.trim() ||
`Install failed with exit code ${code}`;
}
} }
setInstallProgress(bundleId, null, errorMsg); setInstallProgress(bundleId, null, errorMsg);
updateSingleFileProgress({ jobId, phase: "failed", percent: 0, error: errorMsg }); updateSingleFileProgress({ jobId, phase: "failed", percent: 0, error: errorMsg });
+2 -2
View File
@@ -52,8 +52,8 @@ services:
# - COOKIE_SECRET= # - COOKIE_SECRET=
restart: unless-stopped restart: unless-stopped
# --- Security hardening --- # --- Security hardening ---
mem_limit: 4g mem_limit: 6g
memswap_limit: 4g memswap_limit: 6g
cpus: 4 cpus: 4
pids_limit: 512 pids_limit: 512
cap_drop: cap_drop:
+2 -2
View File
@@ -125,7 +125,7 @@ def check_disk_space(path: str, min_bytes: int = 100 * 1024 * 1024) -> None:
def pip_install(package: str, extra_flags: list[str] | None = None) -> None: def pip_install(package: str, extra_flags: list[str] | None = None) -> None:
"""Run pip install for a single package spec. Raises on failure.""" """Run pip install for a single package spec. Raises on failure."""
cmd = [sys.executable, "-m", "pip", "install"] cmd = [sys.executable, "-m", "pip", "install", "--no-cache-dir"]
if extra_flags: if extra_flags:
cmd.extend(extra_flags) cmd.extend(extra_flags)
@@ -471,7 +471,7 @@ def download_models(models: list[dict], models_dir: str) -> list[str]:
except Exception as e: except Exception as e:
return (model_id, e) return (model_id, e)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
futures = { futures = {
pool.submit(_download, i, m): i pool.submit(_download, i, m): i
for i, m in enumerate(models) for i, m in enumerate(models)