Merge pull request #37 from funkypenguin/fix/bound-restart-and-reclaim-stalled

apps/api/src/config.ts

@@ -13,6 +13,16 @@ export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYC
 // minimal while still allowing CF/Imperva challenges to resolve. Raise if specific
 // targets fail with empty content (rare).
 export const CONTENT_PROCESSES = Number(process.env.BROWSER_CONTENT_PROCESSES ?? "2")
+// How long a browser may stay checked out before the pool calls it wedged rather than
+// busy. A scrape's own budget is req.maxTimeout (default 60s), so 3x that is well clear
+// of anything legitimate while still catching a hung checkout within a few minutes.
+export const STALL_TIMEOUT_MS = Number(process.env.BROWSER_STALL_TIMEOUT_MS ?? "180000")
+// Upper bound on a single browser/context close during a recycle. Camoufox can hang on
+// close when a content process is wedged; past this we abandon the close and relaunch.
+export const CLOSE_TIMEOUT_MS = Number(process.env.BROWSER_CLOSE_TIMEOUT_MS ?? "10000")
+// Upper bound on a browser launch. A cold Camoufox start is a few seconds, but launches
+// have been observed to hang indefinitely — without a bound that strands the pool entry.
+export const LAUNCH_TIMEOUT_MS = Number(process.env.BROWSER_LAUNCH_TIMEOUT_MS ?? "90000")
 
 // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single
 // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source

apps/api/src/deps.ts

@@ -3,13 +3,16 @@ import type { OrchestratorDeps } from "@trawl/tiers"
 import type { SessionData } from "@trawl/types"
 import {
   ACQUIRE_TIMEOUT_MS,
+  CLOSE_TIMEOUT_MS,
   CONTENT_PROCESSES,
+  LAUNCH_TIMEOUT_MS,
   POOL_SIZE,
   proxyPool,
   RECYCLE_AFTER_TEMPORARY_CONTEXTS,
   REDIS_URL,
   residentialProxyPool,
   SESSION_TTL,
+  STALL_TIMEOUT_MS,
 } from "./config"
 
 const state: {
@@ -36,6 +39,9 @@ export const initPool = async (): Promise<void> => {
     acquireTimeoutMs: ACQUIRE_TIMEOUT_MS,
     recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS,
     contentProcesses: CONTENT_PROCESSES,
+    stallAfterMs: STALL_TIMEOUT_MS,
+    closeTimeoutMs: CLOSE_TIMEOUT_MS,
+    launchTimeoutMs: LAUNCH_TIMEOUT_MS,
   })
   await state.pool.init()
   state.pool.startHealthCheck()
@@ -54,8 +60,8 @@ export const getDeps = (): OrchestratorDeps => {
   const sc = state.sessionCache
   const pcc = state.persistentContextCache
   return {
-    acquireBrowser: (d: string) => p.acquire(d),
-    releaseBrowser: (id: number) => p.release(id),
+    acquireBrowser: (d: string, budgetMs?: number) => p.acquire(d, budgetMs),
+    releaseBrowser: (id: number, lease?: number) => p.release(id, lease),
     loadSession: (d: string) => (sc ? sc.load(d).catch(() => undefined) : Promise.resolve(undefined)),
     saveSession: (d: string, data: SessionData) => (sc ? sc.save(d, data).catch(() => {}) : Promise.resolve()),
     invalidateSession: (d: string) => (sc ? sc.invalidate(d).catch(() => {}) : Promise.resolve()),

apps/api/src/routes/health.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from "bun:test"
+import type { PoolStats } from "@trawl/types"
+import { healthRoute } from "./health"
+
+const stats = (live: number): PoolStats => ({
+  total: 1,
+  busy: live ? 1 : 0,
+  available: 0,
+  restarts: 0,
+  avgRestarts: 0,
+  stalled: live ? 0 : 1,
+  live,
+})
+
+describe("GET /health", () => {
+  test("returns 200 while the pool has live capacity", async () => {
+    const response = await healthRoute(() => stats(1)).handle(new Request("http://localhost/health"))
+
+    expect(response.status).toBe(200)
+    expect(await response.json()).toMatchObject({ status: "ok", pool: { live: 1 } })
+  })
+
+  test("returns 503 when no live capacity remains", async () => {
+    const response = await healthRoute(() => stats(0)).handle(new Request("http://localhost/health"))
+
+    expect(response.status).toBe(503)
+    expect(await response.json()).toMatchObject({ status: "starting", pool: { live: 0 } })
+  })
+
+  test("returns 503 before the pool is available", async () => {
+    const response = await healthRoute(() => undefined).handle(new Request("http://localhost/health"))
+
+    expect(response.status).toBe(503)
+    expect(await response.json()).toMatchObject({ status: "starting", pool: { total: 0 } })
+  })
+})

apps/api/src/routes/health.ts

@@ -3,21 +3,36 @@ import { Elysia } from "elysia"
 import { startTime } from "../config"
 import { getPool } from "../deps"
 
-export function healthRoute() {
+export function healthRoute(getStats = () => getPool()?.getStats()) {
   return new Elysia().get("/health", ({ set }) => {
-    const pool = getPool()
-    if (!pool) set.status = 503
+    const stats = getStats()
+    // `pool` is assigned before `await pool.init()` completes, so a non-null pool does
+    // not mean any browser is warm — gate readiness on real capacity instead.
+    //
+    // `available + busy > 0` looks like the right test — a busy browser is still a live
+    // browser — but it isn't. A request that hangs mid-solve never reaches the
+    // orchestrator's `finally`, so it never calls release() and its browser stays `busy`
+    // for the life of the process. `busy` therefore counts dead entries, and the pod can
+    // report 200/"ok" indefinitely with zero usable browsers.
+    //
+    // `live` excludes both restarting entries (no browser attached) and entries whose
+    // checkout has outlived the stall threshold, so it cannot be propped up by a wedge.
+    // It still counts genuinely in-flight work, so a merely saturated pool won't flap.
+    const ready = Boolean(stats && stats.live > 0)
+    if (!ready) set.status = 503
     return {
-      status: pool ? "ok" : "starting",
+      status: ready ? "ok" : "starting",
       uptime: Math.floor((Date.now() - startTime) / 1000),
       pool:
-        pool?.getStats() ??
+        stats ??
         ({
           total: 0,
           busy: 0,
           available: 0,
           restarts: 0,
           avgRestarts: 0,
+          stalled: 0,
+          live: 0,
         } satisfies PoolStats),
     }
   })

apps/api/src/routes/stats.ts

@@ -9,11 +9,15 @@ export function statsRoute() {
       available: 0,
       restarts: 0,
       avgRestarts: 0,
+      stalled: 0,
+      live: 0,
     }
     return {
       browsers: stats.total,
       available: stats.available,
       busy: stats.busy,
+      stalled: stats.stalled,
+      live: stats.live,
       restarts: stats.restarts,
       queueDepth: 0,
     }