diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 09fa472..b1dedb9 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -13,6 +13,10 @@ export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYC // minimal while still allowing CF/Imperva challenges to resolve. Raise if specific // targets fail with empty content (rare). export const CONTENT_PROCESSES = Number(process.env.BROWSER_CONTENT_PROCESSES ?? "2") +// How long a browser may stay checked out before the pool calls it wedged rather than +// busy. A scrape's own budget is req.maxTimeout (default 60s), so 3x that is well clear +// of anything legitimate while still catching a hung checkout within a few minutes. +export const STALL_TIMEOUT_MS = Number(process.env.BROWSER_STALL_TIMEOUT_MS ?? "180000") // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source diff --git a/apps/api/src/deps.ts b/apps/api/src/deps.ts index 76d4399..b109eb7 100644 --- a/apps/api/src/deps.ts +++ b/apps/api/src/deps.ts @@ -8,6 +8,7 @@ import { REDIS_URL, residentialProxyPool, SESSION_TTL, + STALL_TIMEOUT_MS, } from "./config" // Single embedded pool — no BullMQ / worker process required. @@ -36,6 +37,7 @@ export async function initPool() { acquireTimeoutMs: ACQUIRE_TIMEOUT_MS, recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS, contentProcesses: CONTENT_PROCESSES, + stallAfterMs: STALL_TIMEOUT_MS, }) await pool.init() pool.startHealthCheck() diff --git a/apps/api/src/routes/health.ts b/apps/api/src/routes/health.ts index ae9fc0c..033e33c 100644 --- a/apps/api/src/routes/health.ts +++ b/apps/api/src/routes/health.ts @@ -6,18 +6,34 @@ import { getPool } from "../deps" export function healthRoute() { return new Elysia().get("/health", ({ set }) => { const pool = getPool() - if (!pool) set.status = 503 + const stats = pool?.getStats() + // `pool` is assigned before `await pool.init()` completes, so a non-null pool does + // not mean any browser is warm — gate readiness on real capacity instead. + // + // `available + busy > 0` looks like the right test — a busy browser is still a live + // browser — but it isn't. A request that hangs mid-solve never reaches the + // orchestrator's `finally`, so it never calls release() and its browser stays `busy` + // for the life of the process. `busy` therefore counts dead entries, and the pod can + // report 200/"ok" indefinitely with zero usable browsers. + // + // `live` excludes both restarting entries (no browser attached) and entries whose + // checkout has outlived the stall threshold, so it cannot be propped up by a wedge. + // It still counts genuinely in-flight work, so a merely saturated pool won't flap. + const ready = Boolean(stats && stats.live > 0) + if (!ready) set.status = 503 return { - status: pool ? "ok" : "starting", + status: ready ? "ok" : "starting", uptime: Math.floor((Date.now() - startTime) / 1000), pool: - pool?.getStats() ?? + stats ?? ({ total: 0, busy: 0, available: 0, restarts: 0, avgRestarts: 0, + stalled: 0, + live: 0, } satisfies PoolStats), } }) diff --git a/apps/api/src/routes/stats.ts b/apps/api/src/routes/stats.ts index 65a2605..b25a98f 100644 --- a/apps/api/src/routes/stats.ts +++ b/apps/api/src/routes/stats.ts @@ -9,11 +9,15 @@ export function statsRoute() { available: 0, restarts: 0, avgRestarts: 0, + stalled: 0, + live: 0, } return { browsers: stats.total, available: stats.available, busy: stats.busy, + stalled: stats.stalled, + live: stats.live, restarts: stats.restarts, queueDepth: 0, } diff --git a/packages/browser/src/pool.ts b/packages/browser/src/pool.ts index 95fad3a..65ac385 100644 --- a/packages/browser/src/pool.ts +++ b/packages/browser/src/pool.ts @@ -40,6 +40,7 @@ export class BrowserPool { private pollIntervalMs: number private recycleAfterTemporaryContexts: number private contentProcesses!: number + private stallAfterMs: number private browserFactory?: BrowserFactory private healthInterval: ReturnType | null = null @@ -49,6 +50,7 @@ export class BrowserPool { pollIntervalMs = 100, recycleAfterTemporaryContexts = 8, contentProcesses = 2, + stallAfterMs = 180_000, browserFactory, }: { poolSize: number @@ -56,6 +58,7 @@ export class BrowserPool { pollIntervalMs?: number recycleAfterTemporaryContexts?: number contentProcesses?: number + stallAfterMs?: number browserFactory?: BrowserFactory }) { this.poolSize = poolSize @@ -63,9 +66,17 @@ export class BrowserPool { this.pollIntervalMs = pollIntervalMs this.recycleAfterTemporaryContexts = recycleAfterTemporaryContexts this.contentProcesses = contentProcesses + this.stallAfterMs = stallAfterMs this.browserFactory = browserFactory } + // A checkout longer than this is not slow, it's wedged: the orchestrator's own budget + // is req.maxTimeout (default 60s), so nothing legitimate holds a browser for minutes. + private isStalled(entry: PoolEntry, now = Date.now()): boolean { + if (!entry.busy || entry.busySince === undefined) return false + return now - entry.busySince > this.stallAfterMs + } + async init(): Promise { for (let i = 0; i < this.poolSize; i++) { // Pick a fingerprint for this instance; the picked OS drives the browser's @@ -188,6 +199,7 @@ export class BrowserPool { if (!entry) return false if (!entry.context || !entry.browser) return false entry.busy = true + entry.busySince = Date.now() entry.lastDomain = domain entry.lastUsedAt = Date.now() resolve({ @@ -244,6 +256,7 @@ export class BrowserPool { const entry = this.entries.find((e) => e.id === id) if (!entry) return entry.busy = false + entry.busySince = undefined // Keep the context alive — CF cookies (cf_clearance, __cf_bm) and browser cache // accumulate, making subsequent challenges faster. Cookies are domain-scoped. if (entry.context) { @@ -307,8 +320,10 @@ export class BrowserPool { } getStats(): PoolStats { + const now = Date.now() const busy = this.entries.filter((e) => e.busy).length const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context).length + const stalled = this.entries.filter((e) => this.isStalled(e, now)).length const totalRestarts = this.entries.reduce((sum, e) => sum + e.restartCount, 0) return { total: this.poolSize, @@ -316,6 +331,10 @@ export class BrowserPool { available, restarts: totalRestarts, avgRestarts: totalRestarts / this.poolSize, + stalled, + // Entries that can serve work now or are genuinely mid-request. Excludes entries + // that are restarting (no browser attached) and entries wedged in a dead checkout. + live: available + (busy - stalled), } } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 7f51821..aba11e3 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -59,6 +59,9 @@ export interface SessionData { export interface PoolBrowser { id: number busy: boolean + // When the current checkout started, or undefined when idle. Used to tell a browser + // that is busy doing work from one whose request wedged and left it busy forever. + busySince?: number lastDomain?: string lastUsedAt?: number restartCount: number @@ -71,6 +74,11 @@ export interface PoolStats { available: number restarts: number avgRestarts: number + // Subset of `busy` that has been checked out longer than the pool's stall threshold. + // A stalled entry is counted in `busy` but is not real capacity — its request wedged + // and will never call release(). `live` is the honest capacity number. + stalled: number + live: number } // Per-instance HTTP-level fingerprint (User-Agent + matching navigator.platform /