diff --git a/README.md b/README.md index 01cb5bb..078f5fb 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ Synology note: many Synology NAS units (DSM 7.x on J4125 / older hardware) ship | ---------------------------- | ------------------------ | ------------------------------------------------------------------------ | | `BROWSER_POOL_SIZE` | `3` | Warm Camoufox Firefox instances | | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before HTTP 429 is returned | +| `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Restart a browser after this many fresh/proxy contexts; set `0` to disable | | `SESSION_TTL_SECONDS` | `3600` | Redis session cache TTL (seconds) | | `REDIS_URL` | `redis://localhost:6379` | Redis connection string | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 2b821a2..c047bcb 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -20,6 +20,7 @@ const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3") // Tune lower for fast-fail feedback in dev; tune higher for very heavy upstream targets. const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000") const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600") +const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS ?? "8") // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source @@ -49,7 +50,11 @@ async function initPool() { console.warn("[api] session cache unavailable — Tier 2 disabled:", err instanceof Error ? err.message : err) } - pool = new BrowserPool({ poolSize: POOL_SIZE, acquireTimeoutMs: ACQUIRE_TIMEOUT_MS }) + pool = new BrowserPool({ + poolSize: POOL_SIZE, + acquireTimeoutMs: ACQUIRE_TIMEOUT_MS, + recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS, + }) await pool.init() pool.startHealthCheck() console.log(`[api] ready — all ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"} warm`) diff --git a/apps/docs/architecture/browser-pool.md b/apps/docs/architecture/browser-pool.md index ae5ae06..b3e32ca 100644 --- a/apps/docs/architecture/browser-pool.md +++ b/apps/docs/architecture/browser-pool.md @@ -19,6 +19,8 @@ interface PoolEntry { lastUsedAt?: number // unix timestamp restartCount: number healthy: boolean + temporaryContextUses: number + restartReason?: string } ``` @@ -42,6 +44,7 @@ new BrowserPool({ poolSize: 3, // BROWSER_POOL_SIZE (default 3) acquireTimeoutMs: 15000, // BROWSER_ACQUIRE_TIMEOUT_MS — 15s default pollIntervalMs: 100, // how often to re-check for an idle browser + recycleAfterTemporaryContexts: 8, }) ``` @@ -51,6 +54,8 @@ When `acquireTimeoutMs` elapses, the API surfaces the rejection as **HTTP 429** `pool.release(id)` marks the browser idle and closes all open pages. `lastDomain` is updated to the domain just served. Cookies are kept to speed up the next request to the same domain. +Tier 3 and Tier 4 create short-lived isolated contexts for fresh challenge solves and proxy escalation. Those contexts are closed by the tier code, but long-running Firefox/Camoufox processes can still retain child content processes after repeated solves. The pool tracks those temporary contexts and restarts the whole browser after `recycleAfterTemporaryContexts` uses so process growth stays bounded. Set `BROWSER_RECYCLE_AFTER_CONTEXTS=0` to disable this recycling. + ## Self-healing A health check runs every 30 seconds: diff --git a/apps/docs/deployment/docker-compose.md b/apps/docs/deployment/docker-compose.md index 6d1dc4d..d6be3db 100644 --- a/apps/docs/deployment/docker-compose.md +++ b/apps/docs/deployment/docker-compose.md @@ -88,6 +88,7 @@ First run builds the web and docs images locally — takes a couple of minutes. |----------|---------|-------------| | `BROWSER_POOL_SIZE` | `3` | Warm browser instances | | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before returning HTTP 429 | +| `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Restart a browser after this many fresh/proxy contexts; set `0` to disable | | `REDIS_URL` | `redis://redis:6379` | Redis connection (set automatically in compose) | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index 3282051..25b996f 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -63,6 +63,17 @@ BROWSER_ACQUIRE_TIMEOUT_MS=30000 # tolerate longer queueing on slow targets When the timeout fires, both `/v1` and `/scrape` return **HTTP 429** with the FlareSolverr v2 error envelope (not a 500). +### `BROWSER_RECYCLE_AFTER_CONTEXTS` + +**Default:** `8` + +How many fresh challenge/proxy contexts a pooled browser can create before TRAWL restarts the full browser process. Tier 3 and Tier 4 use short-lived isolated contexts so Cloudflare sees a clean profile, but some Camoufox/Firefox builds can leave content processes behind even after Playwright closes those contexts. Recycling the browser bounds that process growth without changing Redis session-cache TTLs. + +```ini +BROWSER_RECYCLE_AFTER_CONTEXTS=8 # default - bound long-running browser process growth +BROWSER_RECYCLE_AFTER_CONTEXTS=0 # disable browser recycling +``` + ## Session Cache ### `SESSION_TTL_SECONDS` diff --git a/packages/browser/src/pool.ts b/packages/browser/src/pool.ts index 8603cca..58a7699 100644 --- a/packages/browser/src/pool.ts +++ b/packages/browser/src/pool.ts @@ -18,43 +18,67 @@ export interface BrowserHandle { id: number context: BrowserContext browser: Browser + noteTemporaryContext?: (reason: string) => void } interface PoolEntry extends PoolBrowser { browser: Browser | null context: BrowserContext | null + temporaryContextUses: number + restartReason?: string + restarting?: boolean } +type BrowserFactory = () => Promise<{ browser: Browser; context: BrowserContext }> + export class BrowserPool { private entries: PoolEntry[] = [] private poolSize: number private acquireTimeoutMs: number private pollIntervalMs: number + private recycleAfterTemporaryContexts: number + private browserFactory?: BrowserFactory private healthInterval: ReturnType | null = null constructor({ poolSize, acquireTimeoutMs = 15_000, pollIntervalMs = 100, + recycleAfterTemporaryContexts = 8, + browserFactory, }: { poolSize: number acquireTimeoutMs?: number pollIntervalMs?: number + recycleAfterTemporaryContexts?: number + browserFactory?: BrowserFactory }) { this.poolSize = poolSize this.acquireTimeoutMs = acquireTimeoutMs this.pollIntervalMs = pollIntervalMs + this.recycleAfterTemporaryContexts = recycleAfterTemporaryContexts + this.browserFactory = browserFactory } async init(): Promise { for (let i = 0; i < this.poolSize; i++) { const { browser, context } = await this.launchBrowser() - this.entries.push({ id: i, busy: false, restartCount: 0, healthy: true, browser, context }) + this.entries.push({ + id: i, + busy: false, + restartCount: 0, + healthy: true, + browser, + context, + temporaryContextUses: 0, + }) console.log(`[pool] browser ${i + 1}/${this.poolSize} ready`) } } private async launchBrowser(): Promise<{ browser: Browser; context: BrowserContext }> { + if (this.browserFactory) return this.browserFactory() + // Camoufox patches fingerprint data at the C++/Juggler level — not via JS injection. // CF's JS cannot detect these patches the way it detects overrides of window.chrome, // plugins, WebGL etc. Same browser Byparr uses. @@ -117,7 +141,14 @@ export class BrowserPool { entry.busy = true entry.lastDomain = domain entry.lastUsedAt = Date.now() - resolve({ id: entry.id, context: entry.context, browser: entry.browser }) + resolve({ + id: entry.id, + context: entry.context, + browser: entry.browser, + noteTemporaryContext: (reason: string) => { + this.noteTemporaryContext(entry, reason) + }, + }) return true } @@ -138,7 +169,7 @@ export class BrowserPool { } private pickEntry(domain?: string): PoolEntry | null { - const available = this.entries.filter((e) => !e.busy && e.healthy && e.context) + const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context) if (available.length === 0) return null if (domain) { const sticky = available.find((e) => e.lastDomain === domain) @@ -147,6 +178,15 @@ export class BrowserPool { return available[0] } + private noteTemporaryContext(entry: PoolEntry, reason: string): void { + if (this.recycleAfterTemporaryContexts <= 0) return + + entry.temporaryContextUses++ + if (entry.temporaryContextUses >= this.recycleAfterTemporaryContexts) { + entry.restartReason = `${reason}; ${entry.temporaryContextUses} temporary contexts used` + } + } + release(id: number): void { const entry = this.entries.find((e) => e.id === id) if (!entry) return @@ -157,6 +197,9 @@ export class BrowserPool { const pages: unknown[] = entry.context.pages() ?? [] for (const p of pages) (p as { close: () => Promise }).close().catch(() => {}) } + if (entry.restartReason) { + void this.restartEntry(entry, entry.restartReason) + } } startHealthCheck(): void { @@ -168,15 +211,22 @@ export class BrowserPool { if (entry.busy) continue if (!(entry.browser?.isConnected() ?? false)) { console.warn(`[pool] browser ${entry.id} disconnected, restarting`) - await this.restartEntry(entry) + await this.restartEntry(entry, "browser disconnected") } else { entry.healthy = true } } } - private async restartEntry(entry: PoolEntry): Promise { + private async restartEntry(entry: PoolEntry, reason = "manual restart"): Promise { + if (entry.restarting) { + entry.restartReason ??= reason + return + } + entry.restarting = true entry.healthy = false + entry.restartReason = undefined + console.warn(`[pool] browser ${entry.id} restarting: ${reason}`) try { await entry.context?.close() } catch {} @@ -190,20 +240,24 @@ export class BrowserPool { entry.browser = browser entry.context = context entry.healthy = true + entry.temporaryContextUses = 0 entry.restartCount++ console.log(`[pool] browser ${entry.id} restarted (total: ${entry.restartCount})`) } catch (err) { console.error(`[pool] browser ${entry.id} failed to restart:`, err) + } finally { + entry.restarting = false } } getStats(): PoolStats { const busy = this.entries.filter((e) => e.busy).length + const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context).length const totalRestarts = this.entries.reduce((sum, e) => sum + e.restartCount, 0) return { total: this.poolSize, busy, - available: this.poolSize - busy, + available, restarts: totalRestarts, avgRestarts: totalRestarts / this.poolSize, } diff --git a/packages/browser/tests/pool.test.ts b/packages/browser/tests/pool.test.ts new file mode 100644 index 0000000..f51f122 --- /dev/null +++ b/packages/browser/tests/pool.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test" +import { BrowserPool } from "../src/pool" + +const waitFor = async (predicate: () => boolean) => { + const deadline = Date.now() + 1000 + while (Date.now() < deadline) { + if (predicate()) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error("timed out waiting for condition") +} + +describe("BrowserPool recycling", () => { + test("restarts the browser after the temporary context threshold", async () => { + const browsers: Array<{ closed: boolean; isConnected: () => boolean; close: () => Promise }> = [] + const contexts: Array<{ closed: boolean; pages: () => unknown[]; close: () => Promise }> = [] + + const pool = new BrowserPool({ + poolSize: 1, + recycleAfterTemporaryContexts: 2, + browserFactory: async () => { + const browser = { + closed: false, + isConnected() { + return !this.closed + }, + async close() { + this.closed = true + }, + } + const context = { + closed: false, + pages: () => [], + async close() { + this.closed = true + }, + } + browsers.push(browser) + contexts.push(context) + return { browser, context } + }, + }) + + await pool.init() + + const first = await pool.acquire("example.com") + first.noteTemporaryContext?.("tier3 fresh context") + pool.release(first.id) + + expect(pool.getStats().restarts).toBe(0) + expect(pool.getStats().available).toBe(1) + + const second = await pool.acquire("example.com") + second.noteTemporaryContext?.("tier3 fresh context") + pool.release(second.id) + + await waitFor(() => pool.getStats().restarts === 1) + + expect(contexts[0].closed).toBe(true) + expect(browsers[0].closed).toBe(true) + expect(pool.getStats().available).toBe(1) + expect(browsers).toHaveLength(2) + }) +}) diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index 5995771..cedcf0f 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -35,6 +35,7 @@ export async function runTier3( // challenge evaluation. A fresh context with no prior state gets managed-mode treatment: // CF evaluates in under 1s and the challenge resolves in 3-4s total. const freshCtx = await newFreshContext(handle.browser, { proxy: proxyUrl }) + handle.noteTemporaryContext?.("tier3 fresh context") const page = await freshCtx.newPage() try { diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index 43e83e8..6b80d55 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -40,6 +40,7 @@ export async function runTier4( proxy: { server: proxyUrl }, viewport: null, }) + handle.noteTemporaryContext?.("tier4 proxy context") await proxyContext.addInitScript(() => { window.onerror = () => true window.addEventListener(