Recycle browsers after temporary contexts

README.md

@@ -119,6 +119,7 @@ Synology note: many Synology NAS units (DSM 7.x on J4125 / older hardware) ship
 | ---------------------------- | ------------------------ | ------------------------------------------------------------------------ |
 | `BROWSER_POOL_SIZE`          | `3`                      | Warm Camoufox Firefox instances                                          |
 | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000`                  | How long `acquire()` polls for a free browser before HTTP 429 is returned |
+| `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8`                  | Restart a browser after this many fresh/proxy contexts; set `0` to disable |
 | `SESSION_TTL_SECONDS`        | `3600`                   | Redis session cache TTL (seconds)                                        |
 | `REDIS_URL`                  | `redis://localhost:6379` | Redis connection string                                                  |
 | `RESIDENTIAL_PROXY_URL`      | —                        | Enables Tier 4 proxy escalation                                          |

apps/api/src/index.ts

@@ -20,6 +20,7 @@ const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3")
 // Tune lower for fast-fail feedback in dev; tune higher for very heavy upstream targets.
 const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000")
 const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600")
+const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS ?? "8")
 
 // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single
 // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source
@@ -49,7 +50,11 @@ async function initPool() {
     console.warn("[api] session cache unavailable — Tier 2 disabled:", err instanceof Error ? err.message : err)
   }
 
-  pool = new BrowserPool({ poolSize: POOL_SIZE, acquireTimeoutMs: ACQUIRE_TIMEOUT_MS })
+  pool = new BrowserPool({
+    poolSize: POOL_SIZE,
+    acquireTimeoutMs: ACQUIRE_TIMEOUT_MS,
+    recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS,
+  })
   await pool.init()
   pool.startHealthCheck()
   console.log(`[api] ready — all ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"} warm`)

apps/docs/architecture/browser-pool.md

@@ -19,6 +19,8 @@ interface PoolEntry {
   lastUsedAt?: number      // unix timestamp
   restartCount: number
   healthy: boolean
+  temporaryContextUses: number
+  restartReason?: string
 }
 ```
 
@@ -42,6 +44,7 @@ new BrowserPool({
   poolSize: 3,                // BROWSER_POOL_SIZE (default 3)
   acquireTimeoutMs: 15000,    // BROWSER_ACQUIRE_TIMEOUT_MS — 15s default
   pollIntervalMs: 100,        // how often to re-check for an idle browser
+  recycleAfterTemporaryContexts: 8,
 })
 ```
 
@@ -51,6 +54,8 @@ When `acquireTimeoutMs` elapses, the API surfaces the rejection as **HTTP 429**
 
 `pool.release(id)` marks the browser idle and closes all open pages. `lastDomain` is updated to the domain just served. Cookies are kept to speed up the next request to the same domain.
 
+Tier 3 and Tier 4 create short-lived isolated contexts for fresh challenge solves and proxy escalation. Those contexts are closed by the tier code, but long-running Firefox/Camoufox processes can still retain child content processes after repeated solves. The pool tracks those temporary contexts and restarts the whole browser after `recycleAfterTemporaryContexts` uses so process growth stays bounded. Set `BROWSER_RECYCLE_AFTER_CONTEXTS=0` to disable this recycling.
+
 ## Self-healing
 
 A health check runs every 30 seconds:

apps/docs/deployment/docker-compose.md

@@ -88,6 +88,7 @@ First run builds the web and docs images locally — takes a couple of minutes.
 |----------|---------|-------------|
 | `BROWSER_POOL_SIZE` | `3` | Warm browser instances |
 | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before returning HTTP 429 |
+| `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Restart a browser after this many fresh/proxy contexts; set `0` to disable |
 | `REDIS_URL` | `redis://redis:6379` | Redis connection (set automatically in compose) |
 | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation |
 

apps/docs/getting-started/configuration.md

@@ -63,6 +63,17 @@ BROWSER_ACQUIRE_TIMEOUT_MS=30000   # tolerate longer queueing on slow targets
 
 When the timeout fires, both `/v1` and `/scrape` return **HTTP 429** with the FlareSolverr v2 error envelope (not a 500).
 
+### `BROWSER_RECYCLE_AFTER_CONTEXTS`
+
+**Default:** `8`
+
+How many fresh challenge/proxy contexts a pooled browser can create before TRAWL restarts the full browser process. Tier 3 and Tier 4 use short-lived isolated contexts so Cloudflare sees a clean profile, but some Camoufox/Firefox builds can leave content processes behind even after Playwright closes those contexts. Recycling the browser bounds that process growth without changing Redis session-cache TTLs.
+
+```ini
+BROWSER_RECYCLE_AFTER_CONTEXTS=8   # default - bound long-running browser process growth
+BROWSER_RECYCLE_AFTER_CONTEXTS=0   # disable browser recycling
+```
+
 ## Session Cache
 
 ### `SESSION_TTL_SECONDS`

packages/browser/src/pool.ts

@@ -18,43 +18,67 @@ export interface BrowserHandle {
   id: number
   context: BrowserContext
   browser: Browser
+  noteTemporaryContext?: (reason: string) => void
 }
 
 interface PoolEntry extends PoolBrowser {
   browser: Browser | null
   context: BrowserContext | null
+  temporaryContextUses: number
+  restartReason?: string
+  restarting?: boolean
 }
 
+type BrowserFactory = () => Promise<{ browser: Browser; context: BrowserContext }>
+
 export class BrowserPool {
   private entries: PoolEntry[] = []
   private poolSize: number
   private acquireTimeoutMs: number
   private pollIntervalMs: number
+  private recycleAfterTemporaryContexts: number
+  private browserFactory?: BrowserFactory
   private healthInterval: ReturnType<typeof setInterval> | null = null
 
   constructor({
     poolSize,
     acquireTimeoutMs = 15_000,
     pollIntervalMs = 100,
+    recycleAfterTemporaryContexts = 8,
+    browserFactory,
   }: {
     poolSize: number
     acquireTimeoutMs?: number
     pollIntervalMs?: number
+    recycleAfterTemporaryContexts?: number
+    browserFactory?: BrowserFactory
   }) {
     this.poolSize = poolSize
     this.acquireTimeoutMs = acquireTimeoutMs
     this.pollIntervalMs = pollIntervalMs
+    this.recycleAfterTemporaryContexts = recycleAfterTemporaryContexts
+    this.browserFactory = browserFactory
   }
 
   async init(): Promise<void> {
     for (let i = 0; i < this.poolSize; i++) {
       const { browser, context } = await this.launchBrowser()
-      this.entries.push({ id: i, busy: false, restartCount: 0, healthy: true, browser, context })
+      this.entries.push({
+        id: i,
+        busy: false,
+        restartCount: 0,
+        healthy: true,
+        browser,
+        context,
+        temporaryContextUses: 0,
+      })
       console.log(`[pool] browser ${i + 1}/${this.poolSize} ready`)
     }
   }
 
   private async launchBrowser(): Promise<{ browser: Browser; context: BrowserContext }> {
+    if (this.browserFactory) return this.browserFactory()
+
     // Camoufox patches fingerprint data at the C++/Juggler level — not via JS injection.
     // CF's JS cannot detect these patches the way it detects overrides of window.chrome,
     // plugins, WebGL etc. Same browser Byparr uses.
@@ -117,7 +141,14 @@ export class BrowserPool {
         entry.busy = true
         entry.lastDomain = domain
         entry.lastUsedAt = Date.now()
-        resolve({ id: entry.id, context: entry.context, browser: entry.browser })
+        resolve({
+          id: entry.id,
+          context: entry.context,
+          browser: entry.browser,
+          noteTemporaryContext: (reason: string) => {
+            this.noteTemporaryContext(entry, reason)
+          },
+        })
         return true
       }
 
@@ -138,7 +169,7 @@ export class BrowserPool {
   }
 
   private pickEntry(domain?: string): PoolEntry | null {
-    const available = this.entries.filter((e) => !e.busy && e.healthy && e.context)
+    const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context)
     if (available.length === 0) return null
     if (domain) {
       const sticky = available.find((e) => e.lastDomain === domain)
@@ -147,6 +178,15 @@ export class BrowserPool {
     return available[0]
   }
 
+  private noteTemporaryContext(entry: PoolEntry, reason: string): void {
+    if (this.recycleAfterTemporaryContexts <= 0) return
+
+    entry.temporaryContextUses++
+    if (entry.temporaryContextUses >= this.recycleAfterTemporaryContexts) {
+      entry.restartReason = `${reason}; ${entry.temporaryContextUses} temporary contexts used`
+    }
+  }
+
   release(id: number): void {
     const entry = this.entries.find((e) => e.id === id)
     if (!entry) return
@@ -157,6 +197,9 @@ export class BrowserPool {
       const pages: unknown[] = entry.context.pages() ?? []
       for (const p of pages) (p as { close: () => Promise<void> }).close().catch(() => {})
     }
+    if (entry.restartReason) {
+      void this.restartEntry(entry, entry.restartReason)
+    }
   }
 
   startHealthCheck(): void {
@@ -168,15 +211,22 @@ export class BrowserPool {
       if (entry.busy) continue
       if (!(entry.browser?.isConnected() ?? false)) {
         console.warn(`[pool] browser ${entry.id} disconnected, restarting`)
-        await this.restartEntry(entry)
+        await this.restartEntry(entry, "browser disconnected")
       } else {
         entry.healthy = true
       }
     }
   }
 
-  private async restartEntry(entry: PoolEntry): Promise<void> {
+  private async restartEntry(entry: PoolEntry, reason = "manual restart"): Promise<void> {
+    if (entry.restarting) {
+      entry.restartReason ??= reason
+      return
+    }
+    entry.restarting = true
     entry.healthy = false
+    entry.restartReason = undefined
+    console.warn(`[pool] browser ${entry.id} restarting: ${reason}`)
     try {
       await entry.context?.close()
     } catch {}
@@ -190,20 +240,24 @@ export class BrowserPool {
       entry.browser = browser
       entry.context = context
       entry.healthy = true
+      entry.temporaryContextUses = 0
       entry.restartCount++
       console.log(`[pool] browser ${entry.id} restarted (total: ${entry.restartCount})`)
     } catch (err) {
       console.error(`[pool] browser ${entry.id} failed to restart:`, err)
+    } finally {
+      entry.restarting = false
     }
   }
 
   getStats(): PoolStats {
     const busy = this.entries.filter((e) => e.busy).length
+    const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context).length
     const totalRestarts = this.entries.reduce((sum, e) => sum + e.restartCount, 0)
     return {
       total: this.poolSize,
       busy,
-      available: this.poolSize - busy,
+      available,
       restarts: totalRestarts,
       avgRestarts: totalRestarts / this.poolSize,
     }

packages/browser/tests/pool.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test"
+import { BrowserPool } from "../src/pool"
+
+const waitFor = async (predicate: () => boolean) => {
+  const deadline = Date.now() + 1000
+  while (Date.now() < deadline) {
+    if (predicate()) return
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+  throw new Error("timed out waiting for condition")
+}
+
+describe("BrowserPool recycling", () => {
+  test("restarts the browser after the temporary context threshold", async () => {
+    const browsers: Array<{ closed: boolean; isConnected: () => boolean; close: () => Promise<void> }> = []
+    const contexts: Array<{ closed: boolean; pages: () => unknown[]; close: () => Promise<void> }> = []
+
+    const pool = new BrowserPool({
+      poolSize: 1,
+      recycleAfterTemporaryContexts: 2,
+      browserFactory: async () => {
+        const browser = {
+          closed: false,
+          isConnected() {
+            return !this.closed
+          },
+          async close() {
+            this.closed = true
+          },
+        }
+        const context = {
+          closed: false,
+          pages: () => [],
+          async close() {
+            this.closed = true
+          },
+        }
+        browsers.push(browser)
+        contexts.push(context)
+        return { browser, context }
+      },
+    })
+
+    await pool.init()
+
+    const first = await pool.acquire("example.com")
+    first.noteTemporaryContext?.("tier3 fresh context")
+    pool.release(first.id)
+
+    expect(pool.getStats().restarts).toBe(0)
+    expect(pool.getStats().available).toBe(1)
+
+    const second = await pool.acquire("example.com")
+    second.noteTemporaryContext?.("tier3 fresh context")
+    pool.release(second.id)
+
+    await waitFor(() => pool.getStats().restarts === 1)
+
+    expect(contexts[0].closed).toBe(true)
+    expect(browsers[0].closed).toBe(true)
+    expect(pool.getStats().available).toBe(1)
+    expect(browsers).toHaveLength(2)
+  })
+})

packages/tiers/src/tier3.ts

@@ -35,6 +35,7 @@ export async function runTier3(
   // challenge evaluation. A fresh context with no prior state gets managed-mode treatment:
   // CF evaluates in under 1s and the challenge resolves in 3-4s total.
   const freshCtx = await newFreshContext(handle.browser, { proxy: proxyUrl })
+  handle.noteTemporaryContext?.("tier3 fresh context")
   const page = await freshCtx.newPage()
 
   try {

packages/tiers/src/tier4.ts

@@ -40,6 +40,7 @@ export async function runTier4(
       proxy: { server: proxyUrl },
       viewport: null,
     })
+    handle.noteTemporaryContext?.("tier4 proxy context")
     await proxyContext.addInitScript(() => {
       window.onerror = () => true
       window.addEventListener(