diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 7cd1ff0..937cdba 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -13,6 +13,14 @@ import { runTier4 } from "./tier4" // keeps a long proxy list from blowing the request's maxTimeout budget. const MAX_PROXY_ATTEMPTS = 2 +// True when a Tier 3/4 result indicates the browser's profile was actively rejected +// by the upstream (CF / Imperva / etc.). On these outcomes the orchestrator flags the +// pool for a future recycle; on every other outcome (success, transient error, timeout) +// the browser is kept warm so cookies + cf_clearance survive. +export function shouldFlagForRecycle(status: TierResult["status"]): boolean { + return status === "blocked" || status === "needs-js" +} + export interface OrchestratorDeps { acquireBrowser(domain: string): Promise releaseBrowser(id: number): void @@ -120,6 +128,14 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const remaining3 = maxTimeout - (Date.now() - totalStart) t3 = await runTier3(req.url, handle, remaining3, proxy3, sanitizedHeaders, req.method, req.body) + // Only flag the pool for a recycle when the upstream actively rejected the + // browser's profile ("blocked"/"needs-js"). Successful solves preserve cookies, + // cf_clearance, and TLS fingerprint — recycling after success would force a + // costly cold start on the next request to the same domain. + if (shouldFlagForRecycle(t3.status)) { + handle.noteTemporaryContext?.(`tier3 ${t3.status}`) + } + const pool = deps.proxyPool if (t3.status !== "blocked" || req.proxy || !proxy3 || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break pool.markBad(proxy3) @@ -173,6 +189,12 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const remaining4 = maxTimeout - (Date.now() - totalStart) t4 = await runTier4(req.url, handle, remaining4, proxy4, sanitizedHeaders, req.method, req.body) + // Mirror Tier 3's recycle-on-suspect policy — only flag when the upstream + // explicitly rejected the browser's profile. + if (shouldFlagForRecycle(t4.status)) { + handle.noteTemporaryContext?.(`tier4 ${t4.status}`) + } + const pool = deps.residentialProxyPool if (t4.status !== "blocked" || req.proxy || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break pool.markBad(proxy4) diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index cedcf0f..8a4a557 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -35,7 +35,6 @@ export async function runTier3( // challenge evaluation. A fresh context with no prior state gets managed-mode treatment: // CF evaluates in under 1s and the challenge resolves in 3-4s total. const freshCtx = await newFreshContext(handle.browser, { proxy: proxyUrl }) - handle.noteTemporaryContext?.("tier3 fresh context") const page = await freshCtx.newPage() try { @@ -180,6 +179,9 @@ export async function runTier3( } } finally { await page.close().catch(() => {}) - await freshCtx.close().catch(() => {}) + // Bound context.close() with a timeout — Camoufox/Firefox occasionally hangs on + // close when a content process is wedged, leaking the process until the next + // browser recycle. 5s is well above the typical <500ms close path. + await Promise.race([freshCtx.close(), new Promise((resolve) => setTimeout(resolve, 5000))]).catch(() => {}) } } diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index 6b80d55..fc6db08 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -40,7 +40,6 @@ export async function runTier4( proxy: { server: proxyUrl }, viewport: null, }) - handle.noteTemporaryContext?.("tier4 proxy context") await proxyContext.addInitScript(() => { window.onerror = () => true window.addEventListener( @@ -182,6 +181,11 @@ export async function runTier4( reason: err instanceof Error ? err.message : String(err), } } finally { - await proxyContext?.close().catch(() => {}) + // Same timeout-bounded close as tier3 — see comment there. + if (proxyContext) { + await Promise.race([proxyContext.close(), new Promise((resolve) => setTimeout(resolve, 5000))]).catch( + () => {}, + ) + } } } diff --git a/packages/tiers/tests/orchestrator.test.ts b/packages/tiers/tests/orchestrator.test.ts new file mode 100644 index 0000000..bde54c7 --- /dev/null +++ b/packages/tiers/tests/orchestrator.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test" +import { shouldFlagForRecycle } from "../src/orchestrator" + +// Covers the recycle-on-suspect policy: only `blocked` and `needs-js` outcomes +// should flag the pool for a browser recycle. Successful solves, transient +// errors, and timeouts must NOT trigger a recycle — otherwise warm cookies +// + cf_clearance get thrown away on every request. +describe("shouldFlagForRecycle", () => { + test("flags blocked outcomes", () => { + expect(shouldFlagForRecycle("blocked")).toBe(true) + }) + + test("flags needs-js outcomes", () => { + expect(shouldFlagForRecycle("needs-js")).toBe(true) + }) + + test("does NOT flag success outcomes", () => { + expect(shouldFlagForRecycle("success")).toBe(false) + }) + + test("does NOT flag transient errors", () => { + expect(shouldFlagForRecycle("error")).toBe(false) + }) + + test("does NOT flag timeouts", () => { + expect(shouldFlagForRecycle("timeout")).toBe(false) + }) + + test("does NOT flag skipped outcomes", () => { + expect(shouldFlagForRecycle("skipped")).toBe(false) + }) +})