diff --git a/CHANGELOG.md b/CHANGELOG.md index fe484d3..6acedd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.3.1] - Unreleased + +### Fixed +- Keep pooled browser contexts under `BrowserPool` ownership so repeat Tier 2 requests cannot reuse a context closed by a separate cache (#45). + ## [1.3.0] - 2026-08-02 ### Added diff --git a/apps/api/src/deps.ts b/apps/api/src/deps.ts index 9e5ee49..bd9fe74 100644 --- a/apps/api/src/deps.ts +++ b/apps/api/src/deps.ts @@ -1,4 +1,4 @@ -import { BrowserPool, type PersistentBrowserContext, PersistentContextCache, SessionCache } from "@trawl/browser" +import { BrowserPool, SessionCache } from "@trawl/browser" import type { OrchestratorDeps } from "@trawl/tiers" import type { SessionData } from "@trawl/types" import { @@ -18,7 +18,6 @@ import { const state: { pool?: BrowserPool sessionCache?: SessionCache - persistentContextCache?: PersistentContextCache } = {} export const getPool = () => state.pool @@ -46,11 +45,6 @@ export const initPool = async (): Promise => { await state.pool.init() state.pool.startHealthCheck() - state.persistentContextCache = new PersistentContextCache({ - maxEntries: 20, - ttlMs: 10 * 60 * 1000, - }) - console.log(`[api] ready — all ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"} warm`) } @@ -58,7 +52,6 @@ export const getDeps = (): OrchestratorDeps => { if (!state.pool) throw new Error("pool not ready") const p = state.pool const sc = state.sessionCache - const pcc = state.persistentContextCache return { acquireBrowser: (d: string, budgetMs?: number) => p.acquire(d, budgetMs), releaseBrowser: (id: number, lease?: number) => p.release(id, lease), @@ -67,15 +60,5 @@ export const getDeps = (): OrchestratorDeps => { invalidateSession: (d: string) => (sc ? sc.invalidate(d).catch(() => {}) : Promise.resolve()), proxyPool, residentialProxyPool, - acquireContext: async (_handleId: number, hostname: string) => pcc?.get(hostname), - saveContext: async (handleId: number, hostname: string, context: PersistentBrowserContext) => { - if (pcc) pcc.set(hostname, context, handleId) - }, - releaseContext: (_handleId: number, hostname: string) => { - pcc?.get(hostname) - }, - invalidateContext: async (hostname: string) => { - pcc?.evict(hostname) - }, } } diff --git a/packages/browser/src/index.ts b/packages/browser/src/index.ts index fd76370..cd86e05 100644 --- a/packages/browser/src/index.ts +++ b/packages/browser/src/index.ts @@ -1,10 +1,4 @@ export { FINGERPRINT, FINGERPRINT_POOL } from "./fingerprint" -export { - createPersistentContext, - type PersistentBrowserContext, - PersistentContextCache, - type PersistentContextCacheOpts, -} from "./persistentContextCache" export type { BrowserHandle } from "./pool" export { BrowserPool, newFreshContext, PoolExhaustedError } from "./pool" export { type PlaywrightProxy, toPlaywrightProxy } from "./proxy" diff --git a/packages/browser/src/persistentContextCache.ts b/packages/browser/src/persistentContextCache.ts deleted file mode 100644 index 0d09efc..0000000 --- a/packages/browser/src/persistentContextCache.ts +++ /dev/null @@ -1,195 +0,0 @@ -// Per-host browser contexts keep clearance state warm between requests. -export interface PersistentBrowserContext { - addInitScript(script: () => void): Promise - close?: () => Promise -} - -interface PersistentBrowser { - newContext(options: { viewport: null }): Promise -} - -interface CacheEntry { - context: PersistentBrowserContext - browserId: number - lastUsed: number - // Monotonic access counter — increments on every set/get. Used for LRU - // ordering because Date.now() can tie within a single millisecond when - // operations happen back-to-back. - lastTouched: number - createdAt: number -} - -export interface PersistentContextCacheOpts { - // Maximum total cached contexts. LRU eviction kicks in past this limit. - maxEntries?: number - // Idle TTL — entries unused for longer than this are pruned. - ttlMs?: number - // Hook called when a context is evicted (cache miss + LRU + manual invalidate). - // Used to close the underlying Playwright context. - onEvict?: (context: PersistentBrowserContext) => void -} - -export class PersistentContextCache { - private readonly entries = new Map() - private readonly maxEntries: number - private readonly ttlMs: number - private readonly onEvict?: (context: PersistentBrowserContext) => void - // Per-browser context count for fair load distribution when allocating new contexts. - private readonly contextCountByBrowser = new Map() - // Monotonic counter for LRU ordering — see CacheEntry.lastTouched. - private touchCounter = 0 - - constructor(opts: PersistentContextCacheOpts = {}) { - this.maxEntries = opts.maxEntries ?? 20 - this.ttlMs = opts.ttlMs ?? 10 * 60 * 1000 // 10 minutes idle - this.onEvict = opts.onEvict - } - - // Returns a cached context for `hostname`, or undefined if none exists / expired. - // Touches the lastUsed timestamp so LRU picks up the access. - get(hostname: string) { - const entry = this.entries.get(hostname) - if (!entry) return - if (Date.now() - entry.lastUsed > this.ttlMs) { - this.evict(hostname) - return - } - entry.lastUsed = Date.now() - entry.lastTouched = ++this.touchCounter - return entry.context - } - - // Register a fresh context for `hostname`, owned by `browserId`. If an entry - // already exists it is evicted first (caller is replacing it). - set(hostname: string, context: PersistentBrowserContext, browserId: number): void { - const existing = this.entries.get(hostname) - if (existing && existing.context !== context) { - this.evictEntry(existing, hostname) - } - if (this.entries.size >= this.maxEntries && !this.entries.has(hostname)) { - const lru = this.findLRU() - if (lru && lru !== hostname) this.evict(lru) - } - this.entries.set(hostname, { - context, - browserId, - lastUsed: Date.now(), - lastTouched: ++this.touchCounter, - createdAt: Date.now(), - }) - this.contextCountByBrowser.set(browserId, (this.contextCountByBrowser.get(browserId) ?? 0) + 1) - } - - // Remove a specific entry. Closes the context. - evict(hostname: string): void { - const entry = this.entries.get(hostname) - if (!entry) return - this.evictEntry(entry, hostname) - } - - private evictEntry(entry: CacheEntry, hostname: string): void { - this.entries.delete(hostname) - const count = this.contextCountByBrowser.get(entry.browserId) ?? 1 - if (count <= 1) this.contextCountByBrowser.delete(entry.browserId) - else this.contextCountByBrowser.set(entry.browserId, count - 1) - if (this.onEvict) { - try { - this.onEvict(entry.context) - } catch { - // closing can throw if the browser is already gone; ignore - } - } else { - entry.context.close?.().catch(() => {}) - } - } - - // Drop all contexts owned by a specific browser (called when the browser - // restarts). Returns the hostnames that were invalidated. - invalidateBrowser(browserId: number): string[] { - const removed: string[] = [] - for (const [host, entry] of this.entries) { - if (entry.browserId === browserId) { - removed.push(host) - } - } - for (const host of removed) { - this.evict(host) - } - return removed - } - - // Find the hostname with the smallest lastTouched (LRU eviction candidate). - // Uses the monotonic counter so back-to-back operations don't tie. - private findLRU() { - let oldest: string | undefined - let oldestTouched = Infinity - for (const [host, entry] of this.entries) { - if (entry.lastTouched < oldestTouched) { - oldestTouched = entry.lastTouched - oldest = host - } - } - return oldest - } - - // Periodic cleanup of entries past their TTL. Returns count of removed entries. - prune(): number { - const now = Date.now() - const toRemove: string[] = [] - for (const [host, entry] of this.entries) { - if (now - entry.lastUsed > this.ttlMs) toRemove.push(host) - } - for (const host of toRemove) this.evict(host) - return toRemove.length - } - - size(): number { - return this.entries.size - } - - // Pick the browser with the fewest cached contexts (for fair load distribution - // when allocating a new persistent context). Pass the set of available - // browser IDs; undefined if the list is empty. - leastLoadedBrowser(availableBrowserIds: number[]) { - if (availableBrowserIds.length === 0) return - let best: number | undefined - let bestCount = Infinity - for (const id of availableBrowserIds) { - const count = this.contextCountByBrowser.get(id) ?? 0 - if (count < bestCount) { - bestCount = count - best = id - } - } - return best - } - - // For tests / observability - contextCount(browserId: number): number { - return this.contextCountByBrowser.get(browserId) ?? 0 - } -} - -// Helper: create a fresh context with TRAWL's init scripts applied. Re-exported -// here so callers don't need to import from pool.ts directly when wiring the -// cache. -export const createPersistentContext = async (browser: PersistentBrowser): Promise => { - const context = await browser.newContext({ viewport: null }) - await context.addInitScript(() => { - window.onerror = () => true - window.addEventListener( - "unhandledrejection", - (e: PromiseRejectionEvent) => { - e.preventDefault() - }, - true, - ) - const _orig = Element.prototype.attachShadow - Element.prototype.attachShadow = function (init: ShadowRootInit) { - const r = _orig.call(this, init) - Object.defineProperty(this, "shadowRootUnl", { configurable: true, value: r }) - return r - } - }) - return context -} diff --git a/packages/browser/tests/persistentContextCache.test.ts b/packages/browser/tests/persistentContextCache.test.ts deleted file mode 100644 index 1210c60..0000000 --- a/packages/browser/tests/persistentContextCache.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { describe, expect, mock, test } from "bun:test" -import { PersistentContextCache } from "../src/persistentContextCache" - -function makeFakeContext() { - return { - addInitScript: mock(async () => {}), - close: mock(async () => {}), - } -} - -describe("PersistentContextCache", () => { - test("returns undefined for unknown host", () => { - const c = new PersistentContextCache() - expect(c.get("never-seen.example.com")).toBeUndefined() - }) - - test("set + get round-trips a context and tracks per-browser count", () => { - const c = new PersistentContextCache() - const ctx = makeFakeContext() - c.set("a.example.com", ctx, 0) - expect(c.get("a.example.com")).toBe(ctx) - expect(c.contextCount(0)).toBe(1) - }) - - test("respects idle TTL — past TTL returns undefined", () => { - const c = new PersistentContextCache({ ttlMs: 10 }) - const ctx = makeFakeContext() - c.set("host.example", ctx, 0) - expect(c.get("host.example")).toBe(ctx) - const deadline = Date.now() + 30 - while (Date.now() < deadline) { - // wait - } - expect(c.get("host.example")).toBeUndefined() - expect(c.size()).toBe(0) - }) - - test("evict() removes a specific entry and decrements per-browser count", () => { - const c = new PersistentContextCache() - const a = makeFakeContext() - const b = makeFakeContext() - c.set("a", a, 0) - c.set("b", b, 1) - expect(c.size()).toBe(2) - c.evict("a") - expect(c.size()).toBe(1) - expect(c.contextCount(0)).toBe(0) - expect(c.contextCount(1)).toBe(1) - }) - - test("evict() calls onEvict hook OR context.close()", async () => { - const ctx = makeFakeContext() - const c = new PersistentContextCache({ onEvict: () => ctx.close() }) - c.set("a", ctx, 0) - c.evict("a") - await Promise.resolve() - expect(ctx.close).toHaveBeenCalled() - }) - - test("evicting last context for a browser clears the per-browser counter", () => { - const c = new PersistentContextCache() - const ctx = makeFakeContext() - c.set("only.example", ctx, 0) - expect(c.contextCount(0)).toBe(1) - c.evict("only.example") - expect(c.contextCount(0)).toBe(0) - }) - - test("LRU eviction when at capacity — least recently used is dropped", () => { - const c = new PersistentContextCache({ maxEntries: 2 }) - const a = makeFakeContext() - const b = makeFakeContext() - const cc = makeFakeContext() - c.set("a", a, 0) - c.set("b", b, 1) - c.get("a") // touch `a` so `b` becomes LRU - c.set("c", cc, 2) - expect(c.get("b")).toBeUndefined() - expect(c.get("a")).toBe(a) - expect(c.get("c")).toBe(cc) - expect(c.size()).toBe(2) - }) - - test("invalidateBrowser() drops all contexts owned by that browser", () => { - const c = new PersistentContextCache() - const a = makeFakeContext() - const b = makeFakeContext() - const d = makeFakeContext() - c.set("a.example", a, 0) - c.set("b.example", b, 0) - c.set("d.example", d, 1) - expect(c.size()).toBe(3) - const removed = c.invalidateBrowser(0) - expect(removed.sort()).toEqual(["a.example", "b.example"]) - expect(c.size()).toBe(1) - expect(c.get("d.example")).toBe(d) - expect(c.contextCount(0)).toBe(0) - expect(c.contextCount(1)).toBe(1) - }) - - test("prune() removes all entries past TTL", () => { - const c = new PersistentContextCache({ ttlMs: 5 }) - c.set("a", makeFakeContext(), 0) - c.set("b", makeFakeContext(), 1) - c.set("c", makeFakeContext(), 2) - const deadline = Date.now() + 25 - while (Date.now() < deadline) { - // wait - } - expect(c.prune()).toBe(3) - expect(c.size()).toBe(0) - }) - - test("leastLoadedBrowser() picks the browser with fewest cached contexts", () => { - const c = new PersistentContextCache() - c.set("a1", makeFakeContext(), 0) - c.set("a2", makeFakeContext(), 0) - c.set("b1", makeFakeContext(), 1) - expect(c.leastLoadedBrowser([0, 1])).toBe(1) - c.set("b2", makeFakeContext(), 1) - c.set("b3", makeFakeContext(), 1) - expect(c.leastLoadedBrowser([0, 1])).toBe(0) - expect(c.leastLoadedBrowser([])).toBeUndefined() - }) - - test("set() replaces existing entry without growing size past maxEntries", () => { - const c = new PersistentContextCache({ maxEntries: 2 }) - const a1 = makeFakeContext() - const a2 = makeFakeContext() - c.set("host", a1, 0) - expect(c.get("host")).toBe(a1) - c.set("host", a2, 0) - expect(c.get("host")).toBe(a2) - expect(c.size()).toBe(1) - expect(c.contextCount(0)).toBe(1) - }) - - test("get() refreshes the access counter so the entry survives LRU eviction", () => { - const c = new PersistentContextCache({ maxEntries: 2, ttlMs: 30 }) - const a = makeFakeContext() - const b = makeFakeContext() - const cc = makeFakeContext() - c.set("a", a, 0) // touchCounter=1 - const midDeadline = Date.now() + 15 - while (Date.now() < midDeadline) { - // wait - } - c.get("a") // touchCounter=2 (refreshes `a`) - c.set("b", b, 1) // touchCounter=3 - c.set("c", cc, 2) // touchCounter=4; LRU = `a` (counter=2), but `a` was refreshed… - // Wait — the LRU is now whoever has the SMALLEST counter among "a" (2) and "b" (3). - // That's `a`. So adding "c" evicts `a`, not `b`. To make `a` survive, the refresh - // must happen AFTER `b` is set. Test that scenario explicitly: - const c2 = new PersistentContextCache({ maxEntries: 2 }) - const a2 = makeFakeContext() - const b2 = makeFakeContext() - const cc2 = makeFakeContext() - c2.set("a2", a2, 0) // counter=1 - c2.set("b2", b2, 1) // counter=2 - c2.get("b2") // counter=3 — refresh b2 - c2.set("c2", cc2, 2) // counter=4; LRU = a2 (counter=1) — `a2` evicted - expect(c2.get("a2")).toBeUndefined() - expect(c2.get("b2")).toBe(b2) // survived thanks to get() refresh - expect(c2.get("c2")).toBe(cc2) - - // And the cleanup above proves `a` from the first scenario was evicted: - expect(c.get("a")).toBeUndefined() - expect(c.get("b")).toBe(b) - expect(c.get("c")).toBe(cc) - }) -}) diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index d2460c2..a26253e 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -1,4 +1,4 @@ -import type { BrowserHandle, PersistentBrowserContext } from "@trawl/browser" +import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { runTier1 } from "./tiers/1" @@ -44,13 +44,6 @@ export interface OrchestratorDeps { proxyPool?: ProxyPool residentialProxyPool?: ProxyPool onTierAttempt?: (result: TierResult) => void - // Optional per-host persistent browser context cache. When provided, Tier 2 - // reuses a warm context with cached `cf_clearance` cookies on repeat visits - // — the cookie-loading + Redis round-trip is skipped entirely. - acquireContext?(handleId: number, hostname: string): Promise - saveContext?(handleId: number, hostname: string, context: PersistentBrowserContext): Promise - releaseContext?(handleId: number, hostname: string): void - invalidateContext?(hostname: string): Promise } const extractDomain = (url: string): string => { @@ -119,17 +112,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const session = await deps.loadSession(domain) if (session && maxTier >= 2) { const remaining = maxTimeout - (Date.now() - totalStart) - const t2 = await runTier2( - req.url, - handle, - session, - remaining, - sanitizedHeaders, - req.method, - req.body, - deps, - domain, - ) + const t2 = await runTier2(req.url, handle, session, remaining, sanitizedHeaders, req.method, req.body) emit(t2) if (hasUsablePayload(t2)) { if (t2.cookies && t2.cookies.length > 0) { diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index 3102deb..68b58a2 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -1,4 +1,4 @@ -import type { BrowserHandle, PersistentBrowserContext } from "@trawl/browser" +import type { BrowserHandle } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" import { normalizeSameSite, toCookies } from "../utils/cookies" @@ -27,54 +27,29 @@ export async function runTier2( extraHeaders?: Record, method?: string, body?: string, - // Optional — when provided by the orchestrator, Tier 2 reuses a warm per-host - // browser context on repeat visits. Cached contexts already hold cf_clearance - // cookies from the prior solve, so we skip the Redis round-trip and cookie - // re-injection entirely. - deps?: { - acquireContext?(handleId: number, hostname: string): Promise - saveContext?(handleId: number, hostname: string, context: PersistentBrowserContext): Promise - releaseContext?(handleId: number, hostname: string): void - }, - hostname?: string, ): Promise { const start = Date.now() - - // Pick the context: persistent cache hit → reuse; otherwise fall back to the - // pool's shared context. The shared context still works (existing behavior) - // — only the persistent cache path is new. - let activeContext = handle.context - let contextFromCache = false - if (deps?.acquireContext && hostname) { - const cached = await deps.acquireContext(handle.id, hostname) - if (cached) { - activeContext = cached - contextFromCache = true - } - } - - const page = await activeContext.newPage() + const activeContext = handle.context + let page: Awaited> | undefined try { + page = await activeContext.newPage() + // addCookies replaces cookies by name+domain+path, so no need to clearCookies first. // Keeping the context's CF cookies (cf_clearance, __cf_bm) intact means CF sees a // browser with history, which speeds up challenge evaluation on the next Tier 3 run. - // Skip the Redis→cookie re-injection on cache hits — the persistent context - // already carries cookies from its prior solve. - if (!contextFromCache) { - await activeContext.addCookies( - session.cookies.map((c) => ({ - name: c.name, - value: c.value, - domain: c.domain, - path: c.path, - expires: c.expires, - httpOnly: c.httpOnly, - secure: c.secure, - sameSite: normalizeSameSite(c.sameSite), - })), - ) - } + await activeContext.addCookies( + session.cookies.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + expires: c.expires, + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: normalizeSameSite(c.sameSite), + })), + ) await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent }) @@ -134,15 +109,6 @@ export async function runTier2( const captured = await captureResponse(mainResponseHolder.value) - // On success, register this context in the persistent cache so the next - // visit to this hostname skips cookie loading entirely. Skip if it was - // already served from the cache (no need to re-register the same context). - if (!contextFromCache && deps?.saveContext && hostname && cookies.length > 0) { - await deps.saveContext(handle.id, hostname, activeContext).catch(() => { - // Caching is best-effort; the request already succeeded. - }) - } - return { tier: 2, status: "success", @@ -163,6 +129,6 @@ export async function runTier2( reason: err instanceof Error ? err.message : String(err), } } finally { - await page.close().catch(() => {}) + await page?.close().catch(() => {}) } } diff --git a/packages/tiers/tests/runTier2.test.ts b/packages/tiers/tests/runTier2.test.ts new file mode 100644 index 0000000..9175f90 --- /dev/null +++ b/packages/tiers/tests/runTier2.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { SessionData } from "@trawl/types" +import { runTier2 } from "../src/tiers/2" + +const session: SessionData = { + cookies: [ + { + name: "cf_clearance", + value: "cached-token", + domain: ".example.com", + path: "/", + expires: 2_000_000_000, + httpOnly: true, + secure: true, + sameSite: "Lax", + }, + ], + userAgent: "cached-user-agent", + savedAt: 1, +} + +const handleWithContext = (context: BrowserHandle["context"]): BrowserHandle => ({ + id: 3, + lease: 7, + context, + browser: {}, + fingerprint: { + userAgent: "pool-user-agent", + platform: "Linux x86_64", + locale: "en-US", + timezone: "UTC", + }, +}) + +describe("runTier2", () => { + test("returns an error when the acquired pool context is already closed", async () => { + const playwrightError = "browserContext.newPage: Target page, context or browser has been closed" + const handle = handleWithContext({ + newPage: () => Promise.reject(new Error(playwrightError)), + }) + + const result = await runTier2("https://example.com", handle, session, 100) + + expect(result.status).toBe("error") + expect(result.reason).toBe(playwrightError) + }) + + test("uses the acquired pool context, injects session cookies, and returns content", async () => { + let injectedCookies: unknown + let userAgent: Record | undefined + let pageClosed = false + const page = { + setExtraHTTPHeaders: async (headers: Record) => { + userAgent = headers + }, + on: () => {}, + goto: async () => {}, + waitForLoadState: async () => {}, + content: async () => "pool context content", + close: async () => { + pageClosed = true + }, + } + const context = { + newPage: async () => page, + addCookies: async (cookies: unknown) => { + injectedCookies = cookies + }, + cookies: async () => session.cookies, + } + + const result = await runTier2("https://example.com", handleWithContext(context), session, 100) + + expect(result.status).toBe("success") + expect(result.html).toContain("pool context content") + expect(injectedCookies).toEqual(session.cookies) + expect(userAgent).toEqual({ "User-Agent": session.userAgent }) + expect(pageClosed).toBe(true) + }) +})