mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
refactor(browser): simplify optional pool state
This commit is contained in:
@@ -23,8 +23,8 @@ export class PoolExhaustedError extends Error {
|
||||
export type { BrowserHandle } from "@trawl/types"
|
||||
|
||||
interface PoolEntry extends PoolBrowser {
|
||||
browser: Browser | null
|
||||
context: BrowserContext | null
|
||||
browser?: Browser
|
||||
context?: BrowserContext
|
||||
temporaryContextUses: number
|
||||
restartReason?: string
|
||||
restarting?: boolean
|
||||
@@ -41,7 +41,7 @@ export class BrowserPool {
|
||||
private recycleAfterTemporaryContexts: number
|
||||
private contentProcesses!: number
|
||||
private browserFactory?: BrowserFactory
|
||||
private healthInterval: ReturnType<typeof setInterval> | null = null
|
||||
private healthInterval?: ReturnType<typeof setInterval>
|
||||
|
||||
constructor({
|
||||
poolSize,
|
||||
@@ -207,8 +207,7 @@ export class BrowserPool {
|
||||
const _attachShadow = Element.prototype.attachShadow
|
||||
Element.prototype.attachShadow = function (init: ShadowRootInit) {
|
||||
const shadowRoot = _attachShadow.call(this, init)
|
||||
// biome-ignore lint/suspicious/noExplicitAny: extending DOM element with custom property
|
||||
;(this as any).shadowRootUnl = shadowRoot
|
||||
Object.defineProperty(this, "shadowRootUnl", { configurable: true, value: shadowRoot })
|
||||
return shadowRoot
|
||||
}
|
||||
})
|
||||
@@ -253,9 +252,9 @@ export class BrowserPool {
|
||||
})
|
||||
}
|
||||
|
||||
private pickEntry(domain?: string): PoolEntry | null {
|
||||
private pickEntry(domain?: string): PoolEntry | undefined {
|
||||
const available = this.entries.filter((e) => !e.busy && !e.restarting && e.healthy && e.context)
|
||||
if (available.length === 0) return null
|
||||
if (available.length === 0) return
|
||||
if (domain) {
|
||||
const sticky = available.find((e) => e.lastDomain === domain)
|
||||
if (sticky) return sticky
|
||||
@@ -321,8 +320,8 @@ export class BrowserPool {
|
||||
try {
|
||||
await entry.browser?.close()
|
||||
} catch {}
|
||||
entry.browser = null
|
||||
entry.context = null
|
||||
delete entry.browser
|
||||
delete entry.context
|
||||
try {
|
||||
// On restart, keep the entry's original fingerprint so this browser instance
|
||||
// keeps its identity across restart cycles (otherwise cross-session correlation
|
||||
@@ -355,7 +354,10 @@ export class BrowserPool {
|
||||
}
|
||||
|
||||
async shutdown(): Promise<void> {
|
||||
if (this.healthInterval) clearInterval(this.healthInterval)
|
||||
if (this.healthInterval) {
|
||||
clearInterval(this.healthInterval)
|
||||
delete this.healthInterval
|
||||
}
|
||||
for (const entry of this.entries) {
|
||||
await entry.context?.close().catch(() => {})
|
||||
await entry.browser?.close().catch(() => {})
|
||||
@@ -364,10 +366,7 @@ export class BrowserPool {
|
||||
}
|
||||
}
|
||||
|
||||
// Creates a fresh context from any browser with TRAWL init scripts applied.
|
||||
// A fresh context (no prior cookies/localStorage/service workers) gets CF managed-mode
|
||||
// treatment — challenge resolves in 3-4s vs ~40s for warm/reused contexts.
|
||||
// biome-ignore lint/suspicious/noExplicitAny: camoufox-js doesn't export BrowserContext type
|
||||
// biome-ignore lint/suspicious/noExplicitAny: preserves the caller's Playwright or Patchright context type
|
||||
export const newFreshContext = async (browser: any, options?: { proxy?: string }): Promise<any> => {
|
||||
const context = await browser.newContext({
|
||||
viewport: null,
|
||||
@@ -385,8 +384,7 @@ export const newFreshContext = async (browser: any, options?: { proxy?: string }
|
||||
const _orig = Element.prototype.attachShadow
|
||||
Element.prototype.attachShadow = function (init: ShadowRootInit) {
|
||||
const r = _orig.call(this, init)
|
||||
// biome-ignore lint/suspicious/noExplicitAny: extending DOM element with custom property
|
||||
;(this as any).shadowRootUnl = r
|
||||
Object.defineProperty(this, "shadowRootUnl", { configurable: true, value: r })
|
||||
return r
|
||||
}
|
||||
})
|
||||
|
||||
@@ -18,13 +18,13 @@ export class SessionCache {
|
||||
await this.redis.set(this.key(domain), JSON.stringify(data), "EX", this.ttl)
|
||||
}
|
||||
|
||||
async load(domain: string): Promise<SessionData | null> {
|
||||
async load(domain: string) {
|
||||
const raw = await this.redis.get(this.key(domain))
|
||||
if (!raw) return null
|
||||
if (!raw) return
|
||||
try {
|
||||
return JSON.parse(raw) as SessionData
|
||||
} catch {
|
||||
return null
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ process.on("uncaughtException", (err) => {
|
||||
})
|
||||
|
||||
async function main() {
|
||||
const browser: any = await Camoufox({
|
||||
const browser = await Camoufox({
|
||||
headless: true,
|
||||
geoip: true,
|
||||
humanize: true,
|
||||
@@ -59,7 +59,7 @@ async function main() {
|
||||
const cookies = await ctx.cookies()
|
||||
console.log(
|
||||
"[3] cf_clearance:",
|
||||
cookies.some((c: any) => c.name === "cf_clearance"),
|
||||
cookies.some((c) => c.name === "cf_clearance"),
|
||||
)
|
||||
|
||||
await browser.close()
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Camoufox } from "camoufox-js"
|
||||
|
||||
async function main() {
|
||||
console.log("[test] launching Camoufox...")
|
||||
const browser: any = await Camoufox({
|
||||
const browser = await Camoufox({
|
||||
headless: true,
|
||||
geoip: true,
|
||||
humanize: true,
|
||||
@@ -22,8 +22,8 @@ async function main() {
|
||||
const title = await page.title()
|
||||
console.log("[test] UA:", ua)
|
||||
console.log("[test] Title:", title)
|
||||
console.log("[test] navigator.webdriver:", await page.evaluate(() => (navigator as any).webdriver))
|
||||
console.log("[test] window.chrome exists:", await page.evaluate(() => !!(window as any).chrome))
|
||||
console.log("[test] navigator.webdriver:", await page.evaluate(() => navigator.webdriver))
|
||||
console.log("[test] window.chrome exists:", await page.evaluate(() => "chrome" in window))
|
||||
|
||||
await browser.close()
|
||||
console.log("[test] PASS")
|
||||
|
||||
@@ -17,7 +17,7 @@ async function main() {
|
||||
i_know_what_im_doing: true,
|
||||
})
|
||||
|
||||
const browser = await (firefox as any).launch(opts)
|
||||
const browser = await firefox.launch(opts)
|
||||
const ctx = await browser.newContext({ viewport: null })
|
||||
const page = await ctx.newPage()
|
||||
|
||||
@@ -27,8 +27,8 @@ async function main() {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
})
|
||||
.catch((e: any) => {
|
||||
console.log("[test] goto error:", e.message.slice(0, 80))
|
||||
.catch((error: unknown) => {
|
||||
console.log("[test] goto error:", error instanceof Error ? error.message.slice(0, 80) : String(error))
|
||||
})
|
||||
|
||||
await new Promise((r) => setTimeout(r, 8000))
|
||||
@@ -48,7 +48,7 @@ async function main() {
|
||||
}
|
||||
|
||||
const cookies = await ctx.cookies()
|
||||
console.log("[test] cookies:", cookies.map((c: any) => c.name).join(", "))
|
||||
console.log("[test] cookies:", cookies.map((c) => c.name).join(", "))
|
||||
|
||||
await browser.close()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { afterEach, describe, expect, test } from "bun:test"
|
||||
import { BrowserPool } from "../src/pool"
|
||||
|
||||
const pools: BrowserPool[] = []
|
||||
|
||||
const createPool = (opts: ConstructorParameters<typeof BrowserPool>[0]): BrowserPool => {
|
||||
const pool = new BrowserPool(opts)
|
||||
pools.push(pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(pools.splice(0).map((pool) => pool.shutdown()))
|
||||
})
|
||||
|
||||
const waitFor = async (predicate: () => boolean) => {
|
||||
const deadline = Date.now() + 1000
|
||||
while (Date.now() < deadline) {
|
||||
@@ -52,7 +64,7 @@ describe("BrowserPool recycling", () => {
|
||||
test("restarts the browser after the temporary context threshold", async () => {
|
||||
const { factory, browsers, contexts } = makeFactory()
|
||||
|
||||
const pool = new BrowserPool({
|
||||
const pool = createPool({
|
||||
poolSize: 1,
|
||||
recycleAfterTemporaryContexts: 2,
|
||||
browserFactory: factory,
|
||||
@@ -82,7 +94,7 @@ describe("BrowserPool recycling", () => {
|
||||
test("noteTemporaryContext is no-op when recycleAfterTemporaryContexts=0", async () => {
|
||||
const { factory, browsers } = makeFactory()
|
||||
|
||||
const pool = new BrowserPool({
|
||||
const pool = createPool({
|
||||
poolSize: 1,
|
||||
recycleAfterTemporaryContexts: 0, // disabled
|
||||
browserFactory: factory,
|
||||
@@ -109,7 +121,7 @@ describe("BrowserPool recycling", () => {
|
||||
// that the pool, given N successful acquires, never recycles on its own.
|
||||
const { factory, browsers } = makeFactory()
|
||||
|
||||
const pool = new BrowserPool({
|
||||
const pool = createPool({
|
||||
poolSize: 1,
|
||||
recycleAfterTemporaryContexts: 2,
|
||||
browserFactory: factory,
|
||||
@@ -134,7 +146,7 @@ describe("BrowserPool recycling", () => {
|
||||
// option round-trips through the constructor without error.
|
||||
const { factory } = makeFactory()
|
||||
|
||||
const pool = new BrowserPool({
|
||||
const pool = createPool({
|
||||
poolSize: 1,
|
||||
contentProcesses: 4,
|
||||
browserFactory: factory,
|
||||
|
||||
Reference in New Issue
Block a user