diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 1323501..da9d497 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -2,19 +2,24 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { normalizeHtml } from "./html" +import type { ProxyPool } from "./proxyRotator" import { runTier1 } from "./tier1" import { runTier2 } from "./tier2" import { runTier3 } from "./tier3" import { runTier4 } from "./tier4" +// Bounds how many distinct proxies a single request will try per tier before giving up — +// keeps a long proxy list from blowing the request's maxTimeout budget. +const MAX_PROXY_ATTEMPTS = 2 + export interface OrchestratorDeps { acquireBrowser(domain: string): Promise releaseBrowser(id: number): void loadSession(domain: string): Promise saveSession(domain: string, data: SessionData): Promise invalidateSession(domain: string): Promise - proxyUrl?: string - residentialProxyUrl?: string + proxyPool?: ProxyPool + residentialProxyPool?: ProxyPool onTierAttempt?: (result: TierResult) => void } @@ -100,9 +105,27 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis throw new Error("Max tier reached without success") } - // Tier 3: fresh challenge solve - const remaining3 = maxTimeout - (Date.now() - totalStart) - const t3 = await runTier3(req.url, handle, remaining3, deps.proxyUrl, req.headers) + // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request + // override, then the configured datacenter proxy pool, then none (server's own IP). + // On a "blocked" result from a pool-sourced proxy, mark it bad and retry with the + // next pool proxy before falling through to Tier 4. A per-request override has no + // fallback candidate, so it's tried exactly once. + let proxy3 = req.proxy ?? deps.proxyPool?.next(domain) ?? undefined + let t3: Awaited> + for (let attempt = 0; ; attempt++) { + const remaining3 = maxTimeout - (Date.now() - totalStart) + t3 = await runTier3(req.url, handle, remaining3, proxy3, req.headers) + + const pool = deps.proxyPool + if (t3.status !== "blocked" || req.proxy || !proxy3 || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break + pool.markBad(proxy3) + const next = pool.next(domain) + if (!next || next === proxy3) break + console.log( + `[orchestrator] Tier 3 proxy ${proxy3.replace(/\/\/[^@]*@/, "//**@")} blocked — retrying with next proxy`, + ) + proxy3 = next + } emit(t3) if (t3.status === "success" && t3.html !== undefined) { const cookies: Cookie[] = t3.cookies ?? [] @@ -131,17 +154,28 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis throw new Error("Max tier reached without success") } - // Tier 4: residential proxy escalation — requires RESIDENTIAL_PROXY_URL to be set. - const proxyUrl = deps.residentialProxyUrl - if (!proxyUrl) { + // Tier 4: residential proxy escalation — requires at least one residential proxy, + // supplied either per-request (req.proxy) or via the configured residential pool. + let proxy4 = req.proxy ?? deps.residentialProxyPool?.next(domain) + if (!proxy4) { throw new Error( - `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL to enable Tier 4 proxy escalation.`, + `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`, ) } - console.log(`[orchestrator] Tier 4 via residential proxy: ${proxyUrl.replace(/\/\/[^@]*@/, "//**@")}`) - const remaining4 = maxTimeout - (Date.now() - totalStart) - const t4 = await runTier4(req.url, handle, remaining4, proxyUrl, req.headers) + let t4: Awaited> + for (let attempt = 0; ; attempt++) { + console.log(`[orchestrator] Tier 4 via residential proxy: ${proxy4.replace(/\/\/[^@]*@/, "//**@")}`) + const remaining4 = maxTimeout - (Date.now() - totalStart) + t4 = await runTier4(req.url, handle, remaining4, proxy4, req.headers) + + const pool = deps.residentialProxyPool + if (t4.status !== "blocked" || req.proxy || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break + pool.markBad(proxy4) + const next = pool.next(domain) + if (!next || next === proxy4) break + proxy4 = next + } emit(t4) if (t4.status === "success" && t4.html !== undefined) { const cookies: Cookie[] = t4.cookies ?? []