diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index a0fe724..95b5894 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -1,6 +1,7 @@ export { detectChallengeType, hasHcaptcha, + hasImpervaChallenge, hasRecaptcha, hasTurnstile, isBlocked, @@ -9,7 +10,7 @@ export { } from "./detect" export type { OrchestratorDeps } from "./orchestrator" export { scrape } from "./orchestrator" -export { clearProxyCache, getNextProxy, getRandomProxy } from "./proxyRotator" +export { ProxyPool } from "./proxyRotator" export { solvePageCaptchas } from "./solvers" export { runTier1 } from "./tier1" export { runTier2 } from "./tier2" diff --git a/packages/tiers/src/proxyRotator.ts b/packages/tiers/src/proxyRotator.ts index d02693d..67da844 100644 --- a/packages/tiers/src/proxyRotator.ts +++ b/packages/tiers/src/proxyRotator.ts @@ -1,72 +1,94 @@ -// Rotating proxy pool backed by a continuously updated free proxy list. -// Proxies are fetched once per TTL and returned round-robin. -// These are datacenter proxies — useful for IP rotation on non-CF sites but -// unlikely to help against Cloudflare's managed/Turnstile challenges alone. +// Proxy pool with sticky-per-domain routing, round-robin fallback, and failure cooldown. +// Sourced from a user-supplied comma-separated list or line-delimited file — TRAWL never +// fetches or trusts any third-party proxy list. -const PROXY_LIST_URL = "https://raw.githubusercontent.com/theriturajps/proxy-list/refs/heads/main/proxies.json" +import { readFileSync } from "node:fs" -const CACHE_TTL_MS = 60 * 60 * 1000 // 1 hour +const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxed cooldown" -interface ProxyCache { - proxies: string[] - fetchedAt: number +interface ProxyState { + url: string + badUntil: number } -let cache: ProxyCache | null = null -let cursor = 0 +export class ProxyPool { + private proxies: ProxyState[] + private cursor = 0 + private stickyByDomain = new Map() -async function fetchProxyList(): Promise { - try { - const res = await fetch(PROXY_LIST_URL, { signal: AbortSignal.timeout(10_000) }) - if (!res.ok) throw new Error(`HTTP ${res.status}`) - const json = (await res.json()) as { proxies?: string[] } | string[] - // Handle both { proxies: [...] } object and bare array formats - const raw: string[] = Array.isArray(json) ? json : ((json as { proxies?: string[] }).proxies ?? []) - // Filter out obviously invalid entries (0.0.0.0, private ranges) - return raw.filter((p) => { - if (!p || typeof p !== "string") return false - const ip = p.split(":")[0] - if (!ip || ip === "0.0.0.0") return false - if (ip.startsWith("10.") || ip.startsWith("192.168.") || ip.startsWith("127.")) return false - return true - }) - } catch (err) { - console.warn("[proxy] failed to fetch proxy list:", err instanceof Error ? err.message : err) - return [] + constructor(urls: string[]) { + this.proxies = urls.filter(Boolean).map((url) => ({ url, badUntil: 0 })) + } + + // Builds a pool from a comma-separated env var and/or a line-delimited file (one proxy + // per line, '#' comments allowed). A single URL still works — it's just a 1-element list. + // Returns null if neither source yields any proxies, so callers can treat "no proxy + // configured" the same way they did with the old single-string PROXY_URL/RESIDENTIAL_PROXY_URL. + static fromEnv(urlListEnv?: string, fileEnv?: string): ProxyPool | null { + const urls: string[] = [] + if (urlListEnv) { + urls.push( + ...urlListEnv + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ) + } + if (fileEnv) { + try { + const lines = readFileSync(fileEnv, "utf-8") + .split("\n") + .map((s) => s.trim()) + .filter((s) => s && !s.startsWith("#")) + urls.push(...lines) + } catch (err) { + console.warn(`[proxy] failed to read proxy list file ${fileEnv}:`, err instanceof Error ? err.message : err) + } + } + return urls.length > 0 ? new ProxyPool(urls) : null + } + + get size(): number { + return this.proxies.length + } + + private available(): ProxyState[] { + const now = Date.now() + return this.proxies.filter((p) => p.badUntil <= now) + } + + // Sticky-per-domain: reuse the same proxy for repeat requests to a domain (consistency + // helps avoid re-triggering challenges); falls back to round-robin across available + // proxies for new domains or once the sticky proxy has been marked bad. + next(domain?: string): string | null { + const available = this.available() + if (available.length === 0) return null + + if (domain) { + const sticky = this.stickyByDomain.get(domain) + if (sticky && available.some((p) => p.url === sticky)) return sticky + } + + const proxy = available[this.cursor % available.length] + this.cursor = (this.cursor + 1) % available.length + if (domain) this.stickyByDomain.set(domain, proxy.url) + return proxy.url + } + + random(): string | null { + const available = this.available() + if (available.length === 0) return null + return available[Math.floor(Math.random() * available.length)].url + } + + // Puts a proxy in cooldown after a tier reports "blocked"/"ip-blocked" for it — skipped + // by next()/random() until the cooldown expires. Also drops any sticky-domain mapping + // pointing at it so the next call for that domain picks a different proxy. + markBad(url: string): void { + const entry = this.proxies.find((p) => p.url === url) + if (entry) entry.badUntil = Date.now() + COOLDOWN_MS + for (const [domain, sticky] of this.stickyByDomain) { + if (sticky === url) this.stickyByDomain.delete(domain) + } } } - -async function getProxies(): Promise { - if (cache && Date.now() - cache.fetchedAt < CACHE_TTL_MS && cache.proxies.length > 0) { - return cache.proxies - } - const proxies = await fetchProxyList() - if (proxies.length > 0) { - cache = { proxies, fetchedAt: Date.now() } - cursor = 0 - console.log(`[proxy] loaded ${proxies.length} proxies`) - } - return proxies -} - -// Returns the next proxy in rotation as "http://IP:PORT", or null if unavailable. -export async function getNextProxy(): Promise { - const proxies = await getProxies() - if (proxies.length === 0) return null - const proxy = proxies[cursor % proxies.length] - cursor = (cursor + 1) % proxies.length - return `http://${proxy}` -} - -// Returns a random proxy from the pool (useful for parallel requests). -export async function getRandomProxy(): Promise { - const proxies = await getProxies() - if (proxies.length === 0) return null - const idx = Math.floor(Math.random() * proxies.length) - return `http://${proxies[idx]}` -} - -export function clearProxyCache(): void { - cache = null - cursor = 0 -}