mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
Merge branch 'main' into fix/pr-8-hardening
This commit is contained in:
Germond
+1
-1
@@ -3,7 +3,7 @@ BROWSER_POOL_SIZE=5
|
||||
SESSION_TTL_SECONDS=3600
|
||||
PROXY_URL=
|
||||
RESIDENTIAL_PROXY_URL=
|
||||
PORT_API=8191
|
||||
PORT=8191
|
||||
PORT_WEB=3000
|
||||
PORT_DOCS=3001
|
||||
|
||||
|
||||
@@ -123,7 +123,7 @@ Synology note: many Synology NAS units (DSM 7.x on J4125 / older hardware) ship
|
||||
| `REDIS_URL` | `redis://localhost:6379` | Redis connection string |
|
||||
| `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation |
|
||||
| `STT_URL` | — | Local Whisper endpoint for reCAPTCHA (optional) |
|
||||
| `PORT_API` | `8191` | API listen port |
|
||||
| `PORT` | `8191` | API listen port |
|
||||
|
||||
## Stack
|
||||
|
||||
|
||||
+14
-1
@@ -12,7 +12,7 @@ import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeReques
|
||||
import { Elysia } from "elysia"
|
||||
|
||||
const REDIS_URL = process.env.REDIS_URL ?? "redis://localhost:6379"
|
||||
const PORT = Number(process.env.PORT_API ?? "8191")
|
||||
const PORT = Number(process.env.PORT ?? "8191")
|
||||
const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3")
|
||||
// How long acquire() will poll for a free browser before rejecting with PoolExhaustedError.
|
||||
// 15s covers a full CF challenge burst with pool=3 (queue depth 7, slowest finishes at ~12s).
|
||||
@@ -20,6 +20,17 @@ const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3")
|
||||
const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000")
|
||||
const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600")
|
||||
|
||||
// PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single
|
||||
// URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source
|
||||
// (one proxy per line) for lists too large for a single env var.
|
||||
const proxyPool =
|
||||
ProxyPool.fromEnv(process.env.PROXY_URL || undefined, process.env.PROXY_LIST_FILE || undefined) ?? undefined
|
||||
const residentialProxyPool =
|
||||
ProxyPool.fromEnv(
|
||||
process.env.RESIDENTIAL_PROXY_URL || undefined,
|
||||
process.env.RESIDENTIAL_PROXY_LIST_FILE || undefined,
|
||||
) ?? undefined
|
||||
|
||||
// Single embedded pool — no BullMQ / worker process required.
|
||||
// Redis is optional: without it, session caching (Tier 2 fast path) is disabled
|
||||
// but scraping still works via Tier 1 / Tier 3.
|
||||
@@ -54,6 +65,8 @@ function getDeps() {
|
||||
loadSession: (d: string) => (sc ? sc.load(d).catch(() => null) : Promise.resolve(null)),
|
||||
saveSession: (d: string, data: unknown) => (sc ? sc.save(d, data as never).catch(() => {}) : Promise.resolve()),
|
||||
invalidateSession: (d: string) => (sc ? sc.invalidate(d).catch(() => {}) : Promise.resolve()),
|
||||
proxyPool,
|
||||
residentialProxyPool,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ interface FlareSolverrRequest {
|
||||
maxTimeout?: number // milliseconds, default 60000
|
||||
postData?: string // body for request.post
|
||||
headers?: Record<string, string>
|
||||
proxy?: string // TRAWL extension — not part of the real FlareSolverr contract
|
||||
}
|
||||
```
|
||||
|
||||
@@ -30,6 +31,7 @@ interface FlareSolverrRequest {
|
||||
| `maxTimeout` | number | No | Max wait in ms (default 60000) |
|
||||
| `postData` | string | No | POST body (only for `request.post`). On TRAWL's native `/scrape` endpoint this field is named `body`; the `/v1` adapter maps `postData` → `body` internally so the FlareSolverr wire contract stays unchanged for existing callers. |
|
||||
| `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) |
|
||||
| `proxy` | string | No | **TRAWL-specific extension** (not in the real FlareSolverr v2 contract) — per-request proxy override for Tier 3/4, see [Configuration § Proxies](/getting-started/configuration#proxies) |
|
||||
|
||||
## Response
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ interface ScrapeRequest {
|
||||
maxTier?: 1 | 2 | 3 | 4 // cap escalation at this tier
|
||||
sessionId?: string // sticky session override key
|
||||
headers?: Record<string, string> // custom headers forwarded to the target
|
||||
proxy?: string // per-request proxy override for Tier 3/4
|
||||
}
|
||||
```
|
||||
|
||||
@@ -30,6 +31,7 @@ interface ScrapeRequest {
|
||||
| `maxTier` | 1–4 | 4 | Never escalate beyond this tier |
|
||||
| `sessionId` | string | hostname | Override the Redis session key |
|
||||
| `headers` | object | — | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) |
|
||||
| `proxy` | string | — | Proxy URL used for this request's Tier 3/4 attempts instead of the configured `PROXY_URL`/`RESIDENTIAL_PROXY_URL` pool — see [Configuration § Proxies](/getting-started/configuration#proxies) |
|
||||
|
||||
## Response
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ description: Base URL, content types, and response conventions.
|
||||
http://localhost:8191
|
||||
```
|
||||
|
||||
Or wherever you've mapped `PORT_API` (default `8191`).
|
||||
Or wherever you've mapped `PORT` (default `8191`).
|
||||
|
||||
## Authentication
|
||||
|
||||
|
||||
@@ -62,6 +62,12 @@ On success:
|
||||
|
||||
Uses [Camoufox](https://github.com/daijro/camoufox) — Firefox with fingerprint patching at the C++/Juggler level. CF's detection scripts cannot distinguish it from a real Firefox profile.
|
||||
|
||||
### Imperva/Incapsula challenges
|
||||
|
||||
Tier 3 and Tier 4 also detect and solve Imperva/Incapsula WAF challenges, not just Cloudflare. Imperva's `reese84` (current) / `___utmvc` (legacy) sensor cookies are produced by an obfuscated in-page JS challenge — same principle as Cloudflare's `cf_clearance`: a real browser executing real JS produces the cookie without needing to understand the obfuscation. TRAWL detects the challenge page (`packages/tiers/src/detect.ts`'s `hasImpervaChallenge`) and polls for the sensor cookie (`packages/tiers/src/impervaWait.ts`) instead of the Cloudflare-specific wait loop.
|
||||
|
||||
**Caveat:** unlike Turnstile, Imperva's script sometimes layers in TLS/JA3 and behavioral checks beyond plain cookie generation, and its obfuscation changes periodically — success isn't guaranteed at the same rate as Cloudflare. Some Imperva deployments also show a visible interactive CAPTCHA widget (distinct from hCaptcha/reCAPTCHA) instead of the passive sensor-only path; that variant isn't solved yet.
|
||||
|
||||
## Tier 4 — Residential Proxy Escalation
|
||||
|
||||
Same as Tier 3 but launches the browser with `RESIDENTIAL_PROXY_URL` set as the proxy. Only triggered when:
|
||||
|
||||
@@ -84,10 +84,11 @@ SESSION_TTL_SECONDS=1800 # more conservative
|
||||
|
||||
**Default:** _(empty — no proxy)_
|
||||
|
||||
Datacenter proxy used for Tier 3 (fresh challenge solve). Format: `protocol://user:pass@host:port`.
|
||||
Datacenter proxy pool used for Tier 3 (fresh challenge solve). Format: `protocol://user:pass@host:port`, or a **comma-separated list** for multiple proxies:
|
||||
|
||||
```ini
|
||||
PROXY_URL=http://user:pass@dc-proxy.example.com:8080
|
||||
PROXY_URL=http://user:pass@dc1.example.com:8080,http://user:pass@dc2.example.com:8080
|
||||
```
|
||||
|
||||
Leave empty to run Tier 3 without a proxy (your server's real IP is used).
|
||||
@@ -96,19 +97,51 @@ Leave empty to run Tier 3 without a proxy (your server's real IP is used).
|
||||
|
||||
**Default:** _(empty — Tier 4 disabled)_
|
||||
|
||||
Residential proxy used for Tier 4 (when the datacenter IP is flagged). Same format as `PROXY_URL`. Tier 4 is completely skipped if this variable is not set.
|
||||
Residential proxy pool used for Tier 4 (when the datacenter IP is flagged). Same format as `PROXY_URL` — single URL or comma-separated list. Tier 4 is completely skipped if this variable is not set and no per-request `proxy` override is supplied.
|
||||
|
||||
```ini
|
||||
RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080
|
||||
```
|
||||
|
||||
### `PROXY_LIST_FILE` / `RESIDENTIAL_PROXY_LIST_FILE`
|
||||
|
||||
**Default:** _(empty)_
|
||||
|
||||
Alternative to cramming a large proxy list into `PROXY_URL`/`RESIDENTIAL_PROXY_URL` — path to a file with one proxy URL per line (`#` comments allowed). Merged with the corresponding `*_URL` env var if both are set.
|
||||
|
||||
```ini
|
||||
PROXY_LIST_FILE=/etc/trawl/datacenter-proxies.txt
|
||||
RESIDENTIAL_PROXY_LIST_FILE=/etc/trawl/residential-proxies.txt
|
||||
```
|
||||
|
||||
### Rotation and failure handling
|
||||
|
||||
When more than one proxy is configured, TRAWL picks proxies **sticky-per-domain** — repeat requests to the same hostname keep reusing the same proxy (helps avoid re-triggering challenges), while different domains spread round-robin across the pool. If a tier attempt comes back `"blocked"` using a pool-sourced proxy, that proxy is put in a 5-minute cooldown and the request retries once with the next available proxy before falling through (Tier 3 → Tier 4, or Tier 4 failing outright) — bounded to 2 attempts per tier so a long list can't blow the request's `maxTimeout`.
|
||||
|
||||
### Per-request override
|
||||
|
||||
Both `POST /scrape` and `POST /v1` accept an optional `proxy` field in the request body — when present, it's used directly for that request's Tier 3/4 attempts instead of the configured pool (and isn't retried against other pool proxies on failure, since it's caller-supplied):
|
||||
|
||||
```json
|
||||
{ "url": "https://example.com", "proxy": "http://user:pass@my-proxy.example.com:8080" }
|
||||
```
|
||||
|
||||
Note: `proxy` on `/v1` is a TRAWL-specific extension — it is not part of the real FlareSolverr v2 contract, so other FlareSolverr-compatible clients simply won't send it.
|
||||
|
||||
## Ports
|
||||
|
||||
### `PORT_API`
|
||||
### `PORT`
|
||||
|
||||
**Default:** `8191`
|
||||
|
||||
Port the Elysia API server listens on. Defaults to `8191` — the same port FlareSolverr and Byparr use, so you can swap TRAWL in without changing any *arr app settings.
|
||||
Host port the Docker port mapping forwards to TRAWL's internal listener. Defaults to `8191` — the same port FlareSolverr and Byparr use, so you can swap TRAWL in without changing any *arr app settings. The container itself always listens on `8191` internally; `PORT` only changes the **host-side** port (e.g. `"${PORT:-8191}:8191"` in every compose file).
|
||||
|
||||
To run TRAWL alongside FlareSolverr (or any other service that already binds `8191` on the host), set `PORT` in your shell or `.env` to any free port **before** running `docker compose up`:
|
||||
|
||||
```bash
|
||||
PORT=9191 docker compose up -d
|
||||
# TRAWL reachable at http://localhost:9191, while port 8191 stays free for FlareSolverr.
|
||||
```
|
||||
|
||||
### `PORT_WEB`
|
||||
|
||||
@@ -129,11 +162,13 @@ BROWSER_POOL_SIZE=3
|
||||
BROWSER_ACQUIRE_TIMEOUT_MS=15000
|
||||
SESSION_TTL_SECONDS=3600
|
||||
|
||||
# ── Proxies (optional) ────────────────────────
|
||||
# ── Proxies (optional, comma-separated lists) ─
|
||||
PROXY_URL=
|
||||
RESIDENTIAL_PROXY_URL=
|
||||
PROXY_LIST_FILE=
|
||||
RESIDENTIAL_PROXY_LIST_FILE=
|
||||
|
||||
# ── Ports ─────────────────────────────────────
|
||||
PORT_API=8191
|
||||
PORT=8191
|
||||
PORT_WEB=3000
|
||||
```
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
image: ghcr.io/germondai/trawl:latest
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${PORT_API:-8191}:8191"
|
||||
- "${PORT:-8191}:8191"
|
||||
environment:
|
||||
- REDIS_URL=redis://redis:6379
|
||||
- BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-3}
|
||||
|
||||
@@ -2,7 +2,7 @@ services:
|
||||
trawl:
|
||||
image: ghcr.io/germondai/trawl:latest
|
||||
ports:
|
||||
- "8191:8191"
|
||||
- "${PORT:-8191}:8191"
|
||||
shm_size: 1gb
|
||||
environment:
|
||||
BROWSER_POOL_SIZE: 1
|
||||
|
||||
@@ -9,7 +9,7 @@ services:
|
||||
image: ghcr.io/germondai/trawl:latest
|
||||
restart: always
|
||||
ports:
|
||||
- "8191:8191"
|
||||
- "${PORT:-8191}:8191"
|
||||
shm_size: 1gb
|
||||
mem_limit: 3g
|
||||
environment:
|
||||
|
||||
+2
-2
@@ -7,11 +7,11 @@ services:
|
||||
trawl:
|
||||
image: ghcr.io/germondai/trawl:latest
|
||||
ports:
|
||||
- "8191:8191"
|
||||
- "${PORT:-8191}:8191"
|
||||
shm_size: 1gb
|
||||
environment:
|
||||
REDIS_URL: redis://redis:6379
|
||||
BROWSER_POOL_SIZE: 3
|
||||
BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3}
|
||||
depends_on:
|
||||
- redis
|
||||
healthcheck:
|
||||
|
||||
@@ -223,8 +223,11 @@ export class BrowserPool {
|
||||
// A fresh context (no prior cookies/localStorage/service workers) gets CF managed-mode
|
||||
// treatment — challenge resolves in 3-4s vs ~40s for warm/reused contexts.
|
||||
// biome-ignore lint/suspicious/noExplicitAny: camoufox-js doesn't export BrowserContext type
|
||||
export const newFreshContext = async (browser: any): Promise<any> => {
|
||||
const context = await browser.newContext({ viewport: null })
|
||||
export const newFreshContext = async (browser: any, options?: { proxy?: string }): Promise<any> => {
|
||||
const context = await browser.newContext({
|
||||
viewport: null,
|
||||
...(options?.proxy ? { proxy: { server: options.proxy } } : {}),
|
||||
})
|
||||
await context.addInitScript(() => {
|
||||
window.onerror = () => true
|
||||
window.addEventListener(
|
||||
|
||||
@@ -4,6 +4,7 @@ export type ChallengeType =
|
||||
| "hcaptcha"
|
||||
| "recaptcha"
|
||||
| "cap"
|
||||
| "imperva"
|
||||
| "none"
|
||||
|
||||
export function isCloudflarePage(html: string, headers: Record<string, string>): boolean {
|
||||
@@ -42,9 +43,24 @@ export function hasCapChallenge(html: string): boolean {
|
||||
return /cap-widget|trycap\.dev|data-cap-/i.test(html)
|
||||
}
|
||||
|
||||
// Imperva/Incapsula WAF challenge — sensor-based (reese84, current) or legacy (___utmvc).
|
||||
// Both are produced by an obfuscated in-page JS challenge; no need to understand the
|
||||
// obfuscation, just detect the challenge page and wait for the sensor cookie (see impervaWait.ts).
|
||||
export function hasImpervaChallenge(html: string, headers: Record<string, string> = {}): boolean {
|
||||
const lowerHeaders: Record<string, string> = {}
|
||||
for (const [k, v] of Object.entries(headers)) lowerHeaders[k.toLowerCase()] = v
|
||||
if (lowerHeaders["x-iinfo"]) return true
|
||||
if (/incapsula/i.test(lowerHeaders["x-cdn"] ?? "")) return true
|
||||
if (/incapsula incident id/i.test(html)) return true
|
||||
if (/_incapsula_resource/i.test(html)) return true
|
||||
if (/visid_incap_|incap_ses_|nlbi_|reese84|___utmvc/i.test(html)) return true
|
||||
return false
|
||||
}
|
||||
|
||||
export function detectChallengeType(html: string, headers: Record<string, string> = {}): ChallengeType {
|
||||
if (hasTurnstile(html)) return "cloudflare-turnstile"
|
||||
if (isCloudflarePage(html, headers)) return "cloudflare-interstitial"
|
||||
if (hasImpervaChallenge(html, headers)) return "imperva"
|
||||
if (hasHcaptcha(html)) return "hcaptcha"
|
||||
if (hasRecaptcha(html)) return "recaptcha"
|
||||
if (hasCapChallenge(html)) return "cap"
|
||||
@@ -55,9 +71,10 @@ export function isBlocked(status: number, html: string): boolean {
|
||||
// 202 is used by some CDNs (e.g. IMDB) as a bot-gate before the real response
|
||||
if (status === 202 || status === 403 || status === 429) return true
|
||||
if (isCloudflarePage(html, {})) return true
|
||||
if (hasImpervaChallenge(html)) return true
|
||||
return false
|
||||
}
|
||||
|
||||
export function needsJs(html: string, headers: Record<string, string>): boolean {
|
||||
return isCloudflarePage(html, headers)
|
||||
return isCloudflarePage(html, headers) || hasImpervaChallenge(html, headers)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Imperva/Incapsula sensor-cookie wait — mirrors challengeWait.ts's CF polling loop.
|
||||
// Imperva's reese84 (current) / ___utmvc (legacy) cookies are produced by an obfuscated
|
||||
// in-page JS challenge that a real browser executing real JS satisfies without any
|
||||
// challenge-specific logic — we just wait for the cookie and let the page proceed.
|
||||
//
|
||||
// Known risk: unlike CF Turnstile, Imperva's script sometimes layers in TLS/JA3 and
|
||||
// behavioral checks beyond plain cookie generation — success isn't guaranteed just
|
||||
// because we're a real browser. Validate against a real Imperva-protected site.
|
||||
|
||||
import type { Page } from "patchright"
|
||||
import { hasImpervaChallenge } from "./detect"
|
||||
|
||||
export async function waitForImpervaResolution(
|
||||
page: Page,
|
||||
timeoutMs: number,
|
||||
originalUrl?: string,
|
||||
): Promise<"ok" | "ip-blocked" | "timeout"> {
|
||||
const deadline = Date.now() + Math.max(timeoutMs, 30_000)
|
||||
let sensorCookieAt: number | null = null
|
||||
|
||||
const targetHost = (() => {
|
||||
try {
|
||||
return new URL(originalUrl ?? page.url()).hostname
|
||||
} catch {
|
||||
return ""
|
||||
}
|
||||
})()
|
||||
|
||||
const earlyHtml = await page.content().catch(() => "")
|
||||
if (earlyHtml && !hasImpervaChallenge(earlyHtml)) {
|
||||
await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {})
|
||||
return "ok"
|
||||
}
|
||||
|
||||
// Let Imperva's sensor JS boot up before polling
|
||||
await new Promise((r) => setTimeout(r, 1000))
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const cookies: Array<{ name: string; domain: string }> = await page
|
||||
.context()
|
||||
.cookies()
|
||||
.catch(() => [])
|
||||
const hasSensorCookie = cookies.some(
|
||||
(c) =>
|
||||
(c.name === "reese84" || c.name === "___utmvc") &&
|
||||
targetHost &&
|
||||
(c.domain === targetHost ||
|
||||
c.domain === `.${targetHost}` ||
|
||||
targetHost.endsWith(c.domain.replace(/^\./, ""))),
|
||||
)
|
||||
|
||||
if (hasSensorCookie) {
|
||||
if (sensorCookieAt === null) {
|
||||
sensorCookieAt = Date.now()
|
||||
console.log("[imperva] sensor cookie obtained")
|
||||
}
|
||||
|
||||
const html = await page.content().catch(() => "")
|
||||
if (!hasImpervaChallenge(html)) {
|
||||
await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {})
|
||||
return "ok"
|
||||
}
|
||||
|
||||
// Sensor cookie set but still on the challenge page — Imperva's redirect
|
||||
// sometimes doesn't auto-fire (unlike CF). Navigate ourselves after a grace period.
|
||||
if (originalUrl && Date.now() - sensorCookieAt > 5000) {
|
||||
console.log("[imperva] sensor cookie set but still on challenge page — navigating to original URL")
|
||||
await page.goto(originalUrl, { waitUntil: "domcontentloaded", timeout: 15_000 }).catch(() => {})
|
||||
await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {})
|
||||
const html2 = await page.content().catch(() => "")
|
||||
if (hasImpervaChallenge(html2)) return "ip-blocked"
|
||||
return "ok"
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Page is mid-navigation — keep polling
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, 300))
|
||||
}
|
||||
|
||||
return "timeout"
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
export {
|
||||
detectChallengeType,
|
||||
hasHcaptcha,
|
||||
hasImpervaChallenge,
|
||||
hasRecaptcha,
|
||||
hasTurnstile,
|
||||
isBlocked,
|
||||
@@ -20,6 +21,7 @@ export {
|
||||
type SupportedMethod,
|
||||
sanitizeHeaders,
|
||||
} from "./sanitize"
|
||||
export { ProxyPool } from "./proxyRotator"
|
||||
export { solvePageCaptchas } from "./solvers"
|
||||
export { runTier1 } from "./tier1"
|
||||
export { runTier2 } from "./tier2"
|
||||
|
||||
@@ -8,14 +8,18 @@ import { runTier2 } from "./tier2"
|
||||
import { runTier3 } from "./tier3"
|
||||
import { runTier4 } from "./tier4"
|
||||
|
||||
// Bounds how many distinct proxies a single request will try per tier before giving up —
|
||||
// keeps a long proxy list from blowing the request's maxTimeout budget.
|
||||
const MAX_PROXY_ATTEMPTS = 2
|
||||
|
||||
export interface OrchestratorDeps {
|
||||
acquireBrowser(domain: string): Promise<BrowserHandle>
|
||||
releaseBrowser(id: number): void
|
||||
loadSession(domain: string): Promise<SessionData | null>
|
||||
saveSession(domain: string, data: SessionData): Promise<void>
|
||||
invalidateSession(domain: string): Promise<void>
|
||||
proxyUrl?: string
|
||||
residentialProxyUrl?: string
|
||||
proxyPool?: ProxyPool
|
||||
residentialProxyPool?: ProxyPool
|
||||
onTierAttempt?: (result: TierResult) => void
|
||||
}
|
||||
|
||||
@@ -135,14 +139,14 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
|
||||
throw new Error("Max tier reached without success")
|
||||
}
|
||||
|
||||
// Tier 4: residential proxy escalation — requires RESIDENTIAL_PROXY_URL to be set.
|
||||
const proxyUrl = deps.residentialProxyUrl
|
||||
if (!proxyUrl) {
|
||||
// Tier 4: residential proxy escalation — requires at least one residential proxy,
|
||||
// supplied either per-request (req.proxy) or via the configured residential pool.
|
||||
let proxy4 = req.proxy ?? deps.residentialProxyPool?.next(domain)
|
||||
if (!proxy4) {
|
||||
throw new Error(
|
||||
`Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL to enable Tier 4 proxy escalation.`,
|
||||
`Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`,
|
||||
)
|
||||
}
|
||||
console.log(`[orchestrator] Tier 4 via residential proxy: ${proxyUrl.replace(/\/\/[^@]*@/, "//**@")}`)
|
||||
|
||||
const remaining4 = maxTimeout - (Date.now() - totalStart)
|
||||
const t4 = await runTier4(req.url, handle, remaining4, proxyUrl, sanitizedHeaders, req.method, req.body)
|
||||
|
||||
@@ -1,72 +1,94 @@
|
||||
// Rotating proxy pool backed by a continuously updated free proxy list.
|
||||
// Proxies are fetched once per TTL and returned round-robin.
|
||||
// These are datacenter proxies — useful for IP rotation on non-CF sites but
|
||||
// unlikely to help against Cloudflare's managed/Turnstile challenges alone.
|
||||
// Proxy pool with sticky-per-domain routing, round-robin fallback, and failure cooldown.
|
||||
// Sourced from a user-supplied comma-separated list or line-delimited file — TRAWL never
|
||||
// fetches or trusts any third-party proxy list.
|
||||
|
||||
const PROXY_LIST_URL = "https://raw.githubusercontent.com/theriturajps/proxy-list/refs/heads/main/proxies.json"
|
||||
import { readFileSync } from "node:fs"
|
||||
|
||||
const CACHE_TTL_MS = 60 * 60 * 1000 // 1 hour
|
||||
const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxed cooldown"
|
||||
|
||||
interface ProxyCache {
|
||||
proxies: string[]
|
||||
fetchedAt: number
|
||||
interface ProxyState {
|
||||
url: string
|
||||
badUntil: number
|
||||
}
|
||||
|
||||
let cache: ProxyCache | null = null
|
||||
let cursor = 0
|
||||
export class ProxyPool {
|
||||
private proxies: ProxyState[]
|
||||
private cursor = 0
|
||||
private stickyByDomain = new Map<string, string>()
|
||||
|
||||
async function fetchProxyList(): Promise<string[]> {
|
||||
try {
|
||||
const res = await fetch(PROXY_LIST_URL, { signal: AbortSignal.timeout(10_000) })
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||
const json = (await res.json()) as { proxies?: string[] } | string[]
|
||||
// Handle both { proxies: [...] } object and bare array formats
|
||||
const raw: string[] = Array.isArray(json) ? json : ((json as { proxies?: string[] }).proxies ?? [])
|
||||
// Filter out obviously invalid entries (0.0.0.0, private ranges)
|
||||
return raw.filter((p) => {
|
||||
if (!p || typeof p !== "string") return false
|
||||
const ip = p.split(":")[0]
|
||||
if (!ip || ip === "0.0.0.0") return false
|
||||
if (ip.startsWith("10.") || ip.startsWith("192.168.") || ip.startsWith("127.")) return false
|
||||
return true
|
||||
})
|
||||
} catch (err) {
|
||||
console.warn("[proxy] failed to fetch proxy list:", err instanceof Error ? err.message : err)
|
||||
return []
|
||||
constructor(urls: string[]) {
|
||||
this.proxies = urls.filter(Boolean).map((url) => ({ url, badUntil: 0 }))
|
||||
}
|
||||
|
||||
// Builds a pool from a comma-separated env var and/or a line-delimited file (one proxy
|
||||
// per line, '#' comments allowed). A single URL still works — it's just a 1-element list.
|
||||
// Returns null if neither source yields any proxies, so callers can treat "no proxy
|
||||
// configured" the same way they did with the old single-string PROXY_URL/RESIDENTIAL_PROXY_URL.
|
||||
static fromEnv(urlListEnv?: string, fileEnv?: string): ProxyPool | null {
|
||||
const urls: string[] = []
|
||||
if (urlListEnv) {
|
||||
urls.push(
|
||||
...urlListEnv
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
)
|
||||
}
|
||||
if (fileEnv) {
|
||||
try {
|
||||
const lines = readFileSync(fileEnv, "utf-8")
|
||||
.split("\n")
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s && !s.startsWith("#"))
|
||||
urls.push(...lines)
|
||||
} catch (err) {
|
||||
console.warn(`[proxy] failed to read proxy list file ${fileEnv}:`, err instanceof Error ? err.message : err)
|
||||
}
|
||||
}
|
||||
return urls.length > 0 ? new ProxyPool(urls) : null
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.proxies.length
|
||||
}
|
||||
|
||||
private available(): ProxyState[] {
|
||||
const now = Date.now()
|
||||
return this.proxies.filter((p) => p.badUntil <= now)
|
||||
}
|
||||
|
||||
// Sticky-per-domain: reuse the same proxy for repeat requests to a domain (consistency
|
||||
// helps avoid re-triggering challenges); falls back to round-robin across available
|
||||
// proxies for new domains or once the sticky proxy has been marked bad.
|
||||
next(domain?: string): string | null {
|
||||
const available = this.available()
|
||||
if (available.length === 0) return null
|
||||
|
||||
if (domain) {
|
||||
const sticky = this.stickyByDomain.get(domain)
|
||||
if (sticky && available.some((p) => p.url === sticky)) return sticky
|
||||
}
|
||||
|
||||
const proxy = available[this.cursor % available.length]
|
||||
this.cursor = (this.cursor + 1) % available.length
|
||||
if (domain) this.stickyByDomain.set(domain, proxy.url)
|
||||
return proxy.url
|
||||
}
|
||||
|
||||
random(): string | null {
|
||||
const available = this.available()
|
||||
if (available.length === 0) return null
|
||||
return available[Math.floor(Math.random() * available.length)].url
|
||||
}
|
||||
|
||||
// Puts a proxy in cooldown after a tier reports "blocked"/"ip-blocked" for it — skipped
|
||||
// by next()/random() until the cooldown expires. Also drops any sticky-domain mapping
|
||||
// pointing at it so the next call for that domain picks a different proxy.
|
||||
markBad(url: string): void {
|
||||
const entry = this.proxies.find((p) => p.url === url)
|
||||
if (entry) entry.badUntil = Date.now() + COOLDOWN_MS
|
||||
for (const [domain, sticky] of this.stickyByDomain) {
|
||||
if (sticky === url) this.stickyByDomain.delete(domain)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getProxies(): Promise<string[]> {
|
||||
if (cache && Date.now() - cache.fetchedAt < CACHE_TTL_MS && cache.proxies.length > 0) {
|
||||
return cache.proxies
|
||||
}
|
||||
const proxies = await fetchProxyList()
|
||||
if (proxies.length > 0) {
|
||||
cache = { proxies, fetchedAt: Date.now() }
|
||||
cursor = 0
|
||||
console.log(`[proxy] loaded ${proxies.length} proxies`)
|
||||
}
|
||||
return proxies
|
||||
}
|
||||
|
||||
// Returns the next proxy in rotation as "http://IP:PORT", or null if unavailable.
|
||||
export async function getNextProxy(): Promise<string | null> {
|
||||
const proxies = await getProxies()
|
||||
if (proxies.length === 0) return null
|
||||
const proxy = proxies[cursor % proxies.length]
|
||||
cursor = (cursor + 1) % proxies.length
|
||||
return `http://${proxy}`
|
||||
}
|
||||
|
||||
// Returns a random proxy from the pool (useful for parallel requests).
|
||||
export async function getRandomProxy(): Promise<string | null> {
|
||||
const proxies = await getProxies()
|
||||
if (proxies.length === 0) return null
|
||||
const idx = Math.floor(Math.random() * proxies.length)
|
||||
return `http://${proxies[idx]}`
|
||||
}
|
||||
|
||||
export function clearProxyCache(): void {
|
||||
cache = null
|
||||
cursor = 0
|
||||
}
|
||||
|
||||
@@ -2,10 +2,11 @@ import type { BrowserHandle } from "@trawl/browser"
|
||||
import { FINGERPRINT, newFreshContext } from "@trawl/browser"
|
||||
import type { Cookie, TierResult } from "@trawl/types"
|
||||
import { waitForChallengeResolution } from "./challengeWait"
|
||||
import { isCloudflarePage } from "./detect"
|
||||
import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
|
||||
import { normalizeHtml } from "./html"
|
||||
import type { RouteLike } from "./sanitize"
|
||||
import { routeContinueOverrides } from "./sanitize"
|
||||
import { waitForImpervaResolution } from "./impervaWait"
|
||||
import { solvePageCaptchas } from "./solvers"
|
||||
|
||||
export interface Tier3Result extends TierResult {
|
||||
@@ -21,7 +22,7 @@ export async function runTier3(
|
||||
url: string,
|
||||
handle: BrowserHandle,
|
||||
maxTimeout: number,
|
||||
_proxyUrl?: string,
|
||||
proxyUrl?: string,
|
||||
extraHeaders?: Record<string, string>,
|
||||
method?: string,
|
||||
body?: string,
|
||||
@@ -33,7 +34,7 @@ export async function runTier3(
|
||||
// engine state) that CF's behavioral analysis scores as suspicious — resulting in 40s
|
||||
// challenge evaluation. A fresh context with no prior state gets managed-mode treatment:
|
||||
// CF evaluates in under 1s and the challenge resolves in 3-4s total.
|
||||
const freshCtx = await newFreshContext(handle.browser)
|
||||
const freshCtx = await newFreshContext(handle.browser, { proxy: proxyUrl })
|
||||
const page = await freshCtx.newPage()
|
||||
|
||||
try {
|
||||
@@ -67,7 +68,9 @@ export async function runTier3(
|
||||
if (gotoErr instanceof Error) {
|
||||
const msg = gotoErr.message
|
||||
const isHardFail =
|
||||
/ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED/i.test(msg)
|
||||
/ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test(
|
||||
msg,
|
||||
)
|
||||
if (isHardFail) {
|
||||
return { tier: 3, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] }
|
||||
}
|
||||
@@ -75,7 +78,12 @@ export async function runTier3(
|
||||
}
|
||||
|
||||
const remaining = maxTimeout - (Date.now() - start)
|
||||
const resolution = await waitForChallengeResolution(page, remaining, url)
|
||||
const peekHtml = await page.content().catch(() => "")
|
||||
const challengeType = detectChallengeType(peekHtml)
|
||||
const resolution =
|
||||
challengeType === "imperva"
|
||||
? await waitForImpervaResolution(page, remaining, url)
|
||||
: await waitForChallengeResolution(page, remaining, url)
|
||||
|
||||
if (resolution !== "ok") {
|
||||
return {
|
||||
@@ -84,8 +92,12 @@ export async function runTier3(
|
||||
durationMs: Date.now() - start,
|
||||
reason:
|
||||
resolution === "ip-blocked"
|
||||
? "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)"
|
||||
: "cloudflare-challenge-timeout",
|
||||
? challengeType === "imperva"
|
||||
? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)"
|
||||
: "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)"
|
||||
: challengeType === "imperva"
|
||||
? "imperva-challenge-timeout"
|
||||
: "cloudflare-challenge-timeout",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +130,13 @@ export async function runTier3(
|
||||
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" }
|
||||
}
|
||||
|
||||
if (hasImpervaChallenge(html)) {
|
||||
const pageTitle = await page.title().catch(() => "?")
|
||||
const pageUrl = page.url()
|
||||
console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`)
|
||||
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" }
|
||||
}
|
||||
|
||||
const rawCookies = await freshCtx.cookies()
|
||||
const cookies: Cookie[] = rawCookies.map(
|
||||
(c: {
|
||||
|
||||
@@ -2,10 +2,11 @@ import type { BrowserHandle } from "@trawl/browser"
|
||||
import { FINGERPRINT } from "@trawl/browser"
|
||||
import type { Cookie, TierResult } from "@trawl/types"
|
||||
import { waitForChallengeResolution } from "./challengeWait"
|
||||
import { isCloudflarePage } from "./detect"
|
||||
import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
|
||||
import { normalizeHtml } from "./html"
|
||||
import type { RouteLike } from "./sanitize"
|
||||
import { routeContinueOverrides } from "./sanitize"
|
||||
import { waitForImpervaResolution } from "./impervaWait"
|
||||
|
||||
export interface Tier4Result extends TierResult {
|
||||
tier: 4
|
||||
@@ -93,14 +94,24 @@ export async function runTier4(
|
||||
}
|
||||
|
||||
const remaining = maxTimeout - (Date.now() - start)
|
||||
const resolution = await waitForChallengeResolution(page, remaining, url)
|
||||
const peekHtml = await page.content().catch(() => "")
|
||||
const challengeType = detectChallengeType(peekHtml)
|
||||
const resolution =
|
||||
challengeType === "imperva"
|
||||
? await waitForImpervaResolution(page, remaining, url)
|
||||
: await waitForChallengeResolution(page, remaining, url)
|
||||
|
||||
if (resolution !== "ok") {
|
||||
return {
|
||||
tier: 4,
|
||||
status: resolution === "ip-blocked" ? "blocked" : "timeout",
|
||||
durationMs: Date.now() - start,
|
||||
reason: resolution === "ip-blocked" ? "proxy-ip-blocked" : "cloudflare-challenge-timeout",
|
||||
reason:
|
||||
resolution === "ip-blocked"
|
||||
? "proxy-ip-blocked"
|
||||
: challengeType === "imperva"
|
||||
? "imperva-challenge-timeout"
|
||||
: "cloudflare-challenge-timeout",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,6 +132,15 @@ export async function runTier4(
|
||||
}
|
||||
}
|
||||
|
||||
if (hasImpervaChallenge(html)) {
|
||||
return {
|
||||
tier: 4,
|
||||
status: "blocked",
|
||||
durationMs: Date.now() - start,
|
||||
reason: "imperva-persistent",
|
||||
}
|
||||
}
|
||||
|
||||
const rawCookies = await proxyContext.cookies()
|
||||
const cookies: Cookie[] = rawCookies.map(
|
||||
(c: {
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test"
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { ProxyPool } from "../src/proxyRotator"
|
||||
|
||||
describe("ProxyPool", () => {
|
||||
test("round-robins across proxies for different domains", () => {
|
||||
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080", "http://p3:8080"])
|
||||
const picks = [pool.next("a.com"), pool.next("b.com"), pool.next("c.com"), pool.next("d.com")]
|
||||
// 3 proxies, 4 picks (one new domain each) — round-robin wraps back to the first proxy
|
||||
expect(picks).toEqual(["http://p1:8080", "http://p2:8080", "http://p3:8080", "http://p1:8080"])
|
||||
})
|
||||
|
||||
test("is sticky per-domain on repeat calls", () => {
|
||||
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"])
|
||||
const first = pool.next("example.com")
|
||||
for (let i = 0; i < 5; i++) {
|
||||
expect(pool.next("example.com")).toBe(first)
|
||||
}
|
||||
})
|
||||
|
||||
test("markBad excludes the proxy and clears its sticky mapping", () => {
|
||||
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"])
|
||||
const first = pool.next("example.com")
|
||||
expect(first).toBeTruthy()
|
||||
|
||||
pool.markBad(first as string)
|
||||
|
||||
// Same domain must get a different proxy now (sticky mapping to the bad one was cleared)
|
||||
const after = pool.next("example.com")
|
||||
expect(after).not.toBe(first)
|
||||
expect(after).toBeTruthy()
|
||||
})
|
||||
|
||||
test("returns null once every proxy is marked bad", () => {
|
||||
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"])
|
||||
pool.markBad("http://p1:8080")
|
||||
pool.markBad("http://p2:8080")
|
||||
expect(pool.next("example.com")).toBeNull()
|
||||
expect(pool.random()).toBeNull()
|
||||
})
|
||||
|
||||
test("random() returns one of the configured proxies", () => {
|
||||
const urls = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]
|
||||
const pool = new ProxyPool(urls)
|
||||
for (let i = 0; i < 20; i++) {
|
||||
expect(urls).toContain(pool.random())
|
||||
}
|
||||
})
|
||||
|
||||
test("returns null for an empty pool", () => {
|
||||
const pool = new ProxyPool([])
|
||||
expect(pool.next()).toBeNull()
|
||||
expect(pool.random()).toBeNull()
|
||||
expect(pool.size).toBe(0)
|
||||
})
|
||||
|
||||
describe("fromEnv", () => {
|
||||
let tmpDir: string | undefined
|
||||
|
||||
afterEach(() => {
|
||||
if (tmpDir) rmSync(tmpDir, { recursive: true, force: true })
|
||||
tmpDir = undefined
|
||||
})
|
||||
|
||||
test("parses a single proxy URL", () => {
|
||||
const pool = ProxyPool.fromEnv("http://p1:8080")
|
||||
expect(pool?.size).toBe(1)
|
||||
})
|
||||
|
||||
test("parses a comma-separated list, trimming whitespace and dropping empties", () => {
|
||||
const pool = ProxyPool.fromEnv(" http://p1:8080 , http://p2:8080,,http://p3:8080 ")
|
||||
expect(pool?.size).toBe(3)
|
||||
})
|
||||
|
||||
test("returns null when neither source has any proxies", () => {
|
||||
expect(ProxyPool.fromEnv(undefined, undefined)).toBeNull()
|
||||
expect(ProxyPool.fromEnv("", "")).toBeNull()
|
||||
})
|
||||
|
||||
test("reads proxies from a line-delimited file, ignoring comments and blank lines", () => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "trawl-proxy-test-"))
|
||||
const file = join(tmpDir, "proxies.txt")
|
||||
writeFileSync(file, "http://p1:8080\n# a comment\n\nhttp://p2:8080\n")
|
||||
const pool = ProxyPool.fromEnv(undefined, file)
|
||||
expect(pool?.size).toBe(2)
|
||||
})
|
||||
|
||||
test("merges the env-var list and the file list", () => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "trawl-proxy-test-"))
|
||||
const file = join(tmpDir, "proxies.txt")
|
||||
writeFileSync(file, "http://p2:8080\n")
|
||||
const pool = ProxyPool.fromEnv("http://p1:8080", file)
|
||||
expect(pool?.size).toBe(2)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -21,6 +21,8 @@ export interface ScrapeRequest {
|
||||
// QUERY (RFC 9341) is included — safe verb, body carries the query params.
|
||||
method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "TRACE" | "QUERY"
|
||||
body?: string
|
||||
// Per-request proxy override — bypasses the server-configured proxy pool for this call.
|
||||
proxy?: string
|
||||
}
|
||||
|
||||
export interface TierResult {
|
||||
@@ -72,6 +74,8 @@ export interface FlareSolverrRequest {
|
||||
maxTimeout?: number
|
||||
postData?: string
|
||||
headers?: Record<string, string>
|
||||
// TRAWL extension (not part of the FlareSolverr v2 contract) — per-request proxy override.
|
||||
proxy?: string
|
||||
}
|
||||
|
||||
export interface FlareSolverrResponse {
|
||||
|
||||
Reference in New Issue
Block a user