From df1cf47d083257e5ce0b7f4bae73855610c3edef Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:39:18 +0200 Subject: [PATCH 01/16] fix(browser): support per-context proxy override in fresh contexts --- packages/browser/src/pool.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/browser/src/pool.ts b/packages/browser/src/pool.ts index 2e3dc59..8603cca 100644 --- a/packages/browser/src/pool.ts +++ b/packages/browser/src/pool.ts @@ -223,8 +223,11 @@ export class BrowserPool { // A fresh context (no prior cookies/localStorage/service workers) gets CF managed-mode // treatment — challenge resolves in 3-4s vs ~40s for warm/reused contexts. // biome-ignore lint/suspicious/noExplicitAny: camoufox-js doesn't export BrowserContext type -export const newFreshContext = async (browser: any): Promise => { - const context = await browser.newContext({ viewport: null }) +export const newFreshContext = async (browser: any, options?: { proxy?: string }): Promise => { + const context = await browser.newContext({ + viewport: null, + ...(options?.proxy ? { proxy: { server: options.proxy } } : {}), + }) await context.addInitScript(() => { window.onerror = () => true window.addEventListener( From 3711e2e0416e4f9916582baf1016b126cd320e40 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:39:53 +0200 Subject: [PATCH 02/16] fix(tiers): actually thread proxy url into tier3 browser context --- packages/tiers/src/tier3.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index 97d8597..d22c26e 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -19,7 +19,7 @@ export async function runTier3( url: string, handle: BrowserHandle, maxTimeout: number, - _proxyUrl?: string, + proxyUrl?: string, extraHeaders?: Record, ): Promise { const start = Date.now() @@ -29,7 +29,7 @@ export async function runTier3( // engine state) that CF's behavioral analysis scores as suspicious — resulting in 40s // challenge evaluation. A fresh context with no prior state gets managed-mode treatment: // CF evaluates in under 1s and the challenge resolves in 3-4s total. - const freshCtx = await newFreshContext(handle.browser) + const freshCtx = await newFreshContext(handle.browser, { proxy: proxyUrl }) const page = await freshCtx.newPage() try { @@ -65,7 +65,9 @@ export async function runTier3( if (gotoErr instanceof Error) { const msg = gotoErr.message const isHardFail = - /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED/i.test(msg) + /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test( + msg, + ) if (isHardFail) { return { tier: 3, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] } } From 56e9c2cfe39101bdd88446d1aa39c81b8d0c0e62 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:11 +0200 Subject: [PATCH 03/16] feat(tiers): detect imperva/incapsula waf challenges --- packages/tiers/src/detect.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/tiers/src/detect.ts b/packages/tiers/src/detect.ts index 984cf73..01bc5c1 100644 --- a/packages/tiers/src/detect.ts +++ b/packages/tiers/src/detect.ts @@ -4,6 +4,7 @@ export type ChallengeType = | "hcaptcha" | "recaptcha" | "cap" + | "imperva" | "none" export function isCloudflarePage(html: string, headers: Record): boolean { @@ -42,9 +43,24 @@ export function hasCapChallenge(html: string): boolean { return /cap-widget|trycap\.dev|data-cap-/i.test(html) } +// Imperva/Incapsula WAF challenge — sensor-based (reese84, current) or legacy (___utmvc). +// Both are produced by an obfuscated in-page JS challenge; no need to understand the +// obfuscation, just detect the challenge page and wait for the sensor cookie (see impervaWait.ts). +export function hasImpervaChallenge(html: string, headers: Record = {}): boolean { + const lowerHeaders: Record = {} + for (const [k, v] of Object.entries(headers)) lowerHeaders[k.toLowerCase()] = v + if (lowerHeaders["x-iinfo"]) return true + if (/incapsula/i.test(lowerHeaders["x-cdn"] ?? "")) return true + if (/incapsula incident id/i.test(html)) return true + if (/_incapsula_resource/i.test(html)) return true + if (/visid_incap_|incap_ses_|nlbi_|reese84|___utmvc/i.test(html)) return true + return false +} + export function detectChallengeType(html: string, headers: Record = {}): ChallengeType { if (hasTurnstile(html)) return "cloudflare-turnstile" if (isCloudflarePage(html, headers)) return "cloudflare-interstitial" + if (hasImpervaChallenge(html, headers)) return "imperva" if (hasHcaptcha(html)) return "hcaptcha" if (hasRecaptcha(html)) return "recaptcha" if (hasCapChallenge(html)) return "cap" @@ -55,9 +71,10 @@ export function isBlocked(status: number, html: string): boolean { // 202 is used by some CDNs (e.g. IMDB) as a bot-gate before the real response if (status === 202 || status === 403 || status === 429) return true if (isCloudflarePage(html, {})) return true + if (hasImpervaChallenge(html)) return true return false } export function needsJs(html: string, headers: Record): boolean { - return isCloudflarePage(html, headers) + return isCloudflarePage(html, headers) || hasImpervaChallenge(html, headers) } From 15cae0ff30a1861a0c7f16bebd2732fdb07da329 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:11 +0200 Subject: [PATCH 04/16] feat(tiers): add imperva sensor cookie wait resolver --- packages/tiers/src/impervaWait.ts | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 packages/tiers/src/impervaWait.ts diff --git a/packages/tiers/src/impervaWait.ts b/packages/tiers/src/impervaWait.ts new file mode 100644 index 0000000..f4c06a4 --- /dev/null +++ b/packages/tiers/src/impervaWait.ts @@ -0,0 +1,84 @@ +// Imperva/Incapsula sensor-cookie wait — mirrors challengeWait.ts's CF polling loop. +// Imperva's reese84 (current) / ___utmvc (legacy) cookies are produced by an obfuscated +// in-page JS challenge that a real browser executing real JS satisfies without any +// challenge-specific logic — we just wait for the cookie and let the page proceed. +// +// Known risk: unlike CF Turnstile, Imperva's script sometimes layers in TLS/JA3 and +// behavioral checks beyond plain cookie generation — success isn't guaranteed just +// because we're a real browser. Validate against a real Imperva-protected site. + +import type { Page } from "patchright" +import { hasImpervaChallenge } from "./detect" + +export async function waitForImpervaResolution( + page: Page, + timeoutMs: number, + originalUrl?: string, +): Promise<"ok" | "ip-blocked" | "timeout"> { + const deadline = Date.now() + Math.max(timeoutMs, 30_000) + let sensorCookieAt: number | null = null + + const targetHost = (() => { + try { + return new URL(originalUrl ?? page.url()).hostname + } catch { + return "" + } + })() + + const earlyHtml = await page.content().catch(() => "") + if (earlyHtml && !hasImpervaChallenge(earlyHtml)) { + await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {}) + return "ok" + } + + // Let Imperva's sensor JS boot up before polling + await new Promise((r) => setTimeout(r, 1000)) + + while (Date.now() < deadline) { + try { + const cookies: Array<{ name: string; domain: string }> = await page + .context() + .cookies() + .catch(() => []) + const hasSensorCookie = cookies.some( + (c) => + (c.name === "reese84" || c.name === "___utmvc") && + targetHost && + (c.domain === targetHost || + c.domain === `.${targetHost}` || + targetHost.endsWith(c.domain.replace(/^\./, ""))), + ) + + if (hasSensorCookie) { + if (sensorCookieAt === null) { + sensorCookieAt = Date.now() + console.log("[imperva] sensor cookie obtained") + } + + const html = await page.content().catch(() => "") + if (!hasImpervaChallenge(html)) { + await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {}) + return "ok" + } + + // Sensor cookie set but still on the challenge page — Imperva's redirect + // sometimes doesn't auto-fire (unlike CF). Navigate ourselves after a grace period. + if (originalUrl && Date.now() - sensorCookieAt > 5000) { + console.log("[imperva] sensor cookie set but still on challenge page — navigating to original URL") + await page.goto(originalUrl, { waitUntil: "domcontentloaded", timeout: 15_000 }).catch(() => {}) + await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {}) + const html2 = await page.content().catch(() => "") + if (hasImpervaChallenge(html2)) return "ip-blocked" + return "ok" + } + } + } catch { + // Page is mid-navigation — keep polling + } + + await new Promise((r) => setTimeout(r, 300)) + } + + return "timeout" +} From 2d8769b4f57283bf139257d79fa376716a3234ca Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:17 +0200 Subject: [PATCH 05/16] feat(tiers): solve imperva challenges in tier3 and tier4 --- packages/tiers/src/tier3.ts | 25 +++++++++++++++++++++---- packages/tiers/src/tier4.ts | 26 +++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index d22c26e..b2f1f2d 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import { waitForImpervaResolution } from "./impervaWait" import { solvePageCaptchas } from "./solvers" export interface Tier3Result extends TierResult { @@ -75,7 +76,12 @@ export async function runTier3( } const remaining = maxTimeout - (Date.now() - start) - const resolution = await waitForChallengeResolution(page, remaining, url) + const peekHtml = await page.content().catch(() => "") + const challengeType = detectChallengeType(peekHtml) + const resolution = + challengeType === "imperva" + ? await waitForImpervaResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -84,8 +90,12 @@ export async function runTier3( durationMs: Date.now() - start, reason: resolution === "ip-blocked" - ? "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" - : "cloudflare-challenge-timeout", + ? challengeType === "imperva" + ? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)" + : "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" + : challengeType === "imperva" + ? "imperva-challenge-timeout" + : "cloudflare-challenge-timeout", } } @@ -118,6 +128,13 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" } } + if (hasImpervaChallenge(html)) { + const pageTitle = await page.title().catch(() => "?") + const pageUrl = page.url() + console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } + } + const rawCookies = await freshCtx.cookies() const cookies: Cookie[] = rawCookies.map( (c: { diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index 3e72d0a..941d7c8 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import { waitForImpervaResolution } from "./impervaWait" export interface Tier4Result extends TierResult { tier: 4 @@ -91,14 +92,24 @@ export async function runTier4( } const remaining = maxTimeout - (Date.now() - start) - const resolution = await waitForChallengeResolution(page, remaining, url) + const peekHtml = await page.content().catch(() => "") + const challengeType = detectChallengeType(peekHtml) + const resolution = + challengeType === "imperva" + ? await waitForImpervaResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { tier: 4, status: resolution === "ip-blocked" ? "blocked" : "timeout", durationMs: Date.now() - start, - reason: resolution === "ip-blocked" ? "proxy-ip-blocked" : "cloudflare-challenge-timeout", + reason: + resolution === "ip-blocked" + ? "proxy-ip-blocked" + : challengeType === "imperva" + ? "imperva-challenge-timeout" + : "cloudflare-challenge-timeout", } } @@ -119,6 +130,15 @@ export async function runTier4( } } + if (hasImpervaChallenge(html)) { + return { + tier: 4, + status: "blocked", + durationMs: Date.now() - start, + reason: "imperva-persistent", + } + } + const rawCookies = await proxyContext.cookies() const cookies: Cookie[] = rawCookies.map( (c: { From d4bb9533c9d5d341c380499bc8d2deb8d982201c Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:29 +0200 Subject: [PATCH 06/16] docs: document imperva/incapsula solving in tiered execution --- apps/docs/architecture/tiered-execution.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/docs/architecture/tiered-execution.md b/apps/docs/architecture/tiered-execution.md index 932444c..dc7fe80 100644 --- a/apps/docs/architecture/tiered-execution.md +++ b/apps/docs/architecture/tiered-execution.md @@ -62,6 +62,12 @@ On success: Uses [Camoufox](https://github.com/daijro/camoufox) — Firefox with fingerprint patching at the C++/Juggler level. CF's detection scripts cannot distinguish it from a real Firefox profile. +### Imperva/Incapsula challenges + +Tier 3 and Tier 4 also detect and solve Imperva/Incapsula WAF challenges, not just Cloudflare. Imperva's `reese84` (current) / `___utmvc` (legacy) sensor cookies are produced by an obfuscated in-page JS challenge — same principle as Cloudflare's `cf_clearance`: a real browser executing real JS produces the cookie without needing to understand the obfuscation. TRAWL detects the challenge page (`packages/tiers/src/detect.ts`'s `hasImpervaChallenge`) and polls for the sensor cookie (`packages/tiers/src/impervaWait.ts`) instead of the Cloudflare-specific wait loop. + +**Caveat:** unlike Turnstile, Imperva's script sometimes layers in TLS/JA3 and behavioral checks beyond plain cookie generation, and its obfuscation changes periodically — success isn't guaranteed at the same rate as Cloudflare. Some Imperva deployments also show a visible interactive CAPTCHA widget (distinct from hCaptcha/reCAPTCHA) instead of the passive sensor-only path; that variant isn't solved yet. + ## Tier 4 — Residential Proxy Escalation Same as Tier 3 but launches the browser with `RESIDENTIAL_PROXY_URL` set as the proxy. Only triggered when: From a9f18cf382ef4cf0958bb41ec614cd044cecca12 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:36 +0200 Subject: [PATCH 07/16] feat(tiers): rework proxy rotator into sticky failure-aware pool --- packages/tiers/src/index.ts | 3 +- packages/tiers/src/proxyRotator.ts | 150 +++++++++++++++++------------ 2 files changed, 88 insertions(+), 65 deletions(-) diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index a0fe724..95b5894 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -1,6 +1,7 @@ export { detectChallengeType, hasHcaptcha, + hasImpervaChallenge, hasRecaptcha, hasTurnstile, isBlocked, @@ -9,7 +10,7 @@ export { } from "./detect" export type { OrchestratorDeps } from "./orchestrator" export { scrape } from "./orchestrator" -export { clearProxyCache, getNextProxy, getRandomProxy } from "./proxyRotator" +export { ProxyPool } from "./proxyRotator" export { solvePageCaptchas } from "./solvers" export { runTier1 } from "./tier1" export { runTier2 } from "./tier2" diff --git a/packages/tiers/src/proxyRotator.ts b/packages/tiers/src/proxyRotator.ts index d02693d..67da844 100644 --- a/packages/tiers/src/proxyRotator.ts +++ b/packages/tiers/src/proxyRotator.ts @@ -1,72 +1,94 @@ -// Rotating proxy pool backed by a continuously updated free proxy list. -// Proxies are fetched once per TTL and returned round-robin. -// These are datacenter proxies — useful for IP rotation on non-CF sites but -// unlikely to help against Cloudflare's managed/Turnstile challenges alone. +// Proxy pool with sticky-per-domain routing, round-robin fallback, and failure cooldown. +// Sourced from a user-supplied comma-separated list or line-delimited file — TRAWL never +// fetches or trusts any third-party proxy list. -const PROXY_LIST_URL = "https://raw.githubusercontent.com/theriturajps/proxy-list/refs/heads/main/proxies.json" +import { readFileSync } from "node:fs" -const CACHE_TTL_MS = 60 * 60 * 1000 // 1 hour +const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxed cooldown" -interface ProxyCache { - proxies: string[] - fetchedAt: number +interface ProxyState { + url: string + badUntil: number } -let cache: ProxyCache | null = null -let cursor = 0 +export class ProxyPool { + private proxies: ProxyState[] + private cursor = 0 + private stickyByDomain = new Map() -async function fetchProxyList(): Promise { - try { - const res = await fetch(PROXY_LIST_URL, { signal: AbortSignal.timeout(10_000) }) - if (!res.ok) throw new Error(`HTTP ${res.status}`) - const json = (await res.json()) as { proxies?: string[] } | string[] - // Handle both { proxies: [...] } object and bare array formats - const raw: string[] = Array.isArray(json) ? json : ((json as { proxies?: string[] }).proxies ?? []) - // Filter out obviously invalid entries (0.0.0.0, private ranges) - return raw.filter((p) => { - if (!p || typeof p !== "string") return false - const ip = p.split(":")[0] - if (!ip || ip === "0.0.0.0") return false - if (ip.startsWith("10.") || ip.startsWith("192.168.") || ip.startsWith("127.")) return false - return true - }) - } catch (err) { - console.warn("[proxy] failed to fetch proxy list:", err instanceof Error ? err.message : err) - return [] + constructor(urls: string[]) { + this.proxies = urls.filter(Boolean).map((url) => ({ url, badUntil: 0 })) + } + + // Builds a pool from a comma-separated env var and/or a line-delimited file (one proxy + // per line, '#' comments allowed). A single URL still works — it's just a 1-element list. + // Returns null if neither source yields any proxies, so callers can treat "no proxy + // configured" the same way they did with the old single-string PROXY_URL/RESIDENTIAL_PROXY_URL. + static fromEnv(urlListEnv?: string, fileEnv?: string): ProxyPool | null { + const urls: string[] = [] + if (urlListEnv) { + urls.push( + ...urlListEnv + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ) + } + if (fileEnv) { + try { + const lines = readFileSync(fileEnv, "utf-8") + .split("\n") + .map((s) => s.trim()) + .filter((s) => s && !s.startsWith("#")) + urls.push(...lines) + } catch (err) { + console.warn(`[proxy] failed to read proxy list file ${fileEnv}:`, err instanceof Error ? err.message : err) + } + } + return urls.length > 0 ? new ProxyPool(urls) : null + } + + get size(): number { + return this.proxies.length + } + + private available(): ProxyState[] { + const now = Date.now() + return this.proxies.filter((p) => p.badUntil <= now) + } + + // Sticky-per-domain: reuse the same proxy for repeat requests to a domain (consistency + // helps avoid re-triggering challenges); falls back to round-robin across available + // proxies for new domains or once the sticky proxy has been marked bad. + next(domain?: string): string | null { + const available = this.available() + if (available.length === 0) return null + + if (domain) { + const sticky = this.stickyByDomain.get(domain) + if (sticky && available.some((p) => p.url === sticky)) return sticky + } + + const proxy = available[this.cursor % available.length] + this.cursor = (this.cursor + 1) % available.length + if (domain) this.stickyByDomain.set(domain, proxy.url) + return proxy.url + } + + random(): string | null { + const available = this.available() + if (available.length === 0) return null + return available[Math.floor(Math.random() * available.length)].url + } + + // Puts a proxy in cooldown after a tier reports "blocked"/"ip-blocked" for it — skipped + // by next()/random() until the cooldown expires. Also drops any sticky-domain mapping + // pointing at it so the next call for that domain picks a different proxy. + markBad(url: string): void { + const entry = this.proxies.find((p) => p.url === url) + if (entry) entry.badUntil = Date.now() + COOLDOWN_MS + for (const [domain, sticky] of this.stickyByDomain) { + if (sticky === url) this.stickyByDomain.delete(domain) + } } } - -async function getProxies(): Promise { - if (cache && Date.now() - cache.fetchedAt < CACHE_TTL_MS && cache.proxies.length > 0) { - return cache.proxies - } - const proxies = await fetchProxyList() - if (proxies.length > 0) { - cache = { proxies, fetchedAt: Date.now() } - cursor = 0 - console.log(`[proxy] loaded ${proxies.length} proxies`) - } - return proxies -} - -// Returns the next proxy in rotation as "http://IP:PORT", or null if unavailable. -export async function getNextProxy(): Promise { - const proxies = await getProxies() - if (proxies.length === 0) return null - const proxy = proxies[cursor % proxies.length] - cursor = (cursor + 1) % proxies.length - return `http://${proxy}` -} - -// Returns a random proxy from the pool (useful for parallel requests). -export async function getRandomProxy(): Promise { - const proxies = await getProxies() - if (proxies.length === 0) return null - const idx = Math.floor(Math.random() * proxies.length) - return `http://${proxies[idx]}` -} - -export function clearProxyCache(): void { - cache = null - cursor = 0 -} From cd5b674c34844fb53719b20dd453966f357f16f0 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:42 +0200 Subject: [PATCH 08/16] feat(types): add per-request proxy override field --- packages/types/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 4850e43..fc36577 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -16,6 +16,8 @@ export interface ScrapeRequest { maxTier?: 1 | 2 | 3 | 4 sessionId?: string headers?: Record + // Per-request proxy override — bypasses the server-configured proxy pool for this call. + proxy?: string } export interface TierResult { @@ -67,6 +69,8 @@ export interface FlareSolverrRequest { maxTimeout?: number postData?: string headers?: Record + // TRAWL extension (not part of the FlareSolverr v2 contract) — per-request proxy override. + proxy?: string } export interface FlareSolverrResponse { From 08b595af6a06005af1b9dfd5dc656ebb46ead700 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:42 +0200 Subject: [PATCH 09/16] feat(tiers): route proxy pool selection through orchestrator --- packages/tiers/src/orchestrator.ts | 58 +++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 1323501..da9d497 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -2,19 +2,24 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { normalizeHtml } from "./html" +import type { ProxyPool } from "./proxyRotator" import { runTier1 } from "./tier1" import { runTier2 } from "./tier2" import { runTier3 } from "./tier3" import { runTier4 } from "./tier4" +// Bounds how many distinct proxies a single request will try per tier before giving up — +// keeps a long proxy list from blowing the request's maxTimeout budget. +const MAX_PROXY_ATTEMPTS = 2 + export interface OrchestratorDeps { acquireBrowser(domain: string): Promise releaseBrowser(id: number): void loadSession(domain: string): Promise saveSession(domain: string, data: SessionData): Promise invalidateSession(domain: string): Promise - proxyUrl?: string - residentialProxyUrl?: string + proxyPool?: ProxyPool + residentialProxyPool?: ProxyPool onTierAttempt?: (result: TierResult) => void } @@ -100,9 +105,27 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis throw new Error("Max tier reached without success") } - // Tier 3: fresh challenge solve - const remaining3 = maxTimeout - (Date.now() - totalStart) - const t3 = await runTier3(req.url, handle, remaining3, deps.proxyUrl, req.headers) + // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request + // override, then the configured datacenter proxy pool, then none (server's own IP). + // On a "blocked" result from a pool-sourced proxy, mark it bad and retry with the + // next pool proxy before falling through to Tier 4. A per-request override has no + // fallback candidate, so it's tried exactly once. + let proxy3 = req.proxy ?? deps.proxyPool?.next(domain) ?? undefined + let t3: Awaited> + for (let attempt = 0; ; attempt++) { + const remaining3 = maxTimeout - (Date.now() - totalStart) + t3 = await runTier3(req.url, handle, remaining3, proxy3, req.headers) + + const pool = deps.proxyPool + if (t3.status !== "blocked" || req.proxy || !proxy3 || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break + pool.markBad(proxy3) + const next = pool.next(domain) + if (!next || next === proxy3) break + console.log( + `[orchestrator] Tier 3 proxy ${proxy3.replace(/\/\/[^@]*@/, "//**@")} blocked — retrying with next proxy`, + ) + proxy3 = next + } emit(t3) if (t3.status === "success" && t3.html !== undefined) { const cookies: Cookie[] = t3.cookies ?? [] @@ -131,17 +154,28 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis throw new Error("Max tier reached without success") } - // Tier 4: residential proxy escalation — requires RESIDENTIAL_PROXY_URL to be set. - const proxyUrl = deps.residentialProxyUrl - if (!proxyUrl) { + // Tier 4: residential proxy escalation — requires at least one residential proxy, + // supplied either per-request (req.proxy) or via the configured residential pool. + let proxy4 = req.proxy ?? deps.residentialProxyPool?.next(domain) + if (!proxy4) { throw new Error( - `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL to enable Tier 4 proxy escalation.`, + `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`, ) } - console.log(`[orchestrator] Tier 4 via residential proxy: ${proxyUrl.replace(/\/\/[^@]*@/, "//**@")}`) - const remaining4 = maxTimeout - (Date.now() - totalStart) - const t4 = await runTier4(req.url, handle, remaining4, proxyUrl, req.headers) + let t4: Awaited> + for (let attempt = 0; ; attempt++) { + console.log(`[orchestrator] Tier 4 via residential proxy: ${proxy4.replace(/\/\/[^@]*@/, "//**@")}`) + const remaining4 = maxTimeout - (Date.now() - totalStart) + t4 = await runTier4(req.url, handle, remaining4, proxy4, req.headers) + + const pool = deps.residentialProxyPool + if (t4.status !== "blocked" || req.proxy || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break + pool.markBad(proxy4) + const next = pool.next(domain) + if (!next || next === proxy4) break + proxy4 = next + } emit(t4) if (t4.status === "success" && t4.html !== undefined) { const cookies: Cookie[] = t4.cookies ?? [] From 8d78a1a6e151bcbaeb186bf0ccd2ce00d44e1342 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:42 +0200 Subject: [PATCH 10/16] feat(api): configure proxy pools from env and forward overrides --- apps/api/src/index.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index e31fa4b..abb1249 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,5 @@ import { BrowserPool, PoolExhaustedError, SessionCache } from "@trawl/browser" -import { scrape } from "@trawl/tiers" +import { ProxyPool, scrape } from "@trawl/tiers" import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeRequest } from "@trawl/types" import { Elysia } from "elysia" @@ -12,6 +12,17 @@ const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3") const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000") const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600") +// PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single +// URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source +// (one proxy per line) for lists too large for a single env var. +const proxyPool = + ProxyPool.fromEnv(process.env.PROXY_URL || undefined, process.env.PROXY_LIST_FILE || undefined) ?? undefined +const residentialProxyPool = + ProxyPool.fromEnv( + process.env.RESIDENTIAL_PROXY_URL || undefined, + process.env.RESIDENTIAL_PROXY_LIST_FILE || undefined, + ) ?? undefined + // Single embedded pool — no BullMQ / worker process required. // Redis is optional: without it, session caching (Tier 2 fast path) is disabled // but scraping still works via Tier 1 / Tier 3. @@ -46,6 +57,8 @@ function getDeps() { loadSession: (d: string) => (sc ? sc.load(d).catch(() => null) : Promise.resolve(null)), saveSession: (d: string, data: unknown) => (sc ? sc.save(d, data as never).catch(() => {}) : Promise.resolve()), invalidateSession: (d: string) => (sc ? sc.invalidate(d).catch(() => {}) : Promise.resolve()), + proxyPool, + residentialProxyPool, } } @@ -129,6 +142,7 @@ new Elysia() url: req.url, maxTimeout: req.maxTimeout ?? 60_000, headers: req.headers, + proxy: req.proxy, }, getDeps(), ) From 756f5aeba9ae7061c996b090c852a03215e6089d Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:40:48 +0200 Subject: [PATCH 11/16] test(tiers): add proxy pool unit tests --- packages/tiers/tests/proxyPool.test.ts | 98 ++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 packages/tiers/tests/proxyPool.test.ts diff --git a/packages/tiers/tests/proxyPool.test.ts b/packages/tiers/tests/proxyPool.test.ts new file mode 100644 index 0000000..47cc8cc --- /dev/null +++ b/packages/tiers/tests/proxyPool.test.ts @@ -0,0 +1,98 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { ProxyPool } from "../src/proxyRotator" + +describe("ProxyPool", () => { + test("round-robins across proxies for different domains", () => { + const pool = new ProxyPool(["http://p1:8080", "http://p2:8080", "http://p3:8080"]) + const picks = [pool.next("a.com"), pool.next("b.com"), pool.next("c.com"), pool.next("d.com")] + // 3 proxies, 4 picks (one new domain each) — round-robin wraps back to the first proxy + expect(picks).toEqual(["http://p1:8080", "http://p2:8080", "http://p3:8080", "http://p1:8080"]) + }) + + test("is sticky per-domain on repeat calls", () => { + const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"]) + const first = pool.next("example.com") + for (let i = 0; i < 5; i++) { + expect(pool.next("example.com")).toBe(first) + } + }) + + test("markBad excludes the proxy and clears its sticky mapping", () => { + const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"]) + const first = pool.next("example.com") + expect(first).toBeTruthy() + + pool.markBad(first as string) + + // Same domain must get a different proxy now (sticky mapping to the bad one was cleared) + const after = pool.next("example.com") + expect(after).not.toBe(first) + expect(after).toBeTruthy() + }) + + test("returns null once every proxy is marked bad", () => { + const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"]) + pool.markBad("http://p1:8080") + pool.markBad("http://p2:8080") + expect(pool.next("example.com")).toBeNull() + expect(pool.random()).toBeNull() + }) + + test("random() returns one of the configured proxies", () => { + const urls = ["http://p1:8080", "http://p2:8080", "http://p3:8080"] + const pool = new ProxyPool(urls) + for (let i = 0; i < 20; i++) { + expect(urls).toContain(pool.random()) + } + }) + + test("returns null for an empty pool", () => { + const pool = new ProxyPool([]) + expect(pool.next()).toBeNull() + expect(pool.random()).toBeNull() + expect(pool.size).toBe(0) + }) + + describe("fromEnv", () => { + let tmpDir: string | undefined + + afterEach(() => { + if (tmpDir) rmSync(tmpDir, { recursive: true, force: true }) + tmpDir = undefined + }) + + test("parses a single proxy URL", () => { + const pool = ProxyPool.fromEnv("http://p1:8080") + expect(pool?.size).toBe(1) + }) + + test("parses a comma-separated list, trimming whitespace and dropping empties", () => { + const pool = ProxyPool.fromEnv(" http://p1:8080 , http://p2:8080,,http://p3:8080 ") + expect(pool?.size).toBe(3) + }) + + test("returns null when neither source has any proxies", () => { + expect(ProxyPool.fromEnv(undefined, undefined)).toBeNull() + expect(ProxyPool.fromEnv("", "")).toBeNull() + }) + + test("reads proxies from a line-delimited file, ignoring comments and blank lines", () => { + tmpDir = mkdtempSync(join(tmpdir(), "trawl-proxy-test-")) + const file = join(tmpDir, "proxies.txt") + writeFileSync(file, "http://p1:8080\n# a comment\n\nhttp://p2:8080\n") + const pool = ProxyPool.fromEnv(undefined, file) + expect(pool?.size).toBe(2) + }) + + test("merges the env-var list and the file list", () => { + tmpDir = mkdtempSync(join(tmpdir(), "trawl-proxy-test-")) + const file = join(tmpDir, "proxies.txt") + writeFileSync(file, "http://p2:8080\n") + const pool = ProxyPool.fromEnv("http://p1:8080", file) + expect(pool?.size).toBe(2) + }) + }) +}) From 0bbbb3f9875feed9cd869704290f202944376013 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 16:41:01 +0200 Subject: [PATCH 12/16] docs: document proxy pool config, rotation, and per-request override --- apps/docs/api-reference/flaresolvr-compat.md | 2 ++ apps/docs/api-reference/native-api.md | 2 ++ apps/docs/getting-started/configuration.md | 34 ++++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/apps/docs/api-reference/flaresolvr-compat.md b/apps/docs/api-reference/flaresolvr-compat.md index 41898ca..ba1c9dd 100644 --- a/apps/docs/api-reference/flaresolvr-compat.md +++ b/apps/docs/api-reference/flaresolvr-compat.md @@ -18,6 +18,7 @@ interface FlareSolverrRequest { maxTimeout?: number // milliseconds, default 60000 postData?: string // body for request.post headers?: Record + proxy?: string // TRAWL extension — not part of the real FlareSolverr contract } ``` @@ -30,6 +31,7 @@ interface FlareSolverrRequest { | `maxTimeout` | number | No | Max wait in ms (default 60000) | | `postData` | string | No | POST body (only for `request.post`) | | `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) | +| `proxy` | string | No | **TRAWL-specific extension** (not in the real FlareSolverr v2 contract) — per-request proxy override for Tier 3/4, see [Configuration § Proxies](/getting-started/configuration#proxies) | ## Response diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index 65271eb..02c958c 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -17,6 +17,7 @@ interface ScrapeRequest { maxTier?: 1 | 2 | 3 | 4 // cap escalation at this tier sessionId?: string // sticky session override key headers?: Record // custom headers forwarded to the target + proxy?: string // per-request proxy override for Tier 3/4 } ``` @@ -30,6 +31,7 @@ interface ScrapeRequest { | `maxTier` | 1–4 | 4 | Never escalate beyond this tier | | `sessionId` | string | hostname | Override the Redis session key | | `headers` | object | — | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) | +| `proxy` | string | — | Proxy URL used for this request's Tier 3/4 attempts instead of the configured `PROXY_URL`/`RESIDENTIAL_PROXY_URL` pool — see [Configuration § Proxies](/getting-started/configuration#proxies) | ## Response diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index aba9a84..61555a8 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -84,10 +84,11 @@ SESSION_TTL_SECONDS=1800 # more conservative **Default:** _(empty — no proxy)_ -Datacenter proxy used for Tier 3 (fresh challenge solve). Format: `protocol://user:pass@host:port`. +Datacenter proxy pool used for Tier 3 (fresh challenge solve). Format: `protocol://user:pass@host:port`, or a **comma-separated list** for multiple proxies: ```ini PROXY_URL=http://user:pass@dc-proxy.example.com:8080 +PROXY_URL=http://user:pass@dc1.example.com:8080,http://user:pass@dc2.example.com:8080 ``` Leave empty to run Tier 3 without a proxy (your server's real IP is used). @@ -96,12 +97,37 @@ Leave empty to run Tier 3 without a proxy (your server's real IP is used). **Default:** _(empty — Tier 4 disabled)_ -Residential proxy used for Tier 4 (when the datacenter IP is flagged). Same format as `PROXY_URL`. Tier 4 is completely skipped if this variable is not set. +Residential proxy pool used for Tier 4 (when the datacenter IP is flagged). Same format as `PROXY_URL` — single URL or comma-separated list. Tier 4 is completely skipped if this variable is not set and no per-request `proxy` override is supplied. ```ini RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080 ``` +### `PROXY_LIST_FILE` / `RESIDENTIAL_PROXY_LIST_FILE` + +**Default:** _(empty)_ + +Alternative to cramming a large proxy list into `PROXY_URL`/`RESIDENTIAL_PROXY_URL` — path to a file with one proxy URL per line (`#` comments allowed). Merged with the corresponding `*_URL` env var if both are set. + +```ini +PROXY_LIST_FILE=/etc/trawl/datacenter-proxies.txt +RESIDENTIAL_PROXY_LIST_FILE=/etc/trawl/residential-proxies.txt +``` + +### Rotation and failure handling + +When more than one proxy is configured, TRAWL picks proxies **sticky-per-domain** — repeat requests to the same hostname keep reusing the same proxy (helps avoid re-triggering challenges), while different domains spread round-robin across the pool. If a tier attempt comes back `"blocked"` using a pool-sourced proxy, that proxy is put in a 5-minute cooldown and the request retries once with the next available proxy before falling through (Tier 3 → Tier 4, or Tier 4 failing outright) — bounded to 2 attempts per tier so a long list can't blow the request's `maxTimeout`. + +### Per-request override + +Both `POST /scrape` and `POST /v1` accept an optional `proxy` field in the request body — when present, it's used directly for that request's Tier 3/4 attempts instead of the configured pool (and isn't retried against other pool proxies on failure, since it's caller-supplied): + +```json +{ "url": "https://example.com", "proxy": "http://user:pass@my-proxy.example.com:8080" } +``` + +Note: `proxy` on `/v1` is a TRAWL-specific extension — it is not part of the real FlareSolverr v2 contract, so other FlareSolverr-compatible clients simply won't send it. + ## Ports ### `PORT_API` @@ -129,9 +155,11 @@ BROWSER_POOL_SIZE=3 BROWSER_ACQUIRE_TIMEOUT_MS=15000 SESSION_TTL_SECONDS=3600 -# ── Proxies (optional) ──────────────────────── +# ── Proxies (optional, comma-separated lists) ─ PROXY_URL= RESIDENTIAL_PROXY_URL= +PROXY_LIST_FILE= +RESIDENTIAL_PROXY_LIST_FILE= # ── Ports ───────────────────────────────────── PORT_API=8191 From 0d35b4d698c078c3bd6fba7bfcb9c2e11c8e8347 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 17:00:01 +0200 Subject: [PATCH 13/16] style(web): reformat cta section line wrap --- apps/web/app/components/CtaSection.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/web/app/components/CtaSection.vue b/apps/web/app/components/CtaSection.vue index 4c5e070..770da12 100644 --- a/apps/web/app/components/CtaSection.vue +++ b/apps/web/app/components/CtaSection.vue @@ -124,7 +124,8 @@ const tabs: { id: Tab; label: string; hint: string }[] = [

Older hardware or a Synology NAS without AVX2 / kernel < 5.1? Swap :latest for - :baseline in the image tag above — degrades gracefully down to kernel 3.10 — + :baseline + in the image tag above — degrades gracefully down to kernel 3.10 — same commands, different runtime.

From a78ba4d55a135080cfe1940f62b638689a87bb77 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 18:05:34 +0200 Subject: [PATCH 14/16] fix(compose): make API host port configurable via PORT env var --- docker-compose.full.yml | 2 +- docker-compose.minimal.yml | 2 +- docker-compose.prod.yml | 2 +- docker-compose.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.full.yml b/docker-compose.full.yml index cdcd350..3fbfd14 100644 --- a/docker-compose.full.yml +++ b/docker-compose.full.yml @@ -14,7 +14,7 @@ services: image: ghcr.io/germondai/trawl:latest restart: unless-stopped ports: - - "${PORT_API:-8191}:8191" + - "${PORT:-8191}:8191" environment: - REDIS_URL=redis://redis:6379 - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-3} diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index 59a8c4a..2cdd054 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -2,7 +2,7 @@ services: trawl: image: ghcr.io/germondai/trawl:latest ports: - - "8191:8191" + - "${PORT:-8191}:8191" shm_size: 1gb environment: BROWSER_POOL_SIZE: 1 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a779972..8624156 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -9,7 +9,7 @@ services: image: ghcr.io/germondai/trawl:latest restart: always ports: - - "8191:8191" + - "${PORT:-8191}:8191" shm_size: 1gb mem_limit: 3g environment: diff --git a/docker-compose.yml b/docker-compose.yml index 614f18d..fc2202a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,11 @@ services: trawl: image: ghcr.io/germondai/trawl:latest ports: - - "8191:8191" + - "${PORT:-8191}:8191" shm_size: 1gb environment: REDIS_URL: redis://redis:6379 - BROWSER_POOL_SIZE: 3 + BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3} depends_on: - redis healthcheck: From e05d9afee3cb64ab2ebad8be836598e232298610 Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 18:06:02 +0200 Subject: [PATCH 15/16] refactor(api): rename PORT_API env var to PORT --- apps/api/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index abb1249..e2eafc3 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -4,7 +4,7 @@ import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeReques import { Elysia } from "elysia" const REDIS_URL = process.env.REDIS_URL ?? "redis://localhost:6379" -const PORT = Number(process.env.PORT_API ?? "8191") +const PORT = Number(process.env.PORT ?? "8191") const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3") // How long acquire() will poll for a free browser before rejecting with PoolExhaustedError. // 15s covers a full CF challenge burst with pool=3 (queue depth 7, slowest finishes at ~12s). From 9261d5d92f4bd28019e321174425fa1bdd68c8cc Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 5 Jul 2026 18:06:12 +0200 Subject: [PATCH 16/16] docs: update references to renamed PORT env var --- .env.example | 2 +- README.md | 2 +- apps/docs/api-reference/overview.md | 2 +- apps/docs/getting-started/configuration.md | 13 ++++++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 78ad16e..bd35d35 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,7 @@ BROWSER_POOL_SIZE=5 SESSION_TTL_SECONDS=3600 PROXY_URL= RESIDENTIAL_PROXY_URL= -PORT_API=8191 +PORT=8191 PORT_WEB=3000 PORT_DOCS=3001 diff --git a/README.md b/README.md index aa5efb0..01cb5bb 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Synology note: many Synology NAS units (DSM 7.x on J4125 / older hardware) ship | `REDIS_URL` | `redis://localhost:6379` | Redis connection string | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | | `STT_URL` | — | Local Whisper endpoint for reCAPTCHA (optional) | -| `PORT_API` | `8191` | API listen port | +| `PORT` | `8191` | API listen port | ## Stack diff --git a/apps/docs/api-reference/overview.md b/apps/docs/api-reference/overview.md index 04ad31a..fa8998c 100644 --- a/apps/docs/api-reference/overview.md +++ b/apps/docs/api-reference/overview.md @@ -11,7 +11,7 @@ description: Base URL, content types, and response conventions. http://localhost:8191 ``` -Or wherever you've mapped `PORT_API` (default `8191`). +Or wherever you've mapped `PORT` (default `8191`). ## Authentication diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index 61555a8..3282051 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -130,11 +130,18 @@ Note: `proxy` on `/v1` is a TRAWL-specific extension — it is not part of the r ## Ports -### `PORT_API` +### `PORT` **Default:** `8191` -Port the Elysia API server listens on. Defaults to `8191` — the same port FlareSolverr and Byparr use, so you can swap TRAWL in without changing any *arr app settings. +Host port the Docker port mapping forwards to TRAWL's internal listener. Defaults to `8191` — the same port FlareSolverr and Byparr use, so you can swap TRAWL in without changing any *arr app settings. The container itself always listens on `8191` internally; `PORT` only changes the **host-side** port (e.g. `"${PORT:-8191}:8191"` in every compose file). + +To run TRAWL alongside FlareSolverr (or any other service that already binds `8191` on the host), set `PORT` in your shell or `.env` to any free port **before** running `docker compose up`: + +```bash +PORT=9191 docker compose up -d +# TRAWL reachable at http://localhost:9191, while port 8191 stays free for FlareSolverr. +``` ### `PORT_WEB` @@ -162,6 +169,6 @@ PROXY_LIST_FILE= RESIDENTIAL_PROXY_LIST_FILE= # ── Ports ───────────────────────────────────── -PORT_API=8191 +PORT=8191 PORT_WEB=3000 ```