From f397815fe77b1f8c7dae44ffdf41a213e1c16c1e Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 26 Jul 2026 16:37:28 +0200 Subject: [PATCH] fix(compose): pass upstream proxy configuration --- .env.example | 9 +++- apps/docs/deployment/docker-compose.md | 13 +++++ apps/docs/getting-started/configuration.md | 60 +++++++++++++++++++++- docker-compose.minimal.yml | 4 ++ docker-compose.prod.yml | 4 ++ docker-compose.yml | 4 ++ 6 files changed, 91 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index db4707a..258b506 100644 --- a/.env.example +++ b/.env.example @@ -21,11 +21,16 @@ REDIS_URL=redis://localhost:6379 SESSION_TTL_SECONDS=3600 # Upstream proxies -# One proxy URL or a comma-separated proxy list. +# Tier 3 datacenter proxy: HTTP, authenticated HTTP, or SOCKS5. +# Examples: +# PROXY_URL=http://proxy.example.com:8080 +# PROXY_URL=http://user:pass@proxy.example.com:8080 +# PROXY_URL=socks5://proxy.example.com:1080 +# Separate multiple endpoints with commas to create a pool. PROXY_URL= # Optional file with one proxy URL per line. PROXY_LIST_FILE= -# Residential proxies used by Tier 4. +# Tier 4 residential proxy. Uses the same URL and pool formats. RESIDENTIAL_PROXY_URL= # Optional file with one residential proxy URL per line. RESIDENTIAL_PROXY_LIST_FILE= diff --git a/apps/docs/deployment/docker-compose.md b/apps/docs/deployment/docker-compose.md index 7781432..4dda9fa 100644 --- a/apps/docs/deployment/docker-compose.md +++ b/apps/docs/deployment/docker-compose.md @@ -73,6 +73,7 @@ See [Standalone Containers → Older CPUs & Synology NAS](/deployment/standalone | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before returning HTTP 429 | | `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Restart after this many blocked/needs-js outcomes; set `0` to disable | | `REDIS_URL` | `redis://redis:6379` | Redis connection (set automatically in compose) | +| `PROXY_URL` | — | Optional Tier 3 datacenter proxy or pool | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | | `MITM_PROXY_ENABLED` | `false` | Starts the general HTTP/HTTPS proxy | | `MITM_PROXY_PORT` | `8192` | Proxy listen and published port | @@ -82,6 +83,18 @@ See [Standalone Containers → Older CPUs & Synology NAS](/deployment/standalone All supplied Compose files publish port `8192` and mount the `trawl_proxy_ca` volume. The listener does not start until `MITM_PROXY_ENABLED=true`. See [Proxy Configuration](/proxy/configuration). +All supplied Compose files also pass `PROXY_URL`, `PROXY_LIST_FILE`, `RESIDENTIAL_PROXY_URL`, and +`RESIDENTIAL_PROXY_LIST_FILE` from the local environment or `.env` file. For a single residential +endpoint: + +```ini +# .env +RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080 +``` + +For supported endpoint formats, pools, and mounted list files, see +[Configuration → Proxies](/getting-started/configuration#proxies). + ## Logs ```bash diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index f6945e6..e36504b 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -106,10 +106,18 @@ SESSION_TTL_SECONDS=1800 # more conservative **Default:** _(empty — no proxy)_ -Datacenter proxy pool used for Tier 3 (fresh challenge solve). Format: `protocol://user:pass@host:port`, or a **comma-separated list** for multiple proxies: +Datacenter proxy pool used for Tier 3 (fresh challenge solve). TRAWL passes these endpoints to the +browser and supports HTTP and SOCKS5 forms: ```ini +PROXY_URL=http://dc-proxy.example.com:8080 PROXY_URL=http://user:pass@dc-proxy.example.com:8080 +PROXY_URL=socks5://dc-proxy.example.com:1080 +``` + +HTTP credentials can be embedded in the URL. For multiple endpoints, use a comma-separated list: + +```ini PROXY_URL=http://user:pass@dc1.example.com:8080,http://user:pass@dc2.example.com:8080 ``` @@ -123,8 +131,12 @@ Residential proxy pool used for Tier 4 (when the datacenter IP is flagged). Same ```ini RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080 +RESIDENTIAL_PROXY_URL=socks5://residential.example.com:1080 ``` +Provider labels such as "rotating", "sticky", "country", or "session" do not change the TRAWL +format. Use the hostname, port, and credentials supplied by the provider. + ### `PROXY_LIST_FILE` / `RESIDENTIAL_PROXY_LIST_FILE` **Default:** _(empty)_ @@ -136,6 +148,35 @@ PROXY_LIST_FILE=/etc/trawl/datacenter-proxies.txt RESIDENTIAL_PROXY_LIST_FILE=/etc/trawl/residential-proxies.txt ``` +Example file: + +```text +# /etc/trawl/residential-proxies.txt +http://user:pass@residential-1.example.com:8080 +http://user:pass@residential-2.example.com:8080 +socks5://residential-3.example.com:1080 +``` + +When using Docker, the path is inside the TRAWL container. Mount the file and pass the same +in-container path: + +```yaml +services: + trawl: + environment: + RESIDENTIAL_PROXY_LIST_FILE: /etc/trawl/residential-proxies.txt + volumes: + - ./residential-proxies.txt:/etc/trawl/residential-proxies.txt:ro +``` + +For a single endpoint or a short pool, a local `.env` beside `docker-compose.yml` is enough: + +```ini +RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080 +``` + +The supplied Compose files pass all four proxy variables into the container. + ### Rotation and failure handling When more than one proxy is configured, TRAWL picks proxies **sticky-per-domain** — repeat requests to the same hostname keep reusing the same proxy (helps avoid re-triggering challenges), while different domains spread round-robin across the pool. If a tier attempt comes back `"blocked"` using a pool-sourced proxy, that proxy is put in a 5-minute cooldown and the request retries once with the next available proxy before falling through (Tier 3 → Tier 4, or Tier 4 failing outright) — bounded to 2 attempts per tier so a long list can't blow the request's `maxTimeout`. @@ -150,6 +191,23 @@ Both `POST /scrape` and `POST /v1` accept an optional `proxy` field in the reque Note: `proxy` on `/v1` is a TRAWL-specific extension — it is not part of the real FlareSolverr v2 contract, so other FlareSolverr-compatible clients simply won't send it. +### Test an endpoint + +Test an HTTP proxy independently before starting TRAWL: + +```bash +curl --proxy http://user:pass@proxy.example.com:8080 https://api.ipify.org +``` + +For SOCKS5 with proxy-side DNS resolution: + +```bash +curl --proxy socks5h://proxy.example.com:1080 https://api.ipify.org +``` + +Use the provider's exact endpoint and authentication details. A working `curl` test confirms +connectivity, but the destination can still reject that proxy IP during a browser challenge. + ## Ports ### `PORT` diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index 10bf35e..ef27d31 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -8,6 +8,10 @@ services: shm_size: 1gb environment: BROWSER_POOL_SIZE: 1 + PROXY_URL: ${PROXY_URL:-} + PROXY_LIST_FILE: ${PROXY_LIST_FILE:-} + RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} + RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0} MITM_PROXY_PORT: ${MITM_PROXY_PORT:-8192} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 324e9f6..0c3fd66 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -17,6 +17,10 @@ services: environment: REDIS_URL: redis://redis:6379 BROWSER_POOL_SIZE: 5 + PROXY_URL: ${PROXY_URL:-} + PROXY_LIST_FILE: ${PROXY_LIST_FILE:-} + RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} + RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0} MITM_PROXY_PORT: ${MITM_PROXY_PORT:-8192} diff --git a/docker-compose.yml b/docker-compose.yml index 379e3f8..c559b0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,10 @@ services: environment: REDIS_URL: redis://redis:6379 BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3} + PROXY_URL: ${PROXY_URL:-} + PROXY_LIST_FILE: ${PROXY_LIST_FILE:-} + RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} + RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} # MITM proxy (off by default — opt in via .env) MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0}