feat(proxy): make host required + universal challenge wall detection

apps/api/src/config.ts

@@ -27,17 +27,17 @@ export const residentialProxyPool =
 
 // ── MITM forward-proxy mode ────────────────────────────────────────────────────
 // Optional browser-backed HTTP(S) forward proxy (apps/api/src/proxy). Off by default.
-// When enabled, point a client's proxy setting at MITM_PROXY_PORT and every request it
-// makes is re-issued through the browser pool — for clients (e.g. Prowlarr) that only
-// consume cookies+UA from /v1 and re-fetch themselves, which fails on fingerprint-bound
-// Cloudflare clearances. See proxy/server.ts for the full rationale.
+// When enabled, point a client's HTTP(S) proxy at MITM_PROXY_PORT and every request is
+// re-issued through the browser pool — for clients that only consume cookies+UA from
+// /v1 and re-fetch themselves, which fails on fingerprint-bound Cloudflare clearances.
+// See proxy/server.ts for the full rationale.
 export const MITM_PROXY_ENABLED = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_ENABLED ?? "")
 export const MITM_PROXY_PORT = Number(process.env.MITM_PROXY_PORT ?? "8192")
-// Default 127.0.0.1 so the README's "private interface only" guarantee holds by default —
-// the proxy can impersonate any host to anyone who trusts its CA, so it should only ever
-// be reachable by the trusted client (a localhost-bound Prowlarr/Jackett in the same
-// docker network). Override only if you intentionally want a non-local client to use it.
-export const MITM_PROXY_HOST = process.env.MITM_PROXY_HOST ?? "127.0.0.1"
+// Default 0.0.0.0 — the dominant deployment is docker-compose (clients reach trawl
+// through the docker bridge, which requires a non-loopback bind). Loopback-only
+// operators can set MITM_PROXY_HOST=127.0.0.1. The primary safety guard remains
+// MITM_PROXY_ENABLED=false.
+export const MITM_PROXY_HOST = process.env.MITM_PROXY_HOST ?? "0.0.0.0"
 // CA cert + key live here (persist across restarts so the CA is installed once).
 export const MITM_PROXY_CA_DIR = process.env.MITM_PROXY_CA_DIR ?? "/data/proxy-ca"
 // Cap the tier the proxy will escalate to (e.g. keep it off residential Tier 4).

apps/api/src/proxy/server.ts

@@ -1,22 +1,18 @@
 import { readFileSync } from "node:fs"
 import net from "node:net"
 import tls from "node:tls"
-import { isCloudflarePage, type OrchestratorDeps, scrape } from "@trawl/tiers"
+import { detectChallengeType, isChallengeWall, type OrchestratorDeps, scrape } from "@trawl/tiers"
 import type { Cookie, SupportedMethod } from "@trawl/types"
 import { MitmCa } from "./ca"
 
 // Browser-backed MITM forward proxy.
 //
 // WHY THIS EXISTS: the FlareSolverr `/v1` contract only hands back cookies + user-agent.
-// Clients like Prowlarr take those, then RE-FETCH the target with their own HTTP stack.
-// Against sites whose Cloudflare clearance is bound to the solving browser's full
-// connection fingerprint (not just cookie+UA), that re-fetch is re-challenged and fails —
-// no cookie is portable to a plain HTTP client. See the FlareSolverr adapter for the
-// legacy path.
-//
-// This proxy sidesteps that: point the client's HTTP(S) proxy at it (per-indexer in
-// Prowlarr), and every request the client makes is transparently re-issued through the
-// real browser pool via scrape(), so Cloudflare always sees the fingerprint it cleared.
+// HTTP clients that consume that contract re-fetch the target with their own HTTP stack
+// and get re-challenged on sites whose Cloudflare clearance is bound to the solving
+// browser's full connection fingerprint — the cookie alone isn't portable. This proxy
+// sidesteps that: point the client's HTTP(S) proxy at it and every request is re-issued
+// through the browser pool via scrape(), so Cloudflare sees the fingerprint it cleared.
 //
 // It is a MITM: it terminates the client's TLS using a per-host cert from our own CA
 // (ca.ts). Only expose it to trusted clients on a private interface.
@@ -27,9 +23,9 @@ export interface MitmProxyOptions {
   port: number
   caDir: string
   deps: OrchestratorDeps
-  // Defaults to 127.0.0.1 in caller code so the proxy is unreachable from anything but
-  // the local host (the per-host loopback TLS terminators stay on 127.0.0.1 unconditionally).
-  host?: string
+  // Required — caller resolves `MITM_PROXY_HOST` (or any default it wants) and passes
+  // the resolved string here. Per-host internal TLS terminators stay on 127.0.0.1.
+  host: string
   maxTier?: 1 | 2 | 3 | 4
   maxTimeout?: number
   debug?: boolean
@@ -86,8 +82,8 @@ export function startMitmProxy(opts: MitmProxyOptions): {
   // Bind to loopback by default — a MITM proxy trusts whoever installs its CA, so it must
   // never be exposed off-host unless the operator explicitly opts in via MITM_PROXY_HOST.
   // The per-host internal TLS terminators (above) stay on 127.0.0.1 unconditionally.
-  server.listen(opts.port, opts.host ?? "127.0.0.1", () => {
-    console.log(`[proxy] MITM forward proxy on ${opts.host ?? "127.0.0.1"}:${opts.port}  (CA: ${ca.caCertPath})`)
+  server.listen(opts.port, opts.host, () => {
+    console.log(`[proxy] MITM forward proxy on ${opts.host}:${opts.port}  (CA: ${ca.caCertPath})`)
   })
 
   return { ca, server }
@@ -268,20 +264,25 @@ async function fetchRaw(
       const contentType = respHeaders["content-type"] ?? "application/octet-stream"
       const bodyBuf = Buffer.from((await resp?.body()) ?? new Uint8Array())
 
-      // Challenge interstitials are always small HTML — only sniff those, never binaries.
-      const looksHtml = /text\/html/i.test(contentType)
-      const challenged =
-        attempt === 0 &&
-        (status === 403 || status === 503) &&
-        looksHtml &&
-        isCloudflarePage(bodyBuf.toString("utf8", 0, 4096), {})
+      // Challenge wall detection — universal across all solvable challenge types (CF,
+      // Turnstile/hCaptcha/reCAPTCHA/GeeTest, Imperva, CAP). A "wall" is anything
+      // blocking page access (4xx/5xx, or a lean interstitial stub at 200); in-page
+      // widget captchas on accessible pages (status 200 + widget markers) are NOT walls
+      // — return the page as-is.
+      const challengeType = detectChallengeType(bodyBuf.toString("utf8", 0, 4096), respHeaders)
+      const wall = isChallengeWall(status, bodyBuf.length, challengeType)
 
-      if (challenged) {
-        // Same IP getting re-challenged means rotation has a real shot at clearing CF —
-        // mirror Tier 3's markBad + next().
+      if (wall) {
+        // Hit the wall on this IP — rotate to a fresh proxy next attempt.
         if (proxy && proxyPool) proxyPool.markBad(proxy)
-        // fall through to next attempt; finally still tears down the page and context
+        if (attempt === 0) {
+          // Refresh the cache so attempt 1 has solved cookies to add. Awaited (not
+          // fire-and-forget) so attempt 1 doesn't race the cache write.
+          await scrape({ url, method, body, maxTier: opts.maxTier, maxTimeout }, opts.deps).catch(() => {})
+        }
+        // attempt === 1 still walled — fall through to the final scrape() below.
       } else {
+        // Page is reachable (or just has an in-page widget — content is the answer).
         return { status, contentType, body: bodyBuf }
       }
     } finally {

apps/api/src/routes/v1.ts

@@ -5,7 +5,7 @@ import { Elysia } from "elysia"
 import { buildScrapeRequestFromFlareSolverr, flareSolverrError } from "../adapters/flaresolverr"
 import { getDeps, getPool } from "../deps"
 
-// FlareSolverr v2 compat — always open (Prowlarr/Jackett can't send auth headers)
+// FlareSolverr v2 compat — always open (the v2 spec has no auth header)
 export function v1Route() {
   return new Elysia().post("/v1", async ({ body, set }) => {
     const req = body as FlareSolverrRequest