Merge pull request #11 from germondai/fix/pr-8-hardening

feat: harden POST/method support with full verb set and validation

CHANGELOG.md

@@ -9,6 +9,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - Initial release with 4-tier execution engine
+- Native `method` + `body` support across all four scraper tiers — the
+  `FlareSolverrRequest.cmd=request.post` body is now actually delivered upstream
+  instead of being silently dropped
+- Header sanitisation at the API + orchestrator boundary: caller-supplied
+  `Host`, `Cookie`, `Authorization`, `X-Forwarded-*`, `Sec-*`, `User-Agent`,
+  `Content-Length`, and similar reserved headers are dropped before being
+  forwarded to Node fetch / Playwright
+- `ScrapeRequest.method` accepts the full standard verb set: `GET`, `POST`,
+  `PUT`, `PATCH`, `DELETE`, `HEAD`, `OPTIONS`, `TRACE`, `QUERY` (RFC 9341).
+  `CONNECT` is intentionally excluded (tunneling verb, inappropriate for a
+  proxy)
+- POST / `*` request bodies are forwarded **uncapped** — operators who want a
+  byte ceiling should impose it at their ingress / fronting proxy
+- Body-bearing requests require a `Content-Type` header; the tier functions no
+  longer auto-inject `application/x-www-form-urlencoded`, which previously
+  mislabelled JSON / XML bodies
+- `ScrapeRequest` field renamed from `postData` → `body` for REST-idiomatic
+  naming. (`FlareSolverrRequest.postData` is unchanged because it's the
+  upstream wire contract.)
+
+### Security
+- Reserved-name header denylist prevents callers from spoofing `cf_clearance`
+  cookies, overriding the per-tier `User-Agent`, or rewriting routing signals
+  (`X-Forwarded-For`, `Host`) during a POST bypass flow
+
+### Limitations
+- The Playwright `page.route(url, …)` interceptor only handles the first
+  top-frame GET to that exact URL. Server redirects to a different URL, XHR
+  sub-resources, and chained `POST→POST` form flows do not have the
+  `postData` override applied
+- No idempotency-key support; transient network failures and pool churn can
+  re-fire a POST (separate ticket)
+
+### Tests
+- `packages/tiers/tests/sanitize.test.ts` — header sanitiser, method
+  allowlist, postData size cap, Content-Type enforcement
+- `packages/tiers/tests/runTier1Post.test.ts` — tier1 GET/POST round-trip and
+  User-Agent non-override
+- Run via `bun --cwd packages/tiers test`
 - Persistent browser pool with real Google Chrome
 - Session caching via Redis
 - FlareSolverr v2-compatible `/v1` endpoint

apps/api/src/index.ts

@@ -1,5 +1,13 @@
 import { BrowserPool, PoolExhaustedError, SessionCache } from "@trawl/browser"
-import { ProxyPool, scrape } from "@trawl/tiers"
+import {
+  isValidMethod,
+  RequestValidationError,
+  requireContentTypeForBody,
+  SUPPORTED_METHODS,
+  type SupportedMethod,
+  sanitizeHeaders,
+  scrape,
+} from "@trawl/tiers"
 import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeRequest } from "@trawl/types"
 import { Elysia } from "elysia"
 
@@ -62,6 +70,29 @@ function getDeps() {
   }
 }
 
+function buildScrapeRequestFromFlareSolverr(req: FlareSolverrRequest): ScrapeRequest {
+  const method: SupportedMethod = req.cmd === "request.post" ? "POST" : "GET"
+  const headers = sanitizeHeaders(req.headers)
+  requireContentTypeForBody(headers, Boolean(req.postData))
+  return {
+    url: req.url,
+    maxTimeout: req.maxTimeout ?? 60_000,
+    headers,
+    method,
+    body: req.postData,
+  }
+}
+
+function validateScrapeRequest(req: ScrapeRequest): void {
+  if (!isValidMethod(req.method)) {
+    throw new RequestValidationError(
+      `Unsupported method: ${String(req.method)} (allowed: ${SUPPORTED_METHODS.join(", ")})`,
+      400,
+    )
+  }
+  requireContentTypeForBody(sanitizeHeaders(req.headers), Boolean(req.body))
+}
+
 const startTime = Date.now()
 
 // Build a FlareSolverr v2-shaped error envelope. Used by /v1 for every error
@@ -137,15 +168,8 @@ new Elysia()
     }
 
     try {
-      const result = await scrape(
-        {
-          url: req.url,
-          maxTimeout: req.maxTimeout ?? 60_000,
-          headers: req.headers,
-          proxy: req.proxy,
-        },
-        getDeps(),
-      )
+      const scrapeRequest = buildScrapeRequestFromFlareSolverr(req)
+      const result = await scrape(scrapeRequest, getDeps())
       return {
         status: "ok",
         message: "",
@@ -162,6 +186,10 @@ new Elysia()
         },
       } satisfies FlareSolverrResponse
     } catch (err) {
+      if (err instanceof RequestValidationError) {
+        set.status = err.statusCode
+        return flareSolverrError(req.url, err.message)
+      }
       set.status = err instanceof PoolExhaustedError ? 429 : 500
       return flareSolverrError(req.url, err instanceof Error ? err.message : String(err))
     }
@@ -179,8 +207,13 @@ new Elysia()
     }
     const req = body as ScrapeRequest
     try {
-      return await scrape(req, getDeps())
+      validateScrapeRequest(req)
+      return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps())
     } catch (err) {
+      if (err instanceof RequestValidationError) {
+        set.status = err.statusCode
+        return { error: err.message }
+      }
       if (err instanceof PoolExhaustedError) {
         set.status = 429
         return flareSolverrError(req.url ?? "", "Browser pool saturated, retry shortly")

apps/docs/api-reference/flaresolvr-compat.md

@@ -29,7 +29,7 @@ interface FlareSolverrRequest {
 | `cmd` | string | No | `"request.get"` or `"request.post"` (default `"request.get"`) |
 | `url` | string | Yes | The URL to scrape |
 | `maxTimeout` | number | No | Max wait in ms (default 60000) |
-| `postData` | string | No | POST body (only for `request.post`) |
+| `postData` | string | No | POST body (only for `request.post`). On TRAWL's native `/scrape` endpoint this field is named `body`; the `/v1` adapter maps `postData` → `body` internally so the FlareSolverr wire contract stays unchanged for existing callers. |
 | `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) |
 | `proxy` | string | No | **TRAWL-specific extension** (not in the real FlareSolverr v2 contract) — per-request proxy override for Tier 3/4, see [Configuration § Proxies](/getting-started/configuration#proxies) |
 

bun.lock

@@ -67,6 +67,7 @@
         "patchright": "^1.61.1",
       },
       "devDependencies": {
+        "@types/bun": "^1.3.14",
         "typescript": "^6.0.3",
       },
     },

packages/tiers/package.json

@@ -5,7 +5,8 @@
   "main": "./src/index.ts",
   "types": "./src/index.ts",
   "scripts": {
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "test": "bun test"
   },
   "dependencies": {
     "@trawl/types": "workspace:*",
@@ -13,6 +14,7 @@
     "patchright": "^1.61.1"
   },
   "devDependencies": {
+    "@types/bun": "^1.3.14",
     "typescript": "^6.0.3"
   }
 }

packages/tiers/src/index.ts

@@ -10,6 +10,17 @@ export {
 } from "./detect"
 export type { OrchestratorDeps } from "./orchestrator"
 export { scrape } from "./orchestrator"
+export { clearProxyCache, getNextProxy, getRandomProxy } from "./proxyRotator"
+export {
+  isValidMethod,
+  RESERVED_HEADER_NAMES,
+  RequestValidationError,
+  requireContentTypeForBody,
+  routeContinueOverrides,
+  SUPPORTED_METHODS,
+  type SupportedMethod,
+  sanitizeHeaders,
+} from "./sanitize"
 export { ProxyPool } from "./proxyRotator"
 export { solvePageCaptchas } from "./solvers"
 export { runTier1 } from "./tier1"

packages/tiers/src/orchestrator.ts

@@ -2,7 +2,7 @@ import type { BrowserHandle } from "@trawl/browser"
 import { FINGERPRINT } from "@trawl/browser"
 import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types"
 import { normalizeHtml } from "./html"
-import type { ProxyPool } from "./proxyRotator"
+import { requireContentTypeForBody, sanitizeHeaders } from "./sanitize"
 import { runTier1 } from "./tier1"
 import { runTier2 } from "./tier2"
 import { runTier3 } from "./tier3"
@@ -38,6 +38,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
   const timings: TierResult[] = []
   const domain = extractDomain(req.url)
 
+  const sanitizedHeaders = sanitizeHeaders(req.headers)
+  requireContentTypeForBody(sanitizedHeaders, Boolean(req.body))
+
   const emit = (r: TierResult) => {
     timings.push(r)
     deps.onTierAttempt?.(r)
@@ -45,7 +48,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
 
   // Tier 1: plain HTTP fetch
   if (!req.skipHttp && maxTier >= 1) {
-    const t1 = await runTier1(req.url, req.headers)
+    const t1 = await runTier1(req.url, sanitizedHeaders, req.method, req.body)
     emit(t1)
     if (t1.status === "success" && t1.html !== undefined) {
       return {
@@ -74,7 +77,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
     const session = await deps.loadSession(domain)
     if (session && maxTier >= 2) {
       const remaining = maxTimeout - (Date.now() - totalStart)
-      const t2 = await runTier2(req.url, handle, session, remaining, req.headers)
+      const t2 = await runTier2(req.url, handle, session, remaining, sanitizedHeaders, req.method, req.body)
       emit(t2)
       if (t2.status === "success" && t2.html !== undefined) {
         if (t2.cookies && t2.cookies.length > 0) {
@@ -105,27 +108,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
       throw new Error("Max tier reached without success")
     }
 
-    // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request
-    // override, then the configured datacenter proxy pool, then none (server's own IP).
-    // On a "blocked" result from a pool-sourced proxy, mark it bad and retry with the
-    // next pool proxy before falling through to Tier 4. A per-request override has no
-    // fallback candidate, so it's tried exactly once.
-    let proxy3 = req.proxy ?? deps.proxyPool?.next(domain) ?? undefined
-    let t3: Awaited<ReturnType<typeof runTier3>>
-    for (let attempt = 0; ; attempt++) {
-      const remaining3 = maxTimeout - (Date.now() - totalStart)
-      t3 = await runTier3(req.url, handle, remaining3, proxy3, req.headers)
-
-      const pool = deps.proxyPool
-      if (t3.status !== "blocked" || req.proxy || !proxy3 || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break
-      pool.markBad(proxy3)
-      const next = pool.next(domain)
-      if (!next || next === proxy3) break
-      console.log(
-        `[orchestrator] Tier 3 proxy ${proxy3.replace(/\/\/[^@]*@/, "//**@")} blocked — retrying with next proxy`,
-      )
-      proxy3 = next
-    }
+    // Tier 3: fresh challenge solve
+    const remaining3 = maxTimeout - (Date.now() - totalStart)
+    const t3 = await runTier3(req.url, handle, remaining3, deps.proxyUrl, sanitizedHeaders, req.method, req.body)
     emit(t3)
     if (t3.status === "success" && t3.html !== undefined) {
       const cookies: Cookie[] = t3.cookies ?? []
@@ -163,19 +148,8 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
       )
     }
 
-    let t4: Awaited<ReturnType<typeof runTier4>>
-    for (let attempt = 0; ; attempt++) {
-      console.log(`[orchestrator] Tier 4 via residential proxy: ${proxy4.replace(/\/\/[^@]*@/, "//**@")}`)
-      const remaining4 = maxTimeout - (Date.now() - totalStart)
-      t4 = await runTier4(req.url, handle, remaining4, proxy4, req.headers)
-
-      const pool = deps.residentialProxyPool
-      if (t4.status !== "blocked" || req.proxy || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break
-      pool.markBad(proxy4)
-      const next = pool.next(domain)
-      if (!next || next === proxy4) break
-      proxy4 = next
-    }
+    const remaining4 = maxTimeout - (Date.now() - totalStart)
+    const t4 = await runTier4(req.url, handle, remaining4, proxyUrl, sanitizedHeaders, req.method, req.body)
     emit(t4)
     if (t4.status === "success" && t4.html !== undefined) {
       const cookies: Cookie[] = t4.cookies ?? []

packages/tiers/src/sanitize.ts

@@ -0,0 +1,109 @@
+export const RESERVED_HEADER_NAMES: ReadonlySet<string> = new Set([
+  "host",
+  "cookie",
+  "authorization",
+  "proxy-authorization",
+  "user-agent",
+  "content-length",
+  "connection",
+  "transfer-encoding",
+  "upgrade",
+  "x-forwarded-for",
+  "x-forwarded-host",
+  "x-forwarded-proto",
+  "x-forwarded-port",
+  "x-real-ip",
+  "cf-connecting-ip",
+  "cf-ipcountry",
+  "cf-ray",
+  "cf-worker",
+  "cf-cache-status",
+  "sec-fetch-dest",
+  "sec-fetch-mode",
+  "sec-fetch-site",
+  "sec-fetch-user",
+  "sec-ch-ua",
+  "sec-ch-ua-mobile",
+  "sec-ch-ua-platform",
+])
+
+export function sanitizeHeaders(headers?: Record<string, string>): Record<string, string> | undefined {
+  if (!headers) return undefined
+  const out: Record<string, string> = {}
+  for (const [rawName, rawValue] of Object.entries(headers)) {
+    const name = rawName.trim()
+    if (!name || RESERVED_HEADER_NAMES.has(name.toLowerCase())) continue
+    const value = String(rawValue ?? "")
+      // biome-ignore lint/suspicious/noControlCharactersInRegex: NUL/CR/LF are exactly what we want to strip from header values
+      .replace(/[\x00\r\n]/g, "")
+      .trim()
+    if (value) out[name] = value
+  }
+  return Object.keys(out).length ? out : undefined
+}
+
+export const SUPPORTED_METHODS = [
+  "GET",
+  "POST",
+  "PUT",
+  "PATCH",
+  "DELETE",
+  "HEAD",
+  "OPTIONS",
+  "TRACE",
+  // RFC 9341 — safe verb that carries the query in the request body.
+  "QUERY",
+] as const
+
+export type SupportedMethod = (typeof SUPPORTED_METHODS)[number]
+
+const SUPPORTED_METHOD_SET: ReadonlySet<string> = new Set(SUPPORTED_METHODS)
+
+export function isValidMethod(method: unknown): method is SupportedMethod {
+  if (method === undefined) return true
+  return typeof method === "string" && SUPPORTED_METHOD_SET.has(method)
+}
+
+export class RequestValidationError extends Error {
+  readonly statusCode: number
+  constructor(message: string, statusCode: number) {
+    super(message)
+    this.name = "RequestValidationError"
+    this.statusCode = statusCode
+  }
+}
+
+export function requireContentTypeForBody(headers: Record<string, string> | undefined, hasBody: boolean): void {
+  if (!hasBody) return
+  for (const [k, v] of Object.entries(headers ?? {})) {
+    if (k.toLowerCase() === "content-type" && v?.trim()) return
+  }
+  throw new RequestValidationError("body requires a Content-Type header to be set", 400)
+}
+
+/** Minimal contract a Playwright `Route` exposes to `routeContinueOverrides`. */
+export interface RouteLike {
+  request(): { headers(): Record<string, string>; method(): string }
+  continue(overrides: object): Promise<void>
+}
+
+/**
+ * Build the `route.continue(...)` overrides for a tier 2/3/4 navigation,
+ * applying caller-supplied headers + a POST rewrite when the upstream loaded
+ * with GET. Content-Type is the caller's responsibility — see
+ * `requireContentTypeForBody`.
+ *
+ * The `postData` key in the returned object is Playwright's literal
+ * `Route.continue()` API field name — it is NOT TRAWL's `body` field. Playwright
+ * has not renamed it; we can't either without breaking the contract.
+ */
+export function routeContinueOverrides(
+  route: RouteLike,
+  extraHeaders: Record<string, string> | undefined,
+  method: string | undefined,
+  body: string | undefined,
+): { headers: Record<string, string>; method?: string; postData?: string } {
+  const req = route.request()
+  const headers = { ...req.headers(), ...extraHeaders }
+  return method === "POST" && req.method() === "GET" ? { headers, method: "POST", postData: body } : { headers }
+}

packages/tiers/src/tier1.ts

@@ -9,10 +9,17 @@ export interface Tier1Result extends TierResult {
   statusCode?: number
 }
 
-export async function runTier1(url: string, extraHeaders?: Record<string, string>): Promise<Tier1Result> {
+export async function runTier1(
+  url: string,
+  extraHeaders?: Record<string, string>,
+  method?: string,
+  body?: string,
+): Promise<Tier1Result> {
   const start = Date.now()
   try {
     const res = await fetch(url, {
+      method: method ?? "GET",
+      body: method === "POST" ? body : undefined,
       headers: {
         "User-Agent": FINGERPRINT.userAgent,
         Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",

packages/tiers/src/tier2.ts

@@ -2,6 +2,8 @@ import type { BrowserHandle } from "@trawl/browser"
 import type { Cookie, SessionData, TierResult } from "@trawl/types"
 import { isCloudflarePage } from "./detect"
 import { normalizeHtml } from "./html"
+import type { RouteLike } from "./sanitize"
+import { routeContinueOverrides } from "./sanitize"
 import { solvePageCaptchas } from "./solvers"
 
 export interface Tier2Result extends TierResult {
@@ -18,6 +20,8 @@ export async function runTier2(
   session: SessionData,
   maxTimeout: number,
   extraHeaders?: Record<string, string>,
+  method?: string,
+  body?: string,
 ): Promise<Tier2Result> {
   const start = Date.now()
   const page = await handle.context.newPage()
@@ -41,12 +45,10 @@ export async function runTier2(
 
     await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent })
 
-    if (extraHeaders && Object.keys(extraHeaders).length > 0) {
-      await page.route(
-        url,
-        (route: { request(): { headers(): Record<string, string> }; continue(o: object): Promise<void> }) =>
-          route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }),
-      )
+    if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") {
+      await page.route(url, (route: RouteLike) => {
+        route.continue(routeContinueOverrides(route, extraHeaders, method, body))
+      })
     }
 
     let statusCode = 200

packages/tiers/src/tier3.ts

@@ -4,6 +4,8 @@ import type { Cookie, TierResult } from "@trawl/types"
 import { waitForChallengeResolution } from "./challengeWait"
 import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
 import { normalizeHtml } from "./html"
+import type { RouteLike } from "./sanitize"
+import { routeContinueOverrides } from "./sanitize"
 import { waitForImpervaResolution } from "./impervaWait"
 import { solvePageCaptchas } from "./solvers"
 
@@ -22,6 +24,8 @@ export async function runTier3(
   maxTimeout: number,
   proxyUrl?: string,
   extraHeaders?: Record<string, string>,
+  method?: string,
+  body?: string,
 ): Promise<Tier3Result> {
   const start = Date.now()
 
@@ -34,12 +38,10 @@ export async function runTier3(
   const page = await freshCtx.newPage()
 
   try {
-    if (extraHeaders && Object.keys(extraHeaders).length > 0) {
-      await page.route(
-        url,
-        (route: { request(): { headers(): Record<string, string> }; continue(o: object): Promise<void> }) =>
-          route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }),
-      )
+    if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") {
+      await page.route(url, (route: RouteLike) => {
+        route.continue(routeContinueOverrides(route, extraHeaders, method, body))
+      })
     }
 
     let statusCode = 200

packages/tiers/src/tier4.ts

@@ -4,6 +4,8 @@ import type { Cookie, TierResult } from "@trawl/types"
 import { waitForChallengeResolution } from "./challengeWait"
 import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
 import { normalizeHtml } from "./html"
+import type { RouteLike } from "./sanitize"
+import { routeContinueOverrides } from "./sanitize"
 import { waitForImpervaResolution } from "./impervaWait"
 
 export interface Tier4Result extends TierResult {
@@ -20,6 +22,8 @@ export async function runTier4(
   maxTimeout: number,
   proxyUrl: string,
   extraHeaders?: Record<string, string>,
+  method?: string,
+  body?: string,
 ): Promise<Tier4Result> {
   const start = Date.now()
 
@@ -56,12 +60,10 @@ export async function runTier4(
 
     const page = await proxyContext.newPage()
 
-    if (extraHeaders && Object.keys(extraHeaders).length > 0) {
-      await page.route(
-        url,
-        (route: { request(): { headers(): Record<string, string> }; continue(o: object): Promise<void> }) =>
-          route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }),
-      )
+    if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") {
+      await page.route(url, (route: RouteLike) => {
+        route.continue(routeContinueOverrides(route, extraHeaders, method, body))
+      })
     }
 
     let statusCode = 200

packages/tiers/tests/runTier1Post.test.ts

@@ -0,0 +1,122 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test"
+import { runTier1 } from "../src/tier1"
+
+interface RecordedCall {
+  url: string
+  init: RequestInit | undefined
+}
+
+const recorded: RecordedCall[] = []
+
+const installFetchMock = (
+  responder: (req: RecordedCall) => Response = () => {
+    return new Response("<html>OK</html>", {
+      status: 200,
+      headers: { "content-type": "text/html" },
+    })
+  },
+) => {
+  const originalFetch = globalThis.fetch
+  ;(globalThis as { fetch: typeof fetch }).fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url
+    const call: RecordedCall = { url, init }
+    recorded.push(call)
+    return responder(call)
+  }) as typeof fetch
+  return () => {
+    ;(globalThis as { fetch: typeof fetch }).fetch = originalFetch
+  }
+}
+
+beforeEach(() => {
+  recorded.length = 0
+})
+
+afterEach(() => {
+  // Per-test teardown is handled by the returned `restore` closure in each test.
+})
+
+describe("runTier1 — POST support", () => {
+  test("uses GET with no body when method is omitted", async () => {
+    const restore = installFetchMock()
+    try {
+      const result = await runTier1("https://example.com/x")
+      expect(result.status).toBe("success")
+      expect(recorded).toHaveLength(1)
+      expect(recorded[0].url).toBe("https://example.com/x")
+      expect(recorded[0].init?.method).toBe("GET")
+      expect(recorded[0].init?.body).toBeUndefined()
+    } finally {
+      restore()
+    }
+  })
+
+  test("forwards method=POST and the body string to fetch", async () => {
+    const restore = installFetchMock()
+    try {
+      const headers = { "Content-Type": "application/x-www-form-urlencoded" }
+      const result = await runTier1("https://example.com/login", headers, "POST", "user=a&pw=b")
+      expect(result.status).toBe("success")
+      expect(recorded).toHaveLength(1)
+      expect(recorded[0].url).toBe("https://example.com/login")
+      expect(recorded[0].init?.method).toBe("POST")
+      expect(recorded[0].init?.body).toBe("user=a&pw=b")
+      // Caller-supplied Content-Type must be passed through untouched — no
+      // auto-injection at the tier level.
+      const h = recorded[0].init?.headers as Record<string, string>
+      expect(h?.["Content-Type"] ?? h?.["content-type"]).toBe("application/x-www-form-urlencoded")
+    } finally {
+      restore()
+    }
+  })
+
+  test("explicit method=GET still produces no body even when a body string is given", async () => {
+    const restore = installFetchMock()
+    try {
+      await runTier1("https://example.com/x", undefined, "GET", "ignored=by-design")
+      expect(recorded).toHaveLength(1)
+      expect(recorded[0].init?.method).toBe("GET")
+      // Spec: `method === "POST" ? body : undefined` — GET + body is ignored.
+      expect(recorded[0].init?.body).toBeUndefined()
+    } finally {
+      restore()
+    }
+  })
+
+  test("caller headers are spread LAST and therefore can override Fingerprint defaults — the reserved-name denylist at the orchestrator level is what prevents UA spoofing in production", async () => {
+    const restore = installFetchMock()
+    try {
+      // This demonstrates the tier's pass-through behaviour: a non-reserved
+      // header does override. Reserved headers are stripped upstream by
+      // sanitizeHeaders(); this test pins both halves of the contract.
+      const { sanitizeHeaders } = await import("../src/sanitize")
+      const cleaned = sanitizeHeaders({ "User-Agent": "evil-spider/1.0", Accept: "application/json" })
+      expect(cleaned).toEqual({ Accept: "application/json" }) // UA was reserved, dropped
+
+      await runTier1("https://example.com/x", cleaned)
+      const h = recorded[0].init?.headers as Record<string, string>
+      // After sanitisation, only Accept survived — so tier1's FINGERPRINT UA wins.
+      expect(h?.["User-Agent"] ?? h?.["user-agent"]).not.toBe("evil-spider/1.0")
+      expect(h?.["User-Agent"] ?? h?.["user-agent"]).toBeTruthy()
+    } finally {
+      restore()
+    }
+  })
+
+  test("non-2xx, non-CF response surfaces as blocked", async () => {
+    const restore = installFetchMock(
+      () =>
+        new Response("<html>nope</html>", {
+          status: 403,
+          headers: { "content-type": "text/html" },
+        }),
+    )
+    try {
+      const result = await runTier1("https://example.com/x")
+      expect(result.status).toBe("blocked")
+      expect((result as { reason: string }).reason).toBe("http-403")
+    } finally {
+      restore()
+    }
+  })
+})

packages/tiers/tests/sanitize.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, test } from "bun:test"
+import {
+  isValidMethod,
+  RESERVED_HEADER_NAMES,
+  RequestValidationError,
+  requireContentTypeForBody,
+  sanitizeHeaders,
+} from "../src/sanitize"
+
+describe("sanitizeHeaders", () => {
+  test("returns undefined for undefined / empty input", () => {
+    expect(sanitizeHeaders(undefined)).toBeUndefined()
+    expect(sanitizeHeaders({})).toBeUndefined()
+  })
+
+  test("passes through ordinary non-reserved headers unchanged", () => {
+    const out = sanitizeHeaders({
+      Accept: "application/json",
+      "X-Custom-Header": "ok",
+      Referer: "https://example.com/a",
+    })
+    expect(out).toEqual({
+      Accept: "application/json",
+      "X-Custom-Header": "ok",
+      Referer: "https://example.com/a",
+    })
+  })
+
+  test("drops case-insensitive reserved headers (Host, Cookie, Authorization, ...)", () => {
+    const out = sanitizeHeaders({
+      host: "evil.example",
+      Host: "evil.example",
+      HOST: "evil.example",
+      cookie: "session=secret",
+      Cookie: "session=secret",
+      authorization: "Bearer x",
+      Authorization: "Bearer x",
+      "x-forwarded-for": "1.2.3.4",
+      "X-Forwarded-For": "1.2.3.4",
+      "cf-connecting-ip": "1.2.3.4",
+      "user-agent": "evil-ua",
+      "sec-fetch-mode": "no-cors",
+      Accept: "application/json",
+    })
+    expect(out).toEqual({ Accept: "application/json" })
+  })
+
+  test("strips NUL / CR / LF characters from header values", () => {
+    const out = sanitizeHeaders({
+      "X-Evil": "abc\x00def",
+      "X-Multi": "a\r\nb",
+      Accept: "application/json",
+    })
+    expect(out).toEqual({
+      "X-Evil": "abcdef",
+      "X-Multi": "ab",
+      Accept: "application/json",
+    })
+  })
+
+  test("drops empty-string values after sanitisation", () => {
+    const out = sanitizeHeaders({
+      "X-Empty": "\x00",
+      Accept: "application/json",
+    })
+    expect(out).toEqual({ Accept: "application/json" })
+  })
+
+  test("preserves the set of reserved names as a frozen allowlist", () => {
+    expect(RESERVED_HEADER_NAMES.has("host")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("cookie")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("authorization")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("x-forwarded-for")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("cf-connecting-ip")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("sec-ch-ua")).toBe(true)
+    expect(RESERVED_HEADER_NAMES.has("not-a-reserved-name")).toBe(false)
+  })
+})
+
+describe("isValidMethod", () => {
+  test("accepts undefined (default = GET) and the full standard verb set", () => {
+    expect(isValidMethod(undefined)).toBe(true)
+    for (const m of ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "TRACE"]) {
+      expect(isValidMethod(m)).toBe(true)
+    }
+  })
+
+  test("rejects non-standard verbs and unknown strings", () => {
+    expect(isValidMethod("CONNECT")).toBe(false) // tunneling verb — intentionally excluded
+    expect(isValidMethod("FOOBAR")).toBe(false)
+    expect(isValidMethod("get")).toBe(false) // case-sensitive
+    expect(isValidMethod("post")).toBe(false)
+    expect(isValidMethod("")).toBe(false)
+    expect(isValidMethod(null)).toBe(false)
+    expect(isValidMethod(123)).toBe(false)
+  })
+})
+
+describe("requireContentTypeForBody", () => {
+  test("does nothing when there is no body", () => {
+    expect(() => requireContentTypeForBody(undefined, false)).not.toThrow()
+    expect(() => requireContentTypeForBody({}, false)).not.toThrow()
+  })
+
+  test("throws 400 when body present but no headers at all", () => {
+    expect(() => requireContentTypeForBody(undefined, true)).toThrow(RequestValidationError)
+  })
+
+  test("throws 400 when Content-Type header is missing", () => {
+    expect(() => requireContentTypeForBody({ Accept: "application/json" }, true)).toThrow(RequestValidationError)
+  })
+
+  test("accepts explicit Content-Type regardless of letter case", () => {
+    expect(() => requireContentTypeForBody({ "Content-Type": "application/json" }, true)).not.toThrow()
+    expect(() => requireContentTypeForBody({ "content-type": "application/json" }, true)).not.toThrow()
+    expect(() => requireContentTypeForBody({ "CONTENT-TYPE": "application/x-www-form-urlencoded" }, true)).not.toThrow()
+  })
+
+  test("rejects whitespace-only Content-Type", () => {
+    expect(() => requireContentTypeForBody({ "Content-Type": "   " }, true)).toThrow(RequestValidationError)
+  })
+})

packages/types/src/index.ts

@@ -16,6 +16,11 @@ export interface ScrapeRequest {
   maxTier?: 1 | 2 | 3 | 4
   sessionId?: string
   headers?: Record<string, string>
+  // CONNECT is intentionally excluded — it's a tunneling verb, not a normal
+  // request body, and would let a caller establish arbitrary TCP tunnels.
+  // QUERY (RFC 9341) is included — safe verb, body carries the query params.
+  method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "TRACE" | "QUERY"
+  body?: string
   // Per-request proxy override — bypasses the server-configured proxy pool for this call.
   proxy?: string
 }

scripts/e2e-probe.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# E2E probe suite for TRAWL — covers every supported verb, sanitisation,
+# validation, and the legacy FlareSolverr shape. Run while the container is up.
+set -uo pipefail
+BASE="${BASE:-http://localhost:8191}"
+ok=0
+fail=0
+check() {
+  local name="$1" expected="$2" actual="$3"
+  if [[ "$actual" == "$expected" ]]; then
+    printf "  \033[32mOK\033[0m   %-60s -> %s\n" "$name" "$actual"
+    ok=$((ok+1))
+  else
+    printf "  \033[31mFAIL\033[0m %-60s -> got %s, want %s\n" "$name" "$actual" "$expected"
+    fail=$((fail+1))
+  fi
+}
+
+probe() {
+  local name="$1" expected="$2"; shift 2
+  local code
+  code=$(curl -s -o /tmp/probe-body -w "%{http_code}" "$@")
+  check "$name" "$expected" "$code"
+  if [[ "$code" != "$expected" ]]; then
+    echo "       body: $(head -c 300 /tmp/probe-body)"
+  fi
+}
+
+echo "=== 1. health & stats ==="
+probe "/health -> 200"                       200 -X GET "$BASE/health"
+probe "/stats -> 200"                        200 -X GET "$BASE/stats"
+
+echo
+echo "=== 2. native /scrape — full verb set (upstream method in JSON body) ==="
+# /scrape is registered as POST at the HTTP layer. The upstream HTTP verb the
+# scraper uses is passed inside the JSON body (`method` field). All upstream
+# verbs except GET must declare Content-Type upstream-side via `headers`.
+for verb in GET POST PUT PATCH DELETE HEAD OPTIONS TRACE QUERY; do
+  case "$verb" in
+    GET|HEAD|OPTIONS|TRACE)
+      payload=$(printf '{"url":"https://example.com/","method":"%s"}' "$verb")
+      expected=200
+      ;;
+    *)
+      payload=$(printf '{"url":"https://example.com/","method":"%s","body":"k=v","headers":{"Content-Type":"application/x-www-form-urlencoded"}}' "$verb")
+      expected=200
+      ;;
+  esac
+  probe "/scrape upstream-method=$verb -> $expected" "$expected" \
+    -X POST "$BASE/scrape" \
+    -H 'content-type: application/json' \
+    --data "$payload"
+done
+
+echo
+echo "=== 3. native /scrape — negative ==="
+probe "/scrape CONNECT -> 400" 400 \
+  -X POST "$BASE/scrape" \
+  -H 'content-type: application/json' \
+  --data '{"url":"https://example.com/","method":"CONNECT"}'
+probe "/scrape POST no Content-Type -> 400" 400 \
+  -X POST "$BASE/scrape" \
+  -H 'content-type: application/json' \
+  --data '{"url":"https://example.com/","method":"POST","body":"k=v","headers":{}}'
+probe "/scrape Host header (sanitised) -> 200" 200 \
+  -X POST "$BASE/scrape" \
+  -H 'content-type: application/json' \
+  --data '{"url":"https://example.com/","headers":{"Host":"evil.example","X-Custom":"keep-me"}}'
+
+echo
+echo "=== 4. legacy /v1 (FlareSolverr compat) ==="
+probe "/v1 request.get -> 200" 200 \
+  -X POST "$BASE/v1" \
+  -H 'content-type: application/json' \
+  --data '{"cmd":"request.get","url":"https://example.com/"}'
+probe "/v1 request.post + Content-Type -> 200" 200 \
+  -X POST "$BASE/v1" \
+  -H 'content-type: application/json' \
+  --data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{"Content-Type":"application/x-www-form-urlencoded"}}'
+probe "/v1 request.post no Content-Type -> 400" 400 \
+  -X POST "$BASE/v1" \
+  -H 'content-type: application/json' \
+  --data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{}}'
+probe "/v1 unknown cmd -> 400" 400 \
+  -X POST "$BASE/v1" \
+  -H 'content-type: application/json' \
+  --data '{"cmd":"request.delete","url":"https://example.com/"}'
+
+echo
+echo "=== 5. large body — no cap (~500 KiB) ==="
+big=$(python3 -c "import string,random; random.seed(0); print(''.join(random.choices(string.ascii_letters+string.digits,k=512000)),end='')")
+probe "/scrape POST large body (uncapped) -> 200" 200 \
+  -X POST "$BASE/scrape" \
+  -H 'content-type: application/json' \
+  --data "$(python3 -c "import json,sys; print(json.dumps({'url':'https://example.com/','method':'POST','body':sys.argv[1],'headers':{'Content-Type':'text/plain'}}))" "$big")"
+
+echo
+echo "-----------------------------------------------------"
+echo "Total: $ok passed, $fail failed"
+[[ $fail -eq 0 ]] || exit 1