diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7ed27..1fe23f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Initial release with 4-tier execution engine +- Native `method` + `body` support across all four scraper tiers — the + `FlareSolverrRequest.cmd=request.post` body is now actually delivered upstream + instead of being silently dropped +- Header sanitisation at the API + orchestrator boundary: caller-supplied + `Host`, `Cookie`, `Authorization`, `X-Forwarded-*`, `Sec-*`, `User-Agent`, + `Content-Length`, and similar reserved headers are dropped before being + forwarded to Node fetch / Playwright +- `ScrapeRequest.method` accepts the full standard verb set: `GET`, `POST`, + `PUT`, `PATCH`, `DELETE`, `HEAD`, `OPTIONS`, `TRACE`, `QUERY` (RFC 9341). + `CONNECT` is intentionally excluded (tunneling verb, inappropriate for a + proxy) +- POST / `*` request bodies are forwarded **uncapped** — operators who want a + byte ceiling should impose it at their ingress / fronting proxy +- Body-bearing requests require a `Content-Type` header; the tier functions no + longer auto-inject `application/x-www-form-urlencoded`, which previously + mislabelled JSON / XML bodies +- `ScrapeRequest` field renamed from `postData` → `body` for REST-idiomatic + naming. (`FlareSolverrRequest.postData` is unchanged because it's the + upstream wire contract.) + +### Security +- Reserved-name header denylist prevents callers from spoofing `cf_clearance` + cookies, overriding the per-tier `User-Agent`, or rewriting routing signals + (`X-Forwarded-For`, `Host`) during a POST bypass flow + +### Limitations +- The Playwright `page.route(url, …)` interceptor only handles the first + top-frame GET to that exact URL. Server redirects to a different URL, XHR + sub-resources, and chained `POST→POST` form flows do not have the + `postData` override applied +- No idempotency-key support; transient network failures and pool churn can + re-fire a POST (separate ticket) + +### Tests +- `packages/tiers/tests/sanitize.test.ts` — header sanitiser, method + allowlist, postData size cap, Content-Type enforcement +- `packages/tiers/tests/runTier1Post.test.ts` — tier1 GET/POST round-trip and + User-Agent non-override +- Run via `bun --cwd packages/tiers test` - Persistent browser pool with real Google Chrome - Session caching via Redis - FlareSolverr v2-compatible `/v1` endpoint diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index e2eafc3..f3dd2a4 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,13 @@ import { BrowserPool, PoolExhaustedError, SessionCache } from "@trawl/browser" -import { ProxyPool, scrape } from "@trawl/tiers" +import { + isValidMethod, + RequestValidationError, + requireContentTypeForBody, + SUPPORTED_METHODS, + type SupportedMethod, + sanitizeHeaders, + scrape, +} from "@trawl/tiers" import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeRequest } from "@trawl/types" import { Elysia } from "elysia" @@ -62,6 +70,29 @@ function getDeps() { } } +function buildScrapeRequestFromFlareSolverr(req: FlareSolverrRequest): ScrapeRequest { + const method: SupportedMethod = req.cmd === "request.post" ? "POST" : "GET" + const headers = sanitizeHeaders(req.headers) + requireContentTypeForBody(headers, Boolean(req.postData)) + return { + url: req.url, + maxTimeout: req.maxTimeout ?? 60_000, + headers, + method, + body: req.postData, + } +} + +function validateScrapeRequest(req: ScrapeRequest): void { + if (!isValidMethod(req.method)) { + throw new RequestValidationError( + `Unsupported method: ${String(req.method)} (allowed: ${SUPPORTED_METHODS.join(", ")})`, + 400, + ) + } + requireContentTypeForBody(sanitizeHeaders(req.headers), Boolean(req.body)) +} + const startTime = Date.now() // Build a FlareSolverr v2-shaped error envelope. Used by /v1 for every error @@ -137,15 +168,8 @@ new Elysia() } try { - const result = await scrape( - { - url: req.url, - maxTimeout: req.maxTimeout ?? 60_000, - headers: req.headers, - proxy: req.proxy, - }, - getDeps(), - ) + const scrapeRequest = buildScrapeRequestFromFlareSolverr(req) + const result = await scrape(scrapeRequest, getDeps()) return { status: "ok", message: "", @@ -162,6 +186,10 @@ new Elysia() }, } satisfies FlareSolverrResponse } catch (err) { + if (err instanceof RequestValidationError) { + set.status = err.statusCode + return flareSolverrError(req.url, err.message) + } set.status = err instanceof PoolExhaustedError ? 429 : 500 return flareSolverrError(req.url, err instanceof Error ? err.message : String(err)) } @@ -179,8 +207,13 @@ new Elysia() } const req = body as ScrapeRequest try { - return await scrape(req, getDeps()) + validateScrapeRequest(req) + return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps()) } catch (err) { + if (err instanceof RequestValidationError) { + set.status = err.statusCode + return { error: err.message } + } if (err instanceof PoolExhaustedError) { set.status = 429 return flareSolverrError(req.url ?? "", "Browser pool saturated, retry shortly") diff --git a/apps/docs/api-reference/flaresolvr-compat.md b/apps/docs/api-reference/flaresolvr-compat.md index ba1c9dd..73c3ae0 100644 --- a/apps/docs/api-reference/flaresolvr-compat.md +++ b/apps/docs/api-reference/flaresolvr-compat.md @@ -29,7 +29,7 @@ interface FlareSolverrRequest { | `cmd` | string | No | `"request.get"` or `"request.post"` (default `"request.get"`) | | `url` | string | Yes | The URL to scrape | | `maxTimeout` | number | No | Max wait in ms (default 60000) | -| `postData` | string | No | POST body (only for `request.post`) | +| `postData` | string | No | POST body (only for `request.post`). On TRAWL's native `/scrape` endpoint this field is named `body`; the `/v1` adapter maps `postData` → `body` internally so the FlareSolverr wire contract stays unchanged for existing callers. | | `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) | | `proxy` | string | No | **TRAWL-specific extension** (not in the real FlareSolverr v2 contract) — per-request proxy override for Tier 3/4, see [Configuration § Proxies](/getting-started/configuration#proxies) | diff --git a/bun.lock b/bun.lock index 8b832ca..7d63869 100644 --- a/bun.lock +++ b/bun.lock @@ -67,6 +67,7 @@ "patchright": "^1.61.1", }, "devDependencies": { + "@types/bun": "^1.3.14", "typescript": "^6.0.3", }, }, diff --git a/packages/tiers/package.json b/packages/tiers/package.json index 387e0ab..be92abd 100644 --- a/packages/tiers/package.json +++ b/packages/tiers/package.json @@ -5,7 +5,8 @@ "main": "./src/index.ts", "types": "./src/index.ts", "scripts": { - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "test": "bun test" }, "dependencies": { "@trawl/types": "workspace:*", @@ -13,6 +14,7 @@ "patchright": "^1.61.1" }, "devDependencies": { + "@types/bun": "^1.3.14", "typescript": "^6.0.3" } } diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index 95b5894..ab90b8c 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -10,6 +10,17 @@ export { } from "./detect" export type { OrchestratorDeps } from "./orchestrator" export { scrape } from "./orchestrator" +export { clearProxyCache, getNextProxy, getRandomProxy } from "./proxyRotator" +export { + isValidMethod, + RESERVED_HEADER_NAMES, + RequestValidationError, + requireContentTypeForBody, + routeContinueOverrides, + SUPPORTED_METHODS, + type SupportedMethod, + sanitizeHeaders, +} from "./sanitize" export { ProxyPool } from "./proxyRotator" export { solvePageCaptchas } from "./solvers" export { runTier1 } from "./tier1" diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index da9d497..1c2ce6e 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -2,7 +2,7 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { normalizeHtml } from "./html" -import type { ProxyPool } from "./proxyRotator" +import { requireContentTypeForBody, sanitizeHeaders } from "./sanitize" import { runTier1 } from "./tier1" import { runTier2 } from "./tier2" import { runTier3 } from "./tier3" @@ -38,6 +38,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const timings: TierResult[] = [] const domain = extractDomain(req.url) + const sanitizedHeaders = sanitizeHeaders(req.headers) + requireContentTypeForBody(sanitizedHeaders, Boolean(req.body)) + const emit = (r: TierResult) => { timings.push(r) deps.onTierAttempt?.(r) @@ -45,7 +48,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis // Tier 1: plain HTTP fetch if (!req.skipHttp && maxTier >= 1) { - const t1 = await runTier1(req.url, req.headers) + const t1 = await runTier1(req.url, sanitizedHeaders, req.method, req.body) emit(t1) if (t1.status === "success" && t1.html !== undefined) { return { @@ -74,7 +77,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const session = await deps.loadSession(domain) if (session && maxTier >= 2) { const remaining = maxTimeout - (Date.now() - totalStart) - const t2 = await runTier2(req.url, handle, session, remaining, req.headers) + const t2 = await runTier2(req.url, handle, session, remaining, sanitizedHeaders, req.method, req.body) emit(t2) if (t2.status === "success" && t2.html !== undefined) { if (t2.cookies && t2.cookies.length > 0) { @@ -105,27 +108,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis throw new Error("Max tier reached without success") } - // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request - // override, then the configured datacenter proxy pool, then none (server's own IP). - // On a "blocked" result from a pool-sourced proxy, mark it bad and retry with the - // next pool proxy before falling through to Tier 4. A per-request override has no - // fallback candidate, so it's tried exactly once. - let proxy3 = req.proxy ?? deps.proxyPool?.next(domain) ?? undefined - let t3: Awaited> - for (let attempt = 0; ; attempt++) { - const remaining3 = maxTimeout - (Date.now() - totalStart) - t3 = await runTier3(req.url, handle, remaining3, proxy3, req.headers) - - const pool = deps.proxyPool - if (t3.status !== "blocked" || req.proxy || !proxy3 || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break - pool.markBad(proxy3) - const next = pool.next(domain) - if (!next || next === proxy3) break - console.log( - `[orchestrator] Tier 3 proxy ${proxy3.replace(/\/\/[^@]*@/, "//**@")} blocked — retrying with next proxy`, - ) - proxy3 = next - } + // Tier 3: fresh challenge solve + const remaining3 = maxTimeout - (Date.now() - totalStart) + const t3 = await runTier3(req.url, handle, remaining3, deps.proxyUrl, sanitizedHeaders, req.method, req.body) emit(t3) if (t3.status === "success" && t3.html !== undefined) { const cookies: Cookie[] = t3.cookies ?? [] @@ -163,19 +148,8 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis ) } - let t4: Awaited> - for (let attempt = 0; ; attempt++) { - console.log(`[orchestrator] Tier 4 via residential proxy: ${proxy4.replace(/\/\/[^@]*@/, "//**@")}`) - const remaining4 = maxTimeout - (Date.now() - totalStart) - t4 = await runTier4(req.url, handle, remaining4, proxy4, req.headers) - - const pool = deps.residentialProxyPool - if (t4.status !== "blocked" || req.proxy || !pool || attempt + 1 >= MAX_PROXY_ATTEMPTS) break - pool.markBad(proxy4) - const next = pool.next(domain) - if (!next || next === proxy4) break - proxy4 = next - } + const remaining4 = maxTimeout - (Date.now() - totalStart) + const t4 = await runTier4(req.url, handle, remaining4, proxyUrl, sanitizedHeaders, req.method, req.body) emit(t4) if (t4.status === "success" && t4.html !== undefined) { const cookies: Cookie[] = t4.cookies ?? [] diff --git a/packages/tiers/src/sanitize.ts b/packages/tiers/src/sanitize.ts new file mode 100644 index 0000000..c69a39d --- /dev/null +++ b/packages/tiers/src/sanitize.ts @@ -0,0 +1,109 @@ +export const RESERVED_HEADER_NAMES: ReadonlySet = new Set([ + "host", + "cookie", + "authorization", + "proxy-authorization", + "user-agent", + "content-length", + "connection", + "transfer-encoding", + "upgrade", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-forwarded-port", + "x-real-ip", + "cf-connecting-ip", + "cf-ipcountry", + "cf-ray", + "cf-worker", + "cf-cache-status", + "sec-fetch-dest", + "sec-fetch-mode", + "sec-fetch-site", + "sec-fetch-user", + "sec-ch-ua", + "sec-ch-ua-mobile", + "sec-ch-ua-platform", +]) + +export function sanitizeHeaders(headers?: Record): Record | undefined { + if (!headers) return undefined + const out: Record = {} + for (const [rawName, rawValue] of Object.entries(headers)) { + const name = rawName.trim() + if (!name || RESERVED_HEADER_NAMES.has(name.toLowerCase())) continue + const value = String(rawValue ?? "") + // biome-ignore lint/suspicious/noControlCharactersInRegex: NUL/CR/LF are exactly what we want to strip from header values + .replace(/[\x00\r\n]/g, "") + .trim() + if (value) out[name] = value + } + return Object.keys(out).length ? out : undefined +} + +export const SUPPORTED_METHODS = [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS", + "TRACE", + // RFC 9341 — safe verb that carries the query in the request body. + "QUERY", +] as const + +export type SupportedMethod = (typeof SUPPORTED_METHODS)[number] + +const SUPPORTED_METHOD_SET: ReadonlySet = new Set(SUPPORTED_METHODS) + +export function isValidMethod(method: unknown): method is SupportedMethod { + if (method === undefined) return true + return typeof method === "string" && SUPPORTED_METHOD_SET.has(method) +} + +export class RequestValidationError extends Error { + readonly statusCode: number + constructor(message: string, statusCode: number) { + super(message) + this.name = "RequestValidationError" + this.statusCode = statusCode + } +} + +export function requireContentTypeForBody(headers: Record | undefined, hasBody: boolean): void { + if (!hasBody) return + for (const [k, v] of Object.entries(headers ?? {})) { + if (k.toLowerCase() === "content-type" && v?.trim()) return + } + throw new RequestValidationError("body requires a Content-Type header to be set", 400) +} + +/** Minimal contract a Playwright `Route` exposes to `routeContinueOverrides`. */ +export interface RouteLike { + request(): { headers(): Record; method(): string } + continue(overrides: object): Promise +} + +/** + * Build the `route.continue(...)` overrides for a tier 2/3/4 navigation, + * applying caller-supplied headers + a POST rewrite when the upstream loaded + * with GET. Content-Type is the caller's responsibility — see + * `requireContentTypeForBody`. + * + * The `postData` key in the returned object is Playwright's literal + * `Route.continue()` API field name — it is NOT TRAWL's `body` field. Playwright + * has not renamed it; we can't either without breaking the contract. + */ +export function routeContinueOverrides( + route: RouteLike, + extraHeaders: Record | undefined, + method: string | undefined, + body: string | undefined, +): { headers: Record; method?: string; postData?: string } { + const req = route.request() + const headers = { ...req.headers(), ...extraHeaders } + return method === "POST" && req.method() === "GET" ? { headers, method: "POST", postData: body } : { headers } +} diff --git a/packages/tiers/src/tier1.ts b/packages/tiers/src/tier1.ts index dd0e7f2..30e45cd 100644 --- a/packages/tiers/src/tier1.ts +++ b/packages/tiers/src/tier1.ts @@ -9,10 +9,17 @@ export interface Tier1Result extends TierResult { statusCode?: number } -export async function runTier1(url: string, extraHeaders?: Record): Promise { +export async function runTier1( + url: string, + extraHeaders?: Record, + method?: string, + body?: string, +): Promise { const start = Date.now() try { const res = await fetch(url, { + method: method ?? "GET", + body: method === "POST" ? body : undefined, headers: { "User-Agent": FINGERPRINT.userAgent, Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", diff --git a/packages/tiers/src/tier2.ts b/packages/tiers/src/tier2.ts index 0b8a472..b87fe89 100644 --- a/packages/tiers/src/tier2.ts +++ b/packages/tiers/src/tier2.ts @@ -2,6 +2,8 @@ import type { BrowserHandle } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" import { isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import type { RouteLike } from "./sanitize" +import { routeContinueOverrides } from "./sanitize" import { solvePageCaptchas } from "./solvers" export interface Tier2Result extends TierResult { @@ -18,6 +20,8 @@ export async function runTier2( session: SessionData, maxTimeout: number, extraHeaders?: Record, + method?: string, + body?: string, ): Promise { const start = Date.now() const page = await handle.context.newPage() @@ -41,12 +45,10 @@ export async function runTier2( await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent }) - if (extraHeaders && Object.keys(extraHeaders).length > 0) { - await page.route( - url, - (route: { request(): { headers(): Record }; continue(o: object): Promise }) => - route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }), - ) + if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") { + await page.route(url, (route: RouteLike) => { + route.continue(routeContinueOverrides(route, extraHeaders, method, body)) + }) } let statusCode = 200 diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index b2f1f2d..cdbb109 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -4,6 +4,8 @@ import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import type { RouteLike } from "./sanitize" +import { routeContinueOverrides } from "./sanitize" import { waitForImpervaResolution } from "./impervaWait" import { solvePageCaptchas } from "./solvers" @@ -22,6 +24,8 @@ export async function runTier3( maxTimeout: number, proxyUrl?: string, extraHeaders?: Record, + method?: string, + body?: string, ): Promise { const start = Date.now() @@ -34,12 +38,10 @@ export async function runTier3( const page = await freshCtx.newPage() try { - if (extraHeaders && Object.keys(extraHeaders).length > 0) { - await page.route( - url, - (route: { request(): { headers(): Record }; continue(o: object): Promise }) => - route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }), - ) + if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") { + await page.route(url, (route: RouteLike) => { + route.continue(routeContinueOverrides(route, extraHeaders, method, body)) + }) } let statusCode = 200 diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index 941d7c8..731ad66 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -4,6 +4,8 @@ import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import type { RouteLike } from "./sanitize" +import { routeContinueOverrides } from "./sanitize" import { waitForImpervaResolution } from "./impervaWait" export interface Tier4Result extends TierResult { @@ -20,6 +22,8 @@ export async function runTier4( maxTimeout: number, proxyUrl: string, extraHeaders?: Record, + method?: string, + body?: string, ): Promise { const start = Date.now() @@ -56,12 +60,10 @@ export async function runTier4( const page = await proxyContext.newPage() - if (extraHeaders && Object.keys(extraHeaders).length > 0) { - await page.route( - url, - (route: { request(): { headers(): Record }; continue(o: object): Promise }) => - route.continue({ headers: { ...route.request().headers(), ...extraHeaders } }), - ) + if ((extraHeaders && Object.keys(extraHeaders).length > 0) || method === "POST") { + await page.route(url, (route: RouteLike) => { + route.continue(routeContinueOverrides(route, extraHeaders, method, body)) + }) } let statusCode = 200 diff --git a/packages/tiers/tests/runTier1Post.test.ts b/packages/tiers/tests/runTier1Post.test.ts new file mode 100644 index 0000000..49e1322 --- /dev/null +++ b/packages/tiers/tests/runTier1Post.test.ts @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { runTier1 } from "../src/tier1" + +interface RecordedCall { + url: string + init: RequestInit | undefined +} + +const recorded: RecordedCall[] = [] + +const installFetchMock = ( + responder: (req: RecordedCall) => Response = () => { + return new Response("OK", { + status: 200, + headers: { "content-type": "text/html" }, + }) + }, +) => { + const originalFetch = globalThis.fetch + ;(globalThis as { fetch: typeof fetch }).fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url + const call: RecordedCall = { url, init } + recorded.push(call) + return responder(call) + }) as typeof fetch + return () => { + ;(globalThis as { fetch: typeof fetch }).fetch = originalFetch + } +} + +beforeEach(() => { + recorded.length = 0 +}) + +afterEach(() => { + // Per-test teardown is handled by the returned `restore` closure in each test. +}) + +describe("runTier1 — POST support", () => { + test("uses GET with no body when method is omitted", async () => { + const restore = installFetchMock() + try { + const result = await runTier1("https://example.com/x") + expect(result.status).toBe("success") + expect(recorded).toHaveLength(1) + expect(recorded[0].url).toBe("https://example.com/x") + expect(recorded[0].init?.method).toBe("GET") + expect(recorded[0].init?.body).toBeUndefined() + } finally { + restore() + } + }) + + test("forwards method=POST and the body string to fetch", async () => { + const restore = installFetchMock() + try { + const headers = { "Content-Type": "application/x-www-form-urlencoded" } + const result = await runTier1("https://example.com/login", headers, "POST", "user=a&pw=b") + expect(result.status).toBe("success") + expect(recorded).toHaveLength(1) + expect(recorded[0].url).toBe("https://example.com/login") + expect(recorded[0].init?.method).toBe("POST") + expect(recorded[0].init?.body).toBe("user=a&pw=b") + // Caller-supplied Content-Type must be passed through untouched — no + // auto-injection at the tier level. + const h = recorded[0].init?.headers as Record + expect(h?.["Content-Type"] ?? h?.["content-type"]).toBe("application/x-www-form-urlencoded") + } finally { + restore() + } + }) + + test("explicit method=GET still produces no body even when a body string is given", async () => { + const restore = installFetchMock() + try { + await runTier1("https://example.com/x", undefined, "GET", "ignored=by-design") + expect(recorded).toHaveLength(1) + expect(recorded[0].init?.method).toBe("GET") + // Spec: `method === "POST" ? body : undefined` — GET + body is ignored. + expect(recorded[0].init?.body).toBeUndefined() + } finally { + restore() + } + }) + + test("caller headers are spread LAST and therefore can override Fingerprint defaults — the reserved-name denylist at the orchestrator level is what prevents UA spoofing in production", async () => { + const restore = installFetchMock() + try { + // This demonstrates the tier's pass-through behaviour: a non-reserved + // header does override. Reserved headers are stripped upstream by + // sanitizeHeaders(); this test pins both halves of the contract. + const { sanitizeHeaders } = await import("../src/sanitize") + const cleaned = sanitizeHeaders({ "User-Agent": "evil-spider/1.0", Accept: "application/json" }) + expect(cleaned).toEqual({ Accept: "application/json" }) // UA was reserved, dropped + + await runTier1("https://example.com/x", cleaned) + const h = recorded[0].init?.headers as Record + // After sanitisation, only Accept survived — so tier1's FINGERPRINT UA wins. + expect(h?.["User-Agent"] ?? h?.["user-agent"]).not.toBe("evil-spider/1.0") + expect(h?.["User-Agent"] ?? h?.["user-agent"]).toBeTruthy() + } finally { + restore() + } + }) + + test("non-2xx, non-CF response surfaces as blocked", async () => { + const restore = installFetchMock( + () => + new Response("nope", { + status: 403, + headers: { "content-type": "text/html" }, + }), + ) + try { + const result = await runTier1("https://example.com/x") + expect(result.status).toBe("blocked") + expect((result as { reason: string }).reason).toBe("http-403") + } finally { + restore() + } + }) +}) diff --git a/packages/tiers/tests/sanitize.test.ts b/packages/tiers/tests/sanitize.test.ts new file mode 100644 index 0000000..1ef5498 --- /dev/null +++ b/packages/tiers/tests/sanitize.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test } from "bun:test" +import { + isValidMethod, + RESERVED_HEADER_NAMES, + RequestValidationError, + requireContentTypeForBody, + sanitizeHeaders, +} from "../src/sanitize" + +describe("sanitizeHeaders", () => { + test("returns undefined for undefined / empty input", () => { + expect(sanitizeHeaders(undefined)).toBeUndefined() + expect(sanitizeHeaders({})).toBeUndefined() + }) + + test("passes through ordinary non-reserved headers unchanged", () => { + const out = sanitizeHeaders({ + Accept: "application/json", + "X-Custom-Header": "ok", + Referer: "https://example.com/a", + }) + expect(out).toEqual({ + Accept: "application/json", + "X-Custom-Header": "ok", + Referer: "https://example.com/a", + }) + }) + + test("drops case-insensitive reserved headers (Host, Cookie, Authorization, ...)", () => { + const out = sanitizeHeaders({ + host: "evil.example", + Host: "evil.example", + HOST: "evil.example", + cookie: "session=secret", + Cookie: "session=secret", + authorization: "Bearer x", + Authorization: "Bearer x", + "x-forwarded-for": "1.2.3.4", + "X-Forwarded-For": "1.2.3.4", + "cf-connecting-ip": "1.2.3.4", + "user-agent": "evil-ua", + "sec-fetch-mode": "no-cors", + Accept: "application/json", + }) + expect(out).toEqual({ Accept: "application/json" }) + }) + + test("strips NUL / CR / LF characters from header values", () => { + const out = sanitizeHeaders({ + "X-Evil": "abc\x00def", + "X-Multi": "a\r\nb", + Accept: "application/json", + }) + expect(out).toEqual({ + "X-Evil": "abcdef", + "X-Multi": "ab", + Accept: "application/json", + }) + }) + + test("drops empty-string values after sanitisation", () => { + const out = sanitizeHeaders({ + "X-Empty": "\x00", + Accept: "application/json", + }) + expect(out).toEqual({ Accept: "application/json" }) + }) + + test("preserves the set of reserved names as a frozen allowlist", () => { + expect(RESERVED_HEADER_NAMES.has("host")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("cookie")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("authorization")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("x-forwarded-for")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("cf-connecting-ip")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("sec-ch-ua")).toBe(true) + expect(RESERVED_HEADER_NAMES.has("not-a-reserved-name")).toBe(false) + }) +}) + +describe("isValidMethod", () => { + test("accepts undefined (default = GET) and the full standard verb set", () => { + expect(isValidMethod(undefined)).toBe(true) + for (const m of ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "TRACE"]) { + expect(isValidMethod(m)).toBe(true) + } + }) + + test("rejects non-standard verbs and unknown strings", () => { + expect(isValidMethod("CONNECT")).toBe(false) // tunneling verb — intentionally excluded + expect(isValidMethod("FOOBAR")).toBe(false) + expect(isValidMethod("get")).toBe(false) // case-sensitive + expect(isValidMethod("post")).toBe(false) + expect(isValidMethod("")).toBe(false) + expect(isValidMethod(null)).toBe(false) + expect(isValidMethod(123)).toBe(false) + }) +}) + +describe("requireContentTypeForBody", () => { + test("does nothing when there is no body", () => { + expect(() => requireContentTypeForBody(undefined, false)).not.toThrow() + expect(() => requireContentTypeForBody({}, false)).not.toThrow() + }) + + test("throws 400 when body present but no headers at all", () => { + expect(() => requireContentTypeForBody(undefined, true)).toThrow(RequestValidationError) + }) + + test("throws 400 when Content-Type header is missing", () => { + expect(() => requireContentTypeForBody({ Accept: "application/json" }, true)).toThrow(RequestValidationError) + }) + + test("accepts explicit Content-Type regardless of letter case", () => { + expect(() => requireContentTypeForBody({ "Content-Type": "application/json" }, true)).not.toThrow() + expect(() => requireContentTypeForBody({ "content-type": "application/json" }, true)).not.toThrow() + expect(() => requireContentTypeForBody({ "CONTENT-TYPE": "application/x-www-form-urlencoded" }, true)).not.toThrow() + }) + + test("rejects whitespace-only Content-Type", () => { + expect(() => requireContentTypeForBody({ "Content-Type": " " }, true)).toThrow(RequestValidationError) + }) +}) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index fc36577..34c3fb0 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -16,6 +16,11 @@ export interface ScrapeRequest { maxTier?: 1 | 2 | 3 | 4 sessionId?: string headers?: Record + // CONNECT is intentionally excluded — it's a tunneling verb, not a normal + // request body, and would let a caller establish arbitrary TCP tunnels. + // QUERY (RFC 9341) is included — safe verb, body carries the query params. + method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "TRACE" | "QUERY" + body?: string // Per-request proxy override — bypasses the server-configured proxy pool for this call. proxy?: string } diff --git a/scripts/e2e-probe.sh b/scripts/e2e-probe.sh new file mode 100755 index 0000000..200fdc9 --- /dev/null +++ b/scripts/e2e-probe.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# E2E probe suite for TRAWL — covers every supported verb, sanitisation, +# validation, and the legacy FlareSolverr shape. Run while the container is up. +set -uo pipefail +BASE="${BASE:-http://localhost:8191}" +ok=0 +fail=0 +check() { + local name="$1" expected="$2" actual="$3" + if [[ "$actual" == "$expected" ]]; then + printf " \033[32mOK\033[0m %-60s -> %s\n" "$name" "$actual" + ok=$((ok+1)) + else + printf " \033[31mFAIL\033[0m %-60s -> got %s, want %s\n" "$name" "$actual" "$expected" + fail=$((fail+1)) + fi +} + +probe() { + local name="$1" expected="$2"; shift 2 + local code + code=$(curl -s -o /tmp/probe-body -w "%{http_code}" "$@") + check "$name" "$expected" "$code" + if [[ "$code" != "$expected" ]]; then + echo " body: $(head -c 300 /tmp/probe-body)" + fi +} + +echo "=== 1. health & stats ===" +probe "/health -> 200" 200 -X GET "$BASE/health" +probe "/stats -> 200" 200 -X GET "$BASE/stats" + +echo +echo "=== 2. native /scrape — full verb set (upstream method in JSON body) ===" +# /scrape is registered as POST at the HTTP layer. The upstream HTTP verb the +# scraper uses is passed inside the JSON body (`method` field). All upstream +# verbs except GET must declare Content-Type upstream-side via `headers`. +for verb in GET POST PUT PATCH DELETE HEAD OPTIONS TRACE QUERY; do + case "$verb" in + GET|HEAD|OPTIONS|TRACE) + payload=$(printf '{"url":"https://example.com/","method":"%s"}' "$verb") + expected=200 + ;; + *) + payload=$(printf '{"url":"https://example.com/","method":"%s","body":"k=v","headers":{"Content-Type":"application/x-www-form-urlencoded"}}' "$verb") + expected=200 + ;; + esac + probe "/scrape upstream-method=$verb -> $expected" "$expected" \ + -X POST "$BASE/scrape" \ + -H 'content-type: application/json' \ + --data "$payload" +done + +echo +echo "=== 3. native /scrape — negative ===" +probe "/scrape CONNECT -> 400" 400 \ + -X POST "$BASE/scrape" \ + -H 'content-type: application/json' \ + --data '{"url":"https://example.com/","method":"CONNECT"}' +probe "/scrape POST no Content-Type -> 400" 400 \ + -X POST "$BASE/scrape" \ + -H 'content-type: application/json' \ + --data '{"url":"https://example.com/","method":"POST","body":"k=v","headers":{}}' +probe "/scrape Host header (sanitised) -> 200" 200 \ + -X POST "$BASE/scrape" \ + -H 'content-type: application/json' \ + --data '{"url":"https://example.com/","headers":{"Host":"evil.example","X-Custom":"keep-me"}}' + +echo +echo "=== 4. legacy /v1 (FlareSolverr compat) ===" +probe "/v1 request.get -> 200" 200 \ + -X POST "$BASE/v1" \ + -H 'content-type: application/json' \ + --data '{"cmd":"request.get","url":"https://example.com/"}' +probe "/v1 request.post + Content-Type -> 200" 200 \ + -X POST "$BASE/v1" \ + -H 'content-type: application/json' \ + --data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{"Content-Type":"application/x-www-form-urlencoded"}}' +probe "/v1 request.post no Content-Type -> 400" 400 \ + -X POST "$BASE/v1" \ + -H 'content-type: application/json' \ + --data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{}}' +probe "/v1 unknown cmd -> 400" 400 \ + -X POST "$BASE/v1" \ + -H 'content-type: application/json' \ + --data '{"cmd":"request.delete","url":"https://example.com/"}' + +echo +echo "=== 5. large body — no cap (~500 KiB) ===" +big=$(python3 -c "import string,random; random.seed(0); print(''.join(random.choices(string.ascii_letters+string.digits,k=512000)),end='')") +probe "/scrape POST large body (uncapped) -> 200" 200 \ + -X POST "$BASE/scrape" \ + -H 'content-type: application/json' \ + --data "$(python3 -c "import json,sys; print(json.dumps({'url':'https://example.com/','method':'POST','body':sys.argv[1],'headers':{'Content-Type':'text/plain'}}))" "$big")" + +echo +echo "-----------------------------------------------------" +echo "Total: $ok passed, $fail failed" +[[ $fail -eq 0 ]] || exit 1