diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4286b..3a982a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- Reject non-object request bodies and missing, non-string, or blank `url` values with HTTP 400 before scraper-tier execution (#34). + ## [1.2.0] - 2026-07-26 ### Added diff --git a/apps/api/src/routes/scrape.ts b/apps/api/src/routes/scrape.ts index cf8d781..9a359f5 100644 --- a/apps/api/src/routes/scrape.ts +++ b/apps/api/src/routes/scrape.ts @@ -4,7 +4,7 @@ import type { ScrapeRequest } from "@trawl/types" import { Elysia } from "elysia" import { flareSolverrError } from "../adapters/flaresolverr" import { getDeps, getPool } from "../deps" -import { validateScrapeRequest } from "../validation" +import { requestUrl, validateScrapeRequest } from "../validation" // Native TRAWL API — richer response (tier, timings, sessionCached). // Error mapping: @@ -13,13 +13,13 @@ import { validateScrapeRequest } from "../validation" // 500 — other scrape exception (native { error }) export function scrapeRoute() { return new Elysia().post("/scrape", async ({ body, set }) => { - if (!getPool()) { - set.status = 503 - return { error: "Browser pool initializing, retry in a few seconds" } - } - const req = body as ScrapeRequest try { - validateScrapeRequest(req) + validateScrapeRequest(body) + const req: ScrapeRequest = body + if (!getPool()) { + set.status = 503 + return { error: "Browser pool initializing, retry in a few seconds" } + } return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps()) } catch (err) { if (err instanceof RequestValidationError) { @@ -28,7 +28,7 @@ export function scrapeRoute() { } if (err instanceof PoolExhaustedError) { set.status = 429 - return flareSolverrError(req.url ?? "", "Browser pool saturated, retry shortly") + return flareSolverrError(requestUrl(body), "Browser pool saturated, retry shortly") } set.status = 500 if (err instanceof ScrapeError) { diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts index f68757f..c1515c3 100644 --- a/apps/api/src/routes/v1.ts +++ b/apps/api/src/routes/v1.ts @@ -4,25 +4,28 @@ import type { FlareSolverrRequest, FlareSolverrResponse } from "@trawl/types" import { Elysia } from "elysia" import { buildScrapeRequestFromFlareSolverr, flareSolverrError } from "../adapters/flaresolverr" import { getDeps, getPool } from "../deps" +import { requestUrl, validateFlareSolverrRequest } from "../validation" // FlareSolverr v2 compat — always open (the v2 spec has no auth header) export function v1Route() { return new Elysia().post("/v1", async ({ body, set }) => { - const req = body as FlareSolverrRequest const startTimestamp = Date.now() - const cmd = req.cmd ?? "request.get" - - if (cmd !== "request.get" && cmd !== "request.post") { - set.status = 400 - return flareSolverrError(req.url, `Unknown cmd: ${cmd}`) - } - - if (!getPool()) { - set.status = 503 - return flareSolverrError(req.url, "Browser pool initializing, retry in a few seconds") - } try { + validateFlareSolverrRequest(body) + const req: FlareSolverrRequest = body + const cmd = req.cmd ?? "request.get" + + if (cmd !== "request.get" && cmd !== "request.post") { + set.status = 400 + return flareSolverrError(req.url, `Unknown cmd: ${cmd}`) + } + + if (!getPool()) { + set.status = 503 + return flareSolverrError(req.url, "Browser pool initializing, retry in a few seconds") + } + const scrapeRequest = buildScrapeRequestFromFlareSolverr(req) const result = await scrape(scrapeRequest, getDeps()) return { @@ -43,10 +46,10 @@ export function v1Route() { } catch (err) { if (err instanceof RequestValidationError) { set.status = err.statusCode - return flareSolverrError(req.url, err.message) + return flareSolverrError(requestUrl(body), err.message) } set.status = err instanceof PoolExhaustedError ? 429 : 500 - return flareSolverrError(req.url, err instanceof Error ? err.message : String(err)) + return flareSolverrError(requestUrl(body), err instanceof Error ? err.message : String(err)) } }) } diff --git a/apps/api/src/validation.test.ts b/apps/api/src/validation.test.ts new file mode 100644 index 0000000..8c0df72 --- /dev/null +++ b/apps/api/src/validation.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test" +import { RequestValidationError } from "@trawl/tiers" +import { requestUrl, validateFlareSolverrRequest, validateScrapeRequest } from "./validation" + +const invalidBodies: unknown[] = [undefined, null, [], "text", 42, true] + +describe("API request validation", () => { + for (const body of invalidBodies) { + test(`rejects non-object body ${JSON.stringify(body)}`, () => { + expect(() => validateFlareSolverrRequest(body)).toThrow( + new RequestValidationError("Request body must be a JSON object", 400), + ) + expect(() => validateScrapeRequest(body)).toThrow( + new RequestValidationError("Request body must be a JSON object", 400), + ) + }) + } + + for (const url of [undefined, null, "", " ", 42]) { + test(`rejects invalid url ${JSON.stringify(url)}`, () => { + expect(() => validateFlareSolverrRequest({ url })).toThrow( + new RequestValidationError("url must be a non-empty string", 400), + ) + expect(() => validateScrapeRequest({ url })).toThrow( + new RequestValidationError("url must be a non-empty string", 400), + ) + }) + } + + test("accepts valid request bodies for both API shapes", () => { + expect(() => validateFlareSolverrRequest({ cmd: "request.get", url: "https://example.com" })).not.toThrow() + expect(() => validateScrapeRequest({ method: "GET", url: "https://example.com" })).not.toThrow() + }) + + test("extracts only string URLs for error envelopes", () => { + expect(requestUrl({ url: "https://example.com" })).toBe("https://example.com") + expect(requestUrl({ url: 42 })).toBe("") + expect(requestUrl(null)).toBe("") + }) +}) diff --git a/apps/api/src/validation.ts b/apps/api/src/validation.ts index 1a9dd88..6e796fb 100644 --- a/apps/api/src/validation.ts +++ b/apps/api/src/validation.ts @@ -5,9 +5,37 @@ import { SUPPORTED_METHODS, sanitizeHeaders, } from "@trawl/tiers" -import type { ScrapeRequest } from "@trawl/types" +import type { FlareSolverrRequest, ScrapeRequest } from "@trawl/types" -export function validateScrapeRequest(req: ScrapeRequest): void { +type RequestRecord = Record + +function requireRequestRecord(body: unknown): asserts body is RequestRecord { + if (typeof body !== "object" || body === null || Array.isArray(body)) { + throw new RequestValidationError("Request body must be a JSON object", 400) + } +} + +function requireUrl(req: RequestRecord): void { + if (typeof req.url !== "string" || req.url.trim().length === 0) { + throw new RequestValidationError("url must be a non-empty string", 400) + } +} + +export function requestUrl(body: unknown): string { + if (typeof body !== "object" || body === null || Array.isArray(body)) return "" + const url = (body as RequestRecord).url + return typeof url === "string" ? url : "" +} + +export function validateFlareSolverrRequest(body: unknown): asserts body is FlareSolverrRequest { + requireRequestRecord(body) + requireUrl(body) +} + +export function validateScrapeRequest(body: unknown): asserts body is ScrapeRequest { + requireRequestRecord(body) + requireUrl(body) + const req = body as RequestRecord & Partial if (!isValidMethod(req.method)) { throw new RequestValidationError( `Unsupported method: ${String(req.method)} (allowed: ${SUPPORTED_METHODS.join(", ")})`,