mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
feat(api): validate methods and sanitise headers at the boundary
This commit is contained in:
+44
-12
@@ -1,5 +1,13 @@
|
||||
import { BrowserPool, PoolExhaustedError, SessionCache } from "@trawl/browser"
|
||||
import { scrape } from "@trawl/tiers"
|
||||
import {
|
||||
isValidMethod,
|
||||
RequestValidationError,
|
||||
requireContentTypeForBody,
|
||||
SUPPORTED_METHODS,
|
||||
type SupportedMethod,
|
||||
sanitizeHeaders,
|
||||
scrape,
|
||||
} from "@trawl/tiers"
|
||||
import type { FlareSolverrRequest, FlareSolverrResponse, PoolStats, ScrapeRequest } from "@trawl/types"
|
||||
import { Elysia } from "elysia"
|
||||
|
||||
@@ -49,6 +57,29 @@ function getDeps() {
|
||||
}
|
||||
}
|
||||
|
||||
function buildScrapeRequestFromFlareSolverr(req: FlareSolverrRequest): ScrapeRequest {
|
||||
const method: SupportedMethod = req.cmd === "request.post" ? "POST" : "GET"
|
||||
const headers = sanitizeHeaders(req.headers)
|
||||
requireContentTypeForBody(headers, Boolean(req.postData))
|
||||
return {
|
||||
url: req.url,
|
||||
maxTimeout: req.maxTimeout ?? 60_000,
|
||||
headers,
|
||||
method,
|
||||
body: req.postData,
|
||||
}
|
||||
}
|
||||
|
||||
function validateScrapeRequest(req: ScrapeRequest): void {
|
||||
if (!isValidMethod(req.method)) {
|
||||
throw new RequestValidationError(
|
||||
`Unsupported method: ${String(req.method)} (allowed: ${SUPPORTED_METHODS.join(", ")})`,
|
||||
400,
|
||||
)
|
||||
}
|
||||
requireContentTypeForBody(sanitizeHeaders(req.headers), Boolean(req.body))
|
||||
}
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
// Build a FlareSolverr v2-shaped error envelope. Used by /v1 for every error
|
||||
@@ -124,16 +155,8 @@ new Elysia()
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await scrape(
|
||||
{
|
||||
url: req.url,
|
||||
maxTimeout: req.maxTimeout ?? 60_000,
|
||||
headers: req.headers,
|
||||
method: cmd === "request.post" ? "POST" : "GET",
|
||||
postData: req.postData,
|
||||
},
|
||||
getDeps(),
|
||||
)
|
||||
const scrapeRequest = buildScrapeRequestFromFlareSolverr(req)
|
||||
const result = await scrape(scrapeRequest, getDeps())
|
||||
return {
|
||||
status: "ok",
|
||||
message: "",
|
||||
@@ -150,6 +173,10 @@ new Elysia()
|
||||
},
|
||||
} satisfies FlareSolverrResponse
|
||||
} catch (err) {
|
||||
if (err instanceof RequestValidationError) {
|
||||
set.status = err.statusCode
|
||||
return flareSolverrError(req.url, err.message)
|
||||
}
|
||||
set.status = err instanceof PoolExhaustedError ? 429 : 500
|
||||
return flareSolverrError(req.url, err instanceof Error ? err.message : String(err))
|
||||
}
|
||||
@@ -167,8 +194,13 @@ new Elysia()
|
||||
}
|
||||
const req = body as ScrapeRequest
|
||||
try {
|
||||
return await scrape(req, getDeps())
|
||||
validateScrapeRequest(req)
|
||||
return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps())
|
||||
} catch (err) {
|
||||
if (err instanceof RequestValidationError) {
|
||||
set.status = err.statusCode
|
||||
return { error: err.message }
|
||||
}
|
||||
if (err instanceof PoolExhaustedError) {
|
||||
set.status = 429
|
||||
return flareSolverrError(req.url ?? "", "Browser pool saturated, retry shortly")
|
||||
|
||||
Reference in New Issue
Block a user