diff --git a/CHANGELOG.md b/CHANGELOG.md index 5305f14..61f963e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- Translate Prowlarr's serialized `headers.contentType` metadata at the FlareSolverr `/v1` compatibility boundary and discard `contentLength`, allowing form POST requests to enter the scraper pipeline (#50). + ## [1.3.1] - 2026-08-02 ### Fixed diff --git a/apps/api/src/adapters/flaresolverr.test.ts b/apps/api/src/adapters/flaresolverr.test.ts new file mode 100644 index 0000000..e919ed1 --- /dev/null +++ b/apps/api/src/adapters/flaresolverr.test.ts @@ -0,0 +1,93 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { RequestValidationError, routeContinueOverrides, runTier1 } from "@trawl/tiers" +import type { FlareSolverrRequest } from "@trawl/types" +import { buildScrapeRequestFromFlareSolverr } from "./flaresolverr" + +const PROWLARR_2_5_2_REQUEST: FlareSolverrRequest = { + cmd: "request.post", + url: "https://rutracker.org/forum/login.php", + maxTimeout: 60_000, + postData: "login_username=example&login_password=secret&login=%D0%92%D1%85%D0%BE%D0%B4", + headers: { + contentType: "application/x-www-form-urlencoded", + contentLength: "82", + }, +} + +const originalFetch = globalThis.fetch + +afterEach(() => { + ;(globalThis as { fetch: typeof fetch }).fetch = originalFetch +}) + +describe("FlareSolverr request adapter", () => { + test("translates Prowlarr 2.5.2 serialized content metadata", () => { + const result = buildScrapeRequestFromFlareSolverr(PROWLARR_2_5_2_REQUEST) + + expect(result.method).toBe("POST") + expect(result.body).toBe(PROWLARR_2_5_2_REQUEST.postData) + expect(result.headers?.["Content-Type"]).toBe("application/x-www-form-urlencoded") + expect(result.headers?.contentType).toBeUndefined() + expect(result.headers?.contentLength).toBeUndefined() + }) + + test("recognizes serialized metadata case-insensitively", () => { + const result = buildScrapeRequestFromFlareSolverr({ + ...PROWLARR_2_5_2_REQUEST, + headers: { CONTENTTYPE: "application/json", ContentLength: "2" }, + }) + + expect(result.headers).toEqual({ "Content-Type": "application/json" }) + }) + + test("standard Content-Type takes precedence over Prowlarr metadata", () => { + const result = buildScrapeRequestFromFlareSolverr({ + ...PROWLARR_2_5_2_REQUEST, + headers: { + contentType: "application/x-www-form-urlencoded", + "content-type": "application/json", + }, + }) + + expect(result.headers).toEqual({ "content-type": "application/json" }) + }) + + test("still rejects POST data without content-type information", () => { + expect(() => + buildScrapeRequestFromFlareSolverr({ + cmd: "request.post", + url: "https://example.com/login", + postData: "user=a&pw=b", + }), + ).toThrow(RequestValidationError) + }) + + test("passes the normalized body and header through Tier 1 and browser route overrides", async () => { + const request = buildScrapeRequestFromFlareSolverr(PROWLARR_2_5_2_REQUEST) + let tier1Init: RequestInit | undefined + ;(globalThis as { fetch: typeof fetch }).fetch = (async (_input, init) => { + tier1Init = init + return new Response("ok", { status: 200, headers: { "content-type": "text/html" } }) + }) as typeof fetch + + await runTier1(request.url, request.headers, request.method, request.body) + expect(tier1Init?.method).toBe("POST") + expect(tier1Init?.body).toBe(PROWLARR_2_5_2_REQUEST.postData) + expect((tier1Init?.headers as Record | undefined)?.["Content-Type"]).toBe( + "application/x-www-form-urlencoded", + ) + + const browserOverride = routeContinueOverrides( + { + request: () => ({ headers: () => ({ accept: "text/html" }), method: () => "GET" }), + continue: async () => {}, + }, + request.headers, + request.method, + request.body, + ) + expect(browserOverride.method).toBe("POST") + expect(browserOverride.postData).toBe(PROWLARR_2_5_2_REQUEST.postData) + expect(browserOverride.headers["Content-Type"]).toBe("application/x-www-form-urlencoded") + }) +}) diff --git a/apps/api/src/adapters/flaresolverr.ts b/apps/api/src/adapters/flaresolverr.ts index 1f2b479..98ba0d3 100644 --- a/apps/api/src/adapters/flaresolverr.ts +++ b/apps/api/src/adapters/flaresolverr.ts @@ -2,9 +2,37 @@ import type { SupportedMethod } from "@trawl/tiers" import { normalizeProxy, requireContentTypeForBody, sanitizeHeaders } from "@trawl/tiers" import type { FlareSolverrRequest, FlareSolverrResponse, ScrapeRequest } from "@trawl/types" +function normalizeProwlarrHeaders(headers?: Record): Record | undefined { + if (!headers) return + + const normalized: Record = {} + let serializedContentType: string | undefined + let hasStandardContentType = false + + for (const [name, value] of Object.entries(headers)) { + const lowerName = name.trim().toLowerCase() + if (lowerName === "contenttype") { + serializedContentType ??= value + continue + } + if (lowerName === "contentlength") continue + if (lowerName === "content-type") hasStandardContentType = true + normalized[name] = value + } + + // Prowlarr serializes these HttpHeader properties without HTTP's hyphens. + // Keep an explicitly supplied standard header authoritative, even if invalid; + // the normal body validation below will then reject an empty value. + if (!hasStandardContentType && serializedContentType !== undefined) { + normalized["Content-Type"] = serializedContentType + } + + return normalized +} + export function buildScrapeRequestFromFlareSolverr(req: FlareSolverrRequest): ScrapeRequest { const method: SupportedMethod = req.cmd === "request.post" ? "POST" : "GET" - const headers = sanitizeHeaders(req.headers) + const headers = sanitizeHeaders(normalizeProwlarrHeaders(req.headers)) requireContentTypeForBody(headers, Boolean(req.postData)) return { url: req.url, diff --git a/apps/docs/api-reference/flaresolvr-compat.md b/apps/docs/api-reference/flaresolvr-compat.md index 6ad38a5..5f27aa4 100644 --- a/apps/docs/api-reference/flaresolvr-compat.md +++ b/apps/docs/api-reference/flaresolvr-compat.md @@ -30,7 +30,7 @@ interface FlareSolverrRequest { | `url` | string | Yes | The URL to scrape | | `maxTimeout` | number | No | Max wait in ms (default 60000) | | `postData` | string | No | POST body (only for `request.post`). On TRAWL's native `/scrape` endpoint this field is named `body`; the `/v1` adapter maps `postData` → `body` internally so the FlareSolverr wire contract stays unchanged for existing callers. | -| `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) | +| `headers` | object | No | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers). For Prowlarr compatibility, `contentType` is accepted as `Content-Type`; serialized `contentLength` is ignored and recalculated by the HTTP client. An explicit standard `Content-Type` takes precedence. | | `proxy` | string | No | **TRAWL-specific extension** (not in the real FlareSolverr v2 contract) — per-request proxy override for Tier 3/4, see [Configuration § Proxies](/getting-started/configuration#proxies) | ## Response @@ -118,6 +118,10 @@ cookies = data['solution']['cookies'] ### POST request +POST requests with `postData` must supply a content type. TRAWL accepts either a standard +`Content-Type` header or Prowlarr's serialized `headers.contentType` representation at this +compatibility endpoint. Native `/scrape` requests continue to require the standard header. + ```bash curl -s -X POST http://localhost:8191/v1 \ -H "Content-Type: application/json" \ @@ -125,6 +129,7 @@ curl -s -X POST http://localhost:8191/v1 \ "cmd": "request.post", "url": "https://example.com/api/login", "postData": "username=user&password=pass", + "headers": { "Content-Type": "application/x-www-form-urlencoded" }, "maxTimeout": 30000 }' ```