diff --git a/.env.example b/.env.example index bd35d35..c2d7468 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,25 @@ PORT=8191 PORT_WEB=3000 PORT_DOCS=3001 +# --- MITM forward-proxy mode (optional, off by default) --- +# +# Browser-backed HTTP(S) forward proxy. Point a client's proxy setting at it and every +# request it makes is transparently re-issued through the browser pool — for clients like +# Prowlarr that only consume cookies+UA from /v1 and then re-fetch themselves, which fails +# on sites whose Cloudflare clearance is bound to the browser's connection fingerprint. +# +# It terminates TLS with its own CA (persisted in MITM_PROXY_CA_DIR). Install that CA +# (GET /proxy-ca.crt, or the ca.crt file) into the client's trust store, then set the +# client's proxy to http://:. Only expose it to trusted +# clients on a private interface — it can impersonate any host to a client that trusts it. +MITM_PROXY_ENABLED=false +MITM_PROXY_PORT=8192 +MITM_PROXY_CA_DIR=/data/proxy-ca +# Cap the escalation tier the proxy uses (e.g. 3 to keep it off residential Tier 4). Blank = up to 4. +MITM_PROXY_MAX_TIER= +# Log one line per proxied request (noisy). Errors are always logged. +MITM_PROXY_DEBUG=false + # --- reCAPTCHA v2 audio solving (optional) --- # # By default TRAWL uses Google's own free Speech API to transcribe reCAPTCHA diff --git a/CHANGELOG.md b/CHANGELOG.md index 5318580..8913138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `scripts/bench-targets.sh`, `scripts/bench-success-rate.sh`, `scripts/bench-compare.sh` — observability harnesses for measuring CF challenge latency + bypass success rate. +### Added +- **MITM forward-proxy mode** (`MITM_PROXY_ENABLED`) — a browser-backed HTTP(S) forward proxy. + The FlareSolverr `/v1` contract only returns cookies + user-agent; clients like Prowlarr take + those and re-fetch the target with their own HTTP stack, which is re-challenged on sites whose + Cloudflare clearance is bound to the solving browser's connection fingerprint (e.g. 1337x) — no + cookie is portable to a plain client. Point such a client's HTTP proxy at this instead and every + request is transparently re-issued through the browser pool, returning the raw response bytes + (so `.torrent`/binary downloads pass through intact, not just HTML). Terminates TLS with a + self-generated CA (persisted to `MITM_PROXY_CA_DIR`, downloadable at `GET /proxy-ca.crt`) that + you install into the client's trust store. New env: `MITM_PROXY_ENABLED`, `MITM_PROXY_PORT` + (default 8192), `MITM_PROXY_CA_DIR`, `MITM_PROXY_MAX_TIER`, `MITM_PROXY_DEBUG`. + ## [1.0.0] - 2026-07-10 ### Changed diff --git a/README.md b/README.md index d735320..ea02cdf 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,33 @@ http://localhost:8191 # running on the same host http://trawl:8191 # running via Docker Compose on the same network ``` +### MITM forward-proxy mode (fingerprint-bound Cloudflare, e.g. 1337x) + +Some sites bind their Cloudflare clearance to the solving browser's full connection +fingerprint. The `/v1` flow can't help there: Prowlarr keeps only the cookie + user-agent +and **re-fetches the page with its own HTTP client**, which Cloudflare re-challenges — the +cookie isn't portable. For those indexers, enable the browser-backed forward proxy and add +it to Prowlarr as an **HTTP proxy** (tag it onto just the affected indexers): + +```env +MITM_PROXY_ENABLED=true +MITM_PROXY_PORT=8192 +MITM_PROXY_CA_DIR=/data/proxy-ca # persist the CA (mount a volume) +``` + +1. Install the proxy's CA into the client's trust store so it accepts the per-host certs: + `curl http://:8191/proxy-ca.crt` → add to the Prowlarr container's CA store + (e.g. a linuxserver `/custom-cont-init.d` script that copies it to + `/usr/local/share/ca-certificates/` and runs `update-ca-certificates`). +2. Prowlarr → Settings → Indexer Proxies → **HTTP**, host ``, port `8192`, give it + a tag, and add that tag to the CF-fingerprint-bound indexer. + +Every request that indexer makes (search **and** the `.torrent`/magnet grab) is then re-issued +through the browser pool and returns the exact bytes Cloudflare served the browser. + +> ⚠️ A MITM proxy can impersonate any host to a client that trusts its CA. Only expose it on a +> private interface (localhost / a private Docker network), never publicly. + ## Tiers ``` diff --git a/apps/api/package.json b/apps/api/package.json index a2a5fe1..1c7a56e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -13,10 +13,12 @@ "@trawl/types": "workspace:*", "camoufox-js": "0.11.2", "elysia": "^1.4.29", - "memoirist": "1.2.0" + "memoirist": "1.2.0", + "node-forge": "^1.3.1" }, "devDependencies": { "typescript": "^7.0.2", - "@types/bun": "^1.3.14" + "@types/bun": "^1.3.14", + "@types/node-forge": "^1.3.11" } } diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 09fa472..f244860 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -25,4 +25,22 @@ export const residentialProxyPool = process.env.RESIDENTIAL_PROXY_LIST_FILE || undefined, ) ?? undefined +// ── MITM forward-proxy mode ──────────────────────────────────────────────────── +// Optional browser-backed HTTP(S) forward proxy (apps/api/src/proxy). Off by default. +// When enabled, point a client's proxy setting at MITM_PROXY_PORT and every request it +// makes is re-issued through the browser pool — for clients (e.g. Prowlarr) that only +// consume cookies+UA from /v1 and re-fetch themselves, which fails on fingerprint-bound +// Cloudflare clearances. See proxy/server.ts for the full rationale. +export const MITM_PROXY_ENABLED = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_ENABLED ?? "") +export const MITM_PROXY_PORT = Number(process.env.MITM_PROXY_PORT ?? "8192") +// CA cert + key live here (persist across restarts so the CA is installed once). +export const MITM_PROXY_CA_DIR = process.env.MITM_PROXY_CA_DIR ?? "/data/proxy-ca" +// Cap the tier the proxy will escalate to (e.g. keep it off residential Tier 4). +export const MITM_PROXY_MAX_TIER = process.env.MITM_PROXY_MAX_TIER + ? (Number(process.env.MITM_PROXY_MAX_TIER) as 1 | 2 | 3 | 4) + : undefined +// Log one line per proxied request (method, url, status, content-type, bytes). Off by +// default — proxied clients can be chatty. Errors are always logged. +export const MITM_PROXY_DEBUG = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_DEBUG ?? "") + export const startTime = Date.now() diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index c3e61b8..9baa6a9 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,19 +1,49 @@ import { Elysia } from "elysia" -import { POOL_SIZE, PORT } from "./config" -import { initPool } from "./deps" +import { + MITM_PROXY_CA_DIR, + MITM_PROXY_DEBUG, + MITM_PROXY_ENABLED, + MITM_PROXY_MAX_TIER, + MITM_PROXY_PORT, + POOL_SIZE, + PORT, +} from "./config" +import { getDeps, initPool } from "./deps" import { registerLifecycleHandlers } from "./lifecycle" +import { startMitmProxy } from "./proxy/server" import { healthRoute } from "./routes/health" import { indexRoute } from "./routes/index" +import { proxyCaRoute } from "./routes/proxy-ca" import { scrapeRoute } from "./routes/scrape" import { statsRoute } from "./routes/stats" import { v1Route } from "./routes/v1" -new Elysia().use(indexRoute()).use(healthRoute()).use(statsRoute()).use(v1Route()).use(scrapeRoute()).listen(PORT) +new Elysia() + .use(indexRoute()) + .use(healthRoute()) + .use(statsRoute()) + .use(v1Route()) + .use(scrapeRoute()) + .use(proxyCaRoute()) + .listen(PORT) console.log(`[api] TRAWL starting on :${PORT} (pool: ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"})`) -initPool().catch((err) => { - console.error("[api] startup failed:", err) - process.exit(1) -}) +initPool() + .then(() => { + // Proxy needs a ready pool — start it only after the browsers are warm. + if (MITM_PROXY_ENABLED) { + startMitmProxy({ + port: MITM_PROXY_PORT, + caDir: MITM_PROXY_CA_DIR, + deps: getDeps(), + maxTier: MITM_PROXY_MAX_TIER, + debug: MITM_PROXY_DEBUG, + }) + } + }) + .catch((err) => { + console.error("[api] startup failed:", err) + process.exit(1) + }) registerLifecycleHandlers() diff --git a/apps/api/src/proxy/ca.ts b/apps/api/src/proxy/ca.ts new file mode 100644 index 0000000..f105f09 --- /dev/null +++ b/apps/api/src/proxy/ca.ts @@ -0,0 +1,117 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import forge from "node-forge" + +// A tiny on-the-fly certificate authority for the MITM forward proxy. +// +// The proxy terminates the client's TLS so it can re-issue each request through the +// browser pool (see server.ts). To do that it must present a certificate the client +// trusts for the *target* host. We generate one long-lived CA (persisted to disk so +// the same CA cert can be installed into the client's trust store once) and mint a +// short per-host leaf certificate on demand, signed by that CA. +// +// The CA private key never leaves this container. Installing the CA cert in a client +// lets THIS proxy impersonate any host to THAT client — so the proxy must only ever be +// reachable by the trusted client (e.g. bound to localhost / a private Docker netns). +export class MitmCa { + private readonly caCert: forge.pki.Certificate + private readonly caKey: forge.pki.PrivateKey + private readonly leafKeys: forge.pki.rsa.KeyPair + private readonly certCache = new Map() + readonly caCertPem: string + readonly caCertPath: string + + constructor(dir: string) { + mkdirSync(dir, { recursive: true }) + this.caCertPath = join(dir, "ca.crt") + const keyPath = join(dir, "ca.key") + + if (existsSync(this.caCertPath) && existsSync(keyPath)) { + this.caCertPem = readFileSync(this.caCertPath, "utf8") + this.caCert = forge.pki.certificateFromPem(this.caCertPem) + this.caKey = forge.pki.privateKeyFromPem(readFileSync(keyPath, "utf8")) + } else { + const { cert, key } = createCaCertificate() + this.caCert = cert + this.caKey = key + this.caCertPem = forge.pki.certificateToPem(cert) + writeFileSync(this.caCertPath, this.caCertPem) + writeFileSync(keyPath, forge.pki.privateKeyToPem(key), { mode: 0o600 }) + } + + // One leaf keypair shared across every minted host cert — only the certificate + // (subject + SAN) differs per host, so there's no need to pay RSA keygen per host. + this.leafKeys = forge.pki.rsa.generateKeyPair(2048) + } + + // The shared leaf private key (PEM) — every minted host cert is signed for this key, + // so one key serves all per-host TLS servers. + get leafKeyPem(): string { + return forge.pki.privateKeyToPem(this.leafKeys.privateKey) + } + + // Returns a leaf certificate (PEM) valid for `host`, minting + caching on first use. + // Serve the leaf ALONE: the client trusts our CA directly (it's the root), so no chain + // is needed. (Appending the CA made Bun's TLS stack pick the wrong end-entity cert.) + leafCertPem(host: string): string { + const cached = this.certCache.get(host) + if (cached) return cached + const pem = forge.pki.certificateToPem(this.mintLeaf(host)) + this.certCache.set(host, pem) + return pem + } + + private mintLeaf(host: string): forge.pki.Certificate { + const cert = forge.pki.createCertificate() + cert.publicKey = this.leafKeys.publicKey + cert.serialNumber = randomSerial() + // Backdate 1h to tolerate mild clock skew between proxy and client containers. + cert.validity.notBefore = new Date(Date.now() - 3600_000) + cert.validity.notAfter = new Date(Date.now() + 397 * 24 * 3600_000) // 397d — CA/B leaf max + const subject = [{ name: "commonName", value: host }] + cert.setSubject(subject) + cert.setIssuer(this.caCert.subject.attributes) + cert.setExtensions([ + { name: "basicConstraints", cA: false }, + { name: "keyUsage", digitalSignature: true, keyEncipherment: true }, + { name: "extKeyUsage", serverAuth: true }, + { name: "subjectAltName", altNames: altNamesFor(host) }, + ]) + cert.sign(this.caKey, forge.md.sha256.create()) + return cert + } +} + +function createCaCertificate(): { cert: forge.pki.Certificate; key: forge.pki.PrivateKey } { + const keys = forge.pki.rsa.generateKeyPair(2048) + const cert = forge.pki.createCertificate() + cert.publicKey = keys.publicKey + cert.serialNumber = randomSerial() + cert.validity.notBefore = new Date(Date.now() - 3600_000) + cert.validity.notAfter = new Date(Date.now() + 10 * 365 * 24 * 3600_000) // 10y + const attrs = [ + { name: "commonName", value: "TRAWL MITM Proxy CA" }, + { name: "organizationName", value: "TRAWL" }, + ] + cert.setSubject(attrs) + cert.setIssuer(attrs) + cert.setExtensions([ + { name: "basicConstraints", cA: true, critical: true }, + { name: "keyUsage", keyCertSign: true, cRLSign: true, critical: true }, + ]) + cert.sign(keys.privateKey, forge.md.sha256.create()) + return { cert, key: keys.privateKey } +} + +// SAN must carry an IP entry (type 7) for literal-IP hosts and a DNS entry (type 2) +// otherwise, or strict clients reject the leaf. +function altNamesFor(host: string): forge.pki.CertificateField[] { + const isIp = /^\d{1,3}(\.\d{1,3}){3}$/.test(host) + return [isIp ? { type: 7, ip: host } : { type: 2, value: host }] +} + +// 16 random hex bytes; leading 0 keeps it a positive integer for strict parsers. +function randomSerial(): string { + const bytes = forge.random.getBytesSync(16) + return `00${forge.util.bytesToHex(bytes)}` +} diff --git a/apps/api/src/proxy/server.ts b/apps/api/src/proxy/server.ts new file mode 100644 index 0000000..2f416f3 --- /dev/null +++ b/apps/api/src/proxy/server.ts @@ -0,0 +1,358 @@ +import { readFileSync } from "node:fs" +import net from "node:net" +import tls from "node:tls" +import { isCloudflarePage, type OrchestratorDeps, scrape } from "@trawl/tiers" +import type { Cookie, SupportedMethod } from "@trawl/types" +import { MitmCa } from "./ca" + +// Browser-backed MITM forward proxy. +// +// WHY THIS EXISTS: the FlareSolverr `/v1` contract only hands back cookies + user-agent. +// Clients like Prowlarr take those, then RE-FETCH the target with their own HTTP stack. +// Against sites whose Cloudflare clearance is bound to the solving browser's full +// connection fingerprint (not just cookie+UA), that re-fetch is re-challenged and fails — +// no cookie is portable to a plain HTTP client. See the FlareSolverr adapter for the +// legacy path. +// +// This proxy sidesteps that: point the client's HTTP(S) proxy at it (per-indexer in +// Prowlarr), and every request the client makes is transparently re-issued through the +// real browser pool via scrape(), so Cloudflare always sees the fingerprint it cleared. +// +// It is a MITM: it terminates the client's TLS using a per-host cert from our own CA +// (ca.ts). Only expose it to trusted clients on a private interface. + +const MAX_HEADER_BYTES = 64 * 1024 + +export interface MitmProxyOptions { + port: number + caDir: string + deps: OrchestratorDeps + maxTier?: 1 | 2 | 3 | 4 + maxTimeout?: number + debug?: boolean +} + +export function startMitmProxy(opts: MitmProxyOptions): { ca: MitmCa; server: net.Server } { + const ca = new MitmCa(opts.caDir) + + // Per-host loopback TLS terminators. We know the target host from the CONNECT line, so + // each host gets its own real listening TLS server whose base cert is that host's leaf — + // no SNI routing needed (Bun's node:tls doesn't invoke SNICallback, and can't drive a + // handshake via emit("connection"), so a real listening server per host is the reliable + // path). On CONNECT we bridge the raw client socket to the matching server's loopback + // port; it terminates TLS natively and hands us the decrypted stream. + const tlsPorts = new Map>() + + function tlsPortFor(host: string): Promise { + const existing = tlsPorts.get(host) + if (existing) return existing + const p = new Promise((resolve, reject) => { + const srv = tls.createServer({ key: ca.leafKeyPem, cert: ca.leafCertPem(host) }, (tlsSocket) => { + tlsSocket.on("error", () => tlsSocket.destroy()) + serveRequests(tlsSocket, host, opts) + }) + srv.on("error", reject) + srv.listen(0, "127.0.0.1", () => resolve((srv.address() as net.AddressInfo).port)) + }) + tlsPorts.set(host, p) + return p + } + + const server = net.createServer((clientSocket) => { + clientSocket.once("data", (first) => { + const firstLine = first.toString("latin1").split("\r\n", 1)[0] ?? "" + const [method, target] = firstLine.split(" ") + + if (method === "CONNECT") { + void handleConnect(clientSocket, target ?? "", tlsPortFor) + } else { + // Plain-HTTP proxy request: "GET http://host/path HTTP/1.1" + handlePlainHttp(clientSocket, first, opts).catch(() => clientSocket.destroy()) + } + }) + clientSocket.on("error", () => clientSocket.destroy()) + }) + + server.on("error", (err) => console.error("[proxy] server error:", err instanceof Error ? err.message : err)) + server.listen(opts.port, () => { + console.log(`[proxy] MITM forward proxy on :${opts.port} (CA: ${ca.caCertPath})`) + }) + + return { ca, server } +} + +// CONNECT host:port → 200, then bridge the raw client socket to the host's loopback TLS +// terminator. We pause first because reading the CONNECT line left the socket flowing — +// pipe() resumes it once the bridge is wired, so the client's ClientHello isn't dropped. +async function handleConnect( + clientSocket: net.Socket, + target: string, + tlsPortFor: (host: string) => Promise, +): Promise { + const host = target.split(":")[0] ?? "" + if (!host) { + clientSocket.destroy() + return + } + clientSocket.pause() + let port: number + try { + port = await tlsPortFor(host) + } catch { + clientSocket.destroy() + return + } + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n", () => { + const upstream = net.connect(port, "127.0.0.1", () => { + clientSocket.pipe(upstream) + upstream.pipe(clientSocket) + }) + upstream.on("error", () => clientSocket.destroy()) + clientSocket.on("error", () => upstream.destroy()) + }) +} + +// Read one HTTP/1.1 request off the decrypted stream, re-issue it through the browser, +// and write the solved response back. We answer one request per TLS connection and close +// (Connection: close) — clients open a fresh CONNECT per request, which keeps parsing +// trivial and lets each request pick up the freshest cached session. +function serveRequests(stream: tls.TLSSocket, host: string, opts: MitmProxyOptions): void { + const chunks: Buffer[] = [] + let total = 0 + + const onData = (chunk: Buffer) => { + chunks.push(chunk) + total += chunk.length + const buf = Buffer.concat(chunks) + const headerEnd = buf.indexOf("\r\n\r\n") + + if (headerEnd === -1) { + if (total > MAX_HEADER_BYTES) stream.destroy() + return + } + + const headerText = buf.subarray(0, headerEnd).toString("latin1") + const lines = headerText.split("\r\n") + const [method = "GET", path = "/"] = (lines[0] ?? "").split(" ") + const headers = parseHeaders(lines.slice(1)) + + const contentLength = Number(headers["content-length"] ?? "0") + const bodyStart = headerEnd + 4 + const bodyAvailable = buf.length - bodyStart + if (contentLength > 0 && bodyAvailable < contentLength) return // wait for full body + + stream.off("data", onData) + const body = contentLength > 0 ? buf.subarray(bodyStart, bodyStart + contentLength).toString("utf8") : undefined + const url = `https://${headers.host ?? host}${path}` + + void reissue(stream, url, method as SupportedMethod, body, opts) + } + + stream.on("data", onData) +} + +async function reissue( + stream: tls.TLSSocket, + url: string, + method: SupportedMethod, + body: string | undefined, + opts: MitmProxyOptions, +): Promise { + try { + const res = await fetchRaw(url, method, body, opts) + if (opts.debug) console.log(`[proxy] ${method} ${url} -> ${res.status} ${res.contentType} ${res.body.length}b`) + writeResponse(stream, res.status || 200, res.body, res.contentType) + } catch (err) { + console.error("[proxy] reissue failed for", url, err instanceof Error ? err.message : err) + writeResponse(stream, 502, Buffer.from(`TRAWL proxy error: ${err instanceof Error ? err.message : String(err)}`)) + } +} + +// Re-issue the request through the browser pool and return the RAW response bytes +// (status + content-type + body). Raw bytes are essential: clients download .torrent +// files through this proxy, and rendering them as HTML (page.content()) corrupts the +// bencoded payload. Raw HTML is also what Cardigann-style parsers want. +// +// Fast path is a browser navigation reusing the domain's cached session (cf_clearance). +// If that comes back as a Cloudflare interstitial, we run the full scrape() pipeline to +// solve the challenge (which refreshes the session cache) and retry the raw capture once. +async function fetchRaw( + url: string, + method: SupportedMethod, + body: string | undefined, + opts: MitmProxyOptions, +): Promise<{ status: number; contentType: string; body: Buffer }> { + const domain = new URL(url).hostname + const maxTimeout = opts.maxTimeout ?? 60_000 + + for (let attempt = 0; attempt < 2; attempt++) { + const handle = await opts.deps.acquireBrowser(domain) + const page = await handle.context.newPage() + try { + const session = await opts.deps.loadSession(domain) + if (session?.cookies?.length) { + await handle.context.addCookies(session.cookies.map(toPlaywrightCookie)) + await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent }) + } + if (method !== "GET" || body !== undefined) { + await page.route(url, (route: { continue: (o: Record) => void }) => + route.continue({ method, ...(body !== undefined ? { postData: body } : {}) }), + ) + } + + // A .torrent (application/x-bittorrent) makes Firefox start a DOWNLOAD instead of a + // navigation, so page.goto aborts. Capture it via the download event and read the + // saved file — this still uses the real browser network (correct fingerprint + + // cf_clearance), just the file-download path instead of the document path. + let download: PlaywrightDownload | undefined + const downloadSeen = new Promise((res) => + page.once("download", (d: PlaywrightDownload) => { + download = d + res() + }), + ) + + let resp: PlaywrightResponse | null = null + try { + resp = await page.goto(url, { waitUntil: "domcontentloaded", timeout: maxTimeout }) + } catch (err) { + // A download navigation rejects goto — wait briefly for the download event to land. + await Promise.race([downloadSeen, sleep(3000)]) + if (!download) throw err + } + + if (download) { + const filePath = await download.path() + const buf = filePath ? readFileSync(filePath) : Buffer.alloc(0) + await download.delete().catch(() => {}) + return { status: 200, contentType: contentTypeFor(download.suggestedFilename()), body: buf } + } + + const status: number = resp?.status() ?? 0 + const respHeaders: Record = resp?.headers() ?? {} + const contentType = respHeaders["content-type"] ?? "application/octet-stream" + const bodyBuf = Buffer.from(await resp?.body()) + + // Challenge interstitials are always small HTML — only sniff those, never binaries. + const looksHtml = /text\/html/i.test(contentType) + const challenged = + attempt === 0 && + (status === 403 || status === 503) && + looksHtml && + isCloudflarePage(bodyBuf.toString("utf8", 0, 4096), {}) + + if (challenged) { + // Solve via the full tier pipeline (refreshes the cached session), then retry raw. + await scrape({ url, method, body, maxTier: opts.maxTier, maxTimeout }, opts.deps).catch(() => {}) + continue + } + return { status, contentType, body: bodyBuf } + } finally { + await page.close().catch(() => {}) + opts.deps.releaseBrowser(handle.id) + } + } + + // Both raw attempts came back challenged — return whatever the solver produced as HTML. + const solved = await scrape({ url, method, body, maxTier: opts.maxTier, maxTimeout }, opts.deps) + return { status: solved.statusCode || 200, contentType: "text/html; charset=utf-8", body: Buffer.from(solved.html) } +} + +// Minimal plain-HTTP (non-TLS) proxy support, mainly for completeness / http:// targets. +async function handlePlainHttp(clientSocket: net.Socket, first: Buffer, opts: MitmProxyOptions): Promise { + const headerText = first.toString("latin1") + const line = headerText.split("\r\n", 1)[0] ?? "" + const [method = "GET", absUrl = ""] = line.split(" ") + if (!/^https?:\/\//.test(absUrl)) { + clientSocket.destroy() + return + } + try { + const res = await fetchRaw(absUrl, method as SupportedMethod, undefined, opts) + if (opts.debug) + console.log(`[proxy] ${method} ${absUrl} (plain) -> ${res.status} ${res.contentType} ${res.body.length}b`) + writeResponse(clientSocket, res.status || 200, res.body, res.contentType) + } catch (err) { + writeResponse( + clientSocket, + 502, + Buffer.from(`TRAWL proxy error: ${err instanceof Error ? err.message : String(err)}`), + ) + } +} + +// Minimal structural types for the Playwright objects we touch — camoufox-js doesn't +// re-export Playwright's types (see BrowserHandle in @trawl/types), so we shape just the +// members we call. +interface PlaywrightResponse { + status(): number + headers(): Record + body(): Promise +} +interface PlaywrightDownload { + path(): Promise + suggestedFilename(): string + delete(): Promise +} + +const sleep = (ms: number): Promise => new Promise((res) => setTimeout(res, ms)) + +// Best-effort content type from a downloaded filename — mainly so *arr clients see +// application/x-bittorrent for .torrent files. +function contentTypeFor(filename: string): string { + if (/\.torrent$/i.test(filename)) return "application/x-bittorrent" + if (/\.nzb$/i.test(filename)) return "application/x-nzb" + return "application/octet-stream" +} + +// Playwright's addCookies rejects unknown sameSite spellings — map/whitelist to its enum. +function toPlaywrightCookie(c: Cookie): Record { + const ss = (c.sameSite ?? "").toLowerCase() + const sameSite = ss === "strict" ? "Strict" : ss === "lax" ? "Lax" : ss === "none" ? "None" : undefined + return { + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + expires: c.expires, + httpOnly: c.httpOnly, + secure: c.secure, + ...(sameSite ? { sameSite } : {}), + } +} + +function writeResponse( + sock: net.Socket | tls.TLSSocket, + status: number, + body: Buffer, + contentType = "text/html; charset=utf-8", +): void { + const head = + `HTTP/1.1 ${status} ${reason(status)}\r\n` + + `Content-Type: ${contentType}\r\n` + + `Content-Length: ${body.length}\r\n` + + "Connection: close\r\n\r\n" + sock.write(head) + sock.write(body) + sock.end() +} + +function parseHeaders(lines: string[]): Record { + const out: Record = {} + for (const line of lines) { + const idx = line.indexOf(":") + if (idx > 0) out[line.slice(0, idx).trim().toLowerCase()] = line.slice(idx + 1).trim() + } + return out +} + +function reason(status: number): string { + const map: Record = { + 200: "OK", + 403: "Forbidden", + 404: "Not Found", + 500: "Internal Server Error", + 502: "Bad Gateway", + } + return map[status] ?? "OK" +} diff --git a/apps/api/src/routes/proxy-ca.ts b/apps/api/src/routes/proxy-ca.ts new file mode 100644 index 0000000..64914a1 --- /dev/null +++ b/apps/api/src/routes/proxy-ca.ts @@ -0,0 +1,22 @@ +import { existsSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { Elysia } from "elysia" +import { MITM_PROXY_CA_DIR, MITM_PROXY_ENABLED } from "../config" + +// Serves the MITM proxy CA certificate for easy installation into a client's trust +// store (e.g. `curl http://trawl:8191/proxy-ca.crt`). Only mounted when proxy mode is on. +export function proxyCaRoute() { + const app = new Elysia() + if (!MITM_PROXY_ENABLED) return app + + return app.get("/proxy-ca.crt", ({ set }) => { + const path = join(MITM_PROXY_CA_DIR, "ca.crt") + if (!existsSync(path)) { + set.status = 503 + return "CA not generated yet — start the proxy first" + } + set.headers["content-type"] = "application/x-pem-file" + set.headers["content-disposition"] = 'attachment; filename="trawl-proxy-ca.crt"' + return readFileSync(path, "utf8") + }) +} diff --git a/bun.lock b/bun.lock index 47b6f45..8517a98 100644 --- a/bun.lock +++ b/bun.lock @@ -18,9 +18,11 @@ "camoufox-js": "0.11.2", "elysia": "^1.4.29", "memoirist": "1.2.0", + "node-forge": "^1.3.1" }, "devDependencies": { "@types/bun": "^1.3.14", + "@types/node-forge": "^1.3.11", "typescript": "^7.0.2", }, }, @@ -865,6 +867,8 @@ "@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], + "@types/node-forge": ["@types/node-forge@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw=="], + "@types/resolve": ["@types/resolve@1.20.2", "", {}, "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q=="], "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="],