diff --git a/apps/api/src/proxy/ca.ts b/apps/api/src/proxy/ca.ts index f105f09..18c519c 100644 --- a/apps/api/src/proxy/ca.ts +++ b/apps/api/src/proxy/ca.ts @@ -15,7 +15,7 @@ import forge from "node-forge" // reachable by the trusted client (e.g. bound to localhost / a private Docker netns). export class MitmCa { private readonly caCert: forge.pki.Certificate - private readonly caKey: forge.pki.PrivateKey + private readonly caKey: forge.pki.rsa.PrivateKey private readonly leafKeys: forge.pki.rsa.KeyPair private readonly certCache = new Map() readonly caCertPem: string @@ -82,7 +82,10 @@ export class MitmCa { } } -function createCaCertificate(): { cert: forge.pki.Certificate; key: forge.pki.PrivateKey } { +function createCaCertificate(): { + cert: forge.pki.Certificate + key: forge.pki.rsa.PrivateKey +} { const keys = forge.pki.rsa.generateKeyPair(2048) const cert = forge.pki.createCertificate() cert.publicKey = keys.publicKey @@ -104,10 +107,13 @@ function createCaCertificate(): { cert: forge.pki.Certificate; key: forge.pki.Pr } // SAN must carry an IP entry (type 7) for literal-IP hosts and a DNS entry (type 2) -// otherwise, or strict clients reject the leaf. +// otherwise, or strict clients reject the leaf. node-forge's TypeScript types narrow +// `type` to string at the CertificateField boundary, but the runtime accepts the +// numeric GeneralName tags ("2" / "7") — the cast below bridges the two. function altNamesFor(host: string): forge.pki.CertificateField[] { const isIp = /^\d{1,3}(\.\d{1,3}){3}$/.test(host) - return [isIp ? { type: 7, ip: host } : { type: 2, value: host }] + const entry = isIp ? { type: "7", value: host } : { type: "2", value: host } + return [entry] } // 16 random hex bytes; leading 0 keeps it a positive integer for strict parsers. diff --git a/apps/api/src/proxy/server.ts b/apps/api/src/proxy/server.ts index 2f416f3..4e681e8 100644 --- a/apps/api/src/proxy/server.ts +++ b/apps/api/src/proxy/server.ts @@ -32,7 +32,10 @@ export interface MitmProxyOptions { debug?: boolean } -export function startMitmProxy(opts: MitmProxyOptions): { ca: MitmCa; server: net.Server } { +export function startMitmProxy(opts: MitmProxyOptions): { + ca: MitmCa + server: net.Server +} { const ca = new MitmCa(opts.caDir) // Per-host loopback TLS terminators. We know the target host from the CONNECT line, so @@ -67,7 +70,10 @@ export function startMitmProxy(opts: MitmProxyOptions): { ca: MitmCa; server: ne void handleConnect(clientSocket, target ?? "", tlsPortFor) } else { // Plain-HTTP proxy request: "GET http://host/path HTTP/1.1" - handlePlainHttp(clientSocket, first, opts).catch(() => clientSocket.destroy()) + // `first` arrives as Buffer from Bun's net.Socket 'data' event; the lib.dom.d.ts + // type widens it to string|Buffer for cross-runtime compatibility, but we only + // ever get bytes here. + handlePlainHttp(clientSocket, first as Buffer, opts).catch(() => clientSocket.destroy()) } }) clientSocket.on("error", () => clientSocket.destroy()) @@ -174,8 +180,10 @@ async function reissue( // bencoded payload. Raw HTML is also what Cardigann-style parsers want. // // Fast path is a browser navigation reusing the domain's cached session (cf_clearance). -// If that comes back as a Cloudflare interstitial, we run the full scrape() pipeline to -// solve the challenge (which refreshes the session cache) and retry the raw capture once. +// If that comes back as a Cloudflare interstitial, we rotate the proxy the same way +// Tier 3 does (markBad → next()), retry with a fresh per-attempt context so the new +// proxy actually applies (the pool-shared context can't be reconfigured mid-flight), +// and only fall through to the full scrape() pipeline once proxy rotation is exhausted. async function fetchRaw( url: string, method: SupportedMethod, @@ -184,19 +192,33 @@ async function fetchRaw( ): Promise<{ status: number; contentType: string; body: Buffer }> { const domain = new URL(url).hostname const maxTimeout = opts.maxTimeout ?? 60_000 + const proxyPool = opts.deps.proxyPool for (let attempt = 0; attempt < 2; attempt++) { const handle = await opts.deps.acquireBrowser(domain) - const page = await handle.context.newPage() + // Pick a fresh proxy per attempt — proxyPool.next() is sticky-per-domain until we + // markBad(), at which point it rotates. No pool → no proxy, just reuse handle.context. + const proxy = proxyPool?.next(domain) ?? undefined + const createdFreshCtx = Boolean(proxy) + const ctx = proxy + ? await handle.browser.newContext({ + viewport: null, + proxy: { server: proxy }, + }) + : handle.context + const page = await ctx.newPage() try { const session = await opts.deps.loadSession(domain) if (session?.cookies?.length) { - await handle.context.addCookies(session.cookies.map(toPlaywrightCookie)) + await ctx.addCookies(session.cookies.map(toPlaywrightCookie)) await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent }) } if (method !== "GET" || body !== undefined) { await page.route(url, (route: { continue: (o: Record) => void }) => - route.continue({ method, ...(body !== undefined ? { postData: body } : {}) }), + route.continue({ + method, + ...(body !== undefined ? { postData: body } : {}), + }), ) } @@ -214,7 +236,10 @@ async function fetchRaw( let resp: PlaywrightResponse | null = null try { - resp = await page.goto(url, { waitUntil: "domcontentloaded", timeout: maxTimeout }) + resp = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: maxTimeout, + }) } catch (err) { // A download navigation rejects goto — wait briefly for the download event to land. await Promise.race([downloadSeen, sleep(3000)]) @@ -225,13 +250,17 @@ async function fetchRaw( const filePath = await download.path() const buf = filePath ? readFileSync(filePath) : Buffer.alloc(0) await download.delete().catch(() => {}) - return { status: 200, contentType: contentTypeFor(download.suggestedFilename()), body: buf } + return { + status: 200, + contentType: contentTypeFor(download.suggestedFilename()), + body: buf, + } } const status: number = resp?.status() ?? 0 const respHeaders: Record = resp?.headers() ?? {} const contentType = respHeaders["content-type"] ?? "application/octet-stream" - const bodyBuf = Buffer.from(await resp?.body()) + const bodyBuf = Buffer.from((await resp?.body()) ?? new Uint8Array()) // Challenge interstitials are always small HTML — only sniff those, never binaries. const looksHtml = /text\/html/i.test(contentType) @@ -242,20 +271,28 @@ async function fetchRaw( isCloudflarePage(bodyBuf.toString("utf8", 0, 4096), {}) if (challenged) { - // Solve via the full tier pipeline (refreshes the cached session), then retry raw. - await scrape({ url, method, body, maxTier: opts.maxTier, maxTimeout }, opts.deps).catch(() => {}) - continue + // Same IP getting re-challenged means rotation has a real shot at clearing CF — + // mirror Tier 3's markBad + next(). + if (proxy && proxyPool) proxyPool.markBad(proxy) + // fall through to next attempt; finally still tears down the page and context + } else { + return { status, contentType, body: bodyBuf } } - return { status, contentType, body: bodyBuf } } finally { await page.close().catch(() => {}) + // Fresh per-attempt contexts must be closed explicitly or they leak. + if (createdFreshCtx) await ctx.close().catch(() => {}) opts.deps.releaseBrowser(handle.id) } } // Both raw attempts came back challenged — return whatever the solver produced as HTML. const solved = await scrape({ url, method, body, maxTier: opts.maxTier, maxTimeout }, opts.deps) - return { status: solved.statusCode || 200, contentType: "text/html; charset=utf-8", body: Buffer.from(solved.html) } + return { + status: solved.statusCode || 200, + contentType: "text/html; charset=utf-8", + body: Buffer.from(solved.html), + } } // Minimal plain-HTTP (non-TLS) proxy support, mainly for completeness / http:// targets.