From b37979996cdaadb092ec3ca673ebdcfd11122c8e Mon Sep 17 00:00:00 2001 From: germondai Date: Sun, 9 Aug 2026 19:59:11 +0200 Subject: [PATCH] fix(tiers): prevent stale challenge responses --- CHANGELOG.md | 1 + packages/tiers/src/tiers/2.ts | 29 +++++---- packages/tiers/src/tiers/3.ts | 27 +++----- packages/tiers/src/tiers/4.ts | 26 +++----- packages/tiers/src/utils/challengeWait.ts | 36 +++++++---- packages/tiers/src/utils/detect.ts | 11 +++- packages/tiers/src/utils/mainResponse.ts | 40 ++++++++++++ .../tiers/tests/cloudflareDetection.test.ts | 23 +++++++ packages/tiers/tests/mainResponse.test.ts | 62 +++++++++++++++++++ 9 files changed, 192 insertions(+), 63 deletions(-) create mode 100644 packages/tiers/src/utils/mainResponse.ts create mode 100644 packages/tiers/tests/cloudflareDetection.test.ts create mode 100644 packages/tiers/tests/mainResponse.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b7cd131..4c757ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- Keep browser-tier status, headers, content type, and raw body aligned with the latest main-frame navigation response across redirects, and prevent persistent Cloudflare challenges from being returned as successful rendered pages (#53). - Translate Prowlarr's serialized `headers.contentType` metadata at the FlareSolverr `/v1` compatibility boundary and discard `contentLength`, allowing form POST requests to enter the scraper pipeline (#50). - Bound Camoufox memory growth by counting every Tier 3/4 temporary context and rolling-replacing browsers at `BROWSER_RECYCLE_AFTER_CONTEXTS`, while keeping existing capacity available during warm-up. Replacement launches are serialized, cleanup is timeout-bounded, and failed launches retain the usable browser (#52). diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index 78490d7..45ad067 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -4,7 +4,8 @@ import { solvePageCaptchas } from "../solvers" import { normalizeSameSite, toCookies } from "../utils/cookies" import { hasAkamaiChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" import { normalizeHtml } from "../utils/html" -import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" +import { trackMainDocumentResponses } from "../utils/mainResponse" +import { captureResponse, isTextContentType } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" @@ -60,14 +61,7 @@ export async function runTier2( }) } - let statusCode = 200 - const mainResponseHolder: { value?: MinimalResponse } = {} - page.on("response", (res: MinimalResponse) => { - if (res.url() === url) { - statusCode = res.status() - if (!mainResponseHolder.value) mainResponseHolder.value = res - } - }) + const mainResponse = trackMainDocumentResponses(page) await page.goto(url, { waitUntil: "domcontentloaded", timeout: maxTimeout }) await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {}) @@ -83,7 +77,7 @@ export async function runTier2( } } - if (isCloudflarePage(html, {})) { + if (isCloudflarePage(html, mainResponse.headers)) { return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } @@ -93,8 +87,8 @@ export async function runTier2( return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" } } - if (isBlocked(statusCode, html)) { - return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + if (isBlocked(mainResponse.status, html)) { + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } } // Attempt to solve any embedded captcha widgets (Turnstile, reCAPTCHA, hCaptcha). @@ -106,9 +100,14 @@ export async function runTier2( captchasSolved = result.solved } + const finalHtml = await page.content() + if (isCloudflarePage(finalHtml, mainResponse.headers)) { + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } + } + const cookies: Cookie[] = toCookies(await activeContext.cookies()) - const captured = await captureResponse(mainResponseHolder.value) + const captured = await captureResponse(mainResponse.response) return { tier: 2, @@ -117,10 +116,10 @@ export async function runTier2( effectiveUrl: page.url(), // For HTML/text content-types, `html` is the rendered DOM. For binary, leave // empty so /scrape consumers know to use `body`/`contentType`. - html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(html) : "", + html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(finalHtml) : "", ...captured, cookies, - statusCode, + statusCode: mainResponse.status, captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, } } catch (err) { diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index ecae817..a8d9779 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -15,8 +15,9 @@ import { } from "../utils/detect" import { normalizeHtml } from "../utils/html" import { waitForImpervaResolution } from "../utils/impervaWait" +import { trackMainDocumentResponses } from "../utils/mainResponse" import { isHardNetworkFailure } from "../utils/network" -import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" +import { captureResponse, isTextContentType } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" @@ -64,17 +65,7 @@ export async function runTier3( }) } - let statusCode = 200 - const mainResponseHolder: { value?: MinimalResponse } = {} - page.on("response", (res: MinimalResponse) => { - try { - const resUrl = res.url() - if (resUrl === url || resUrl.startsWith(url.replace(/\/$/, ""))) { - statusCode = res.status() - if (!mainResponseHolder.value) mainResponseHolder.value = res - } - } catch {} - }) + const mainResponse = trackMainDocumentResponses(page) // CF challenges can trigger sub-navigations that throw "navigation interrupted" — // we catch those so we can continue. Hard failures (DNS, connection refused) are @@ -100,7 +91,7 @@ export async function runTier3( ? await waitForImpervaResolution(page, remaining, url) : challengeType === "akamai" ? await waitForAkamaiResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url, () => mainResponse.headers) if (resolution !== "ok") { return { @@ -150,7 +141,7 @@ export async function runTier3( return { tier: 3, status: "error", durationMs: Date.now() - start, reason: errMsg } } - if (isCloudflarePage(html, {})) { + if (isCloudflarePage(html, mainResponse.headers)) { const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] cloudflare-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) @@ -171,13 +162,13 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" } } - if (isBlocked(statusCode, html)) { - return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + if (isBlocked(mainResponse.status, html)) { + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } } const cookies: Cookie[] = toCookies(await freshCtx.cookies()) - const captured = await captureResponse(mainResponseHolder.value) + const captured = await captureResponse(mainResponse.response) return { tier: 3, @@ -188,7 +179,7 @@ export async function runTier3( ...captured, cookies, userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), - statusCode, + statusCode: mainResponse.status, captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, } } catch (err) { diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index a9a8f0b..cfb74dd 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -15,8 +15,9 @@ import { } from "../utils/detect" import { normalizeHtml } from "../utils/html" import { waitForImpervaResolution } from "../utils/impervaWait" +import { trackMainDocumentResponses } from "../utils/mainResponse" import { isHardNetworkFailure } from "../utils/network" -import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" +import { captureResponse, isTextContentType } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" @@ -66,16 +67,7 @@ export async function runTier4( }) } - let statusCode = 200 - const mainResponseHolder: { value?: MinimalResponse } = {} - page.on("response", (res: MinimalResponse) => { - try { - if (res.url() === url || res.url().startsWith(url.replace(/\/$/, ""))) { - statusCode = res.status() - if (!mainResponseHolder.value) mainResponseHolder.value = res - } - } catch {} - }) + const mainResponse = trackMainDocumentResponses(page) const gotoErr = await page .goto(url, { @@ -96,7 +88,7 @@ export async function runTier4( ? await waitForImpervaResolution(page, remaining, url) : challengeType === "akamai" ? await waitForAkamaiResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url, () => mainResponse.headers) if (resolution !== "ok") { return { @@ -136,7 +128,7 @@ export async function runTier4( } } - if (isCloudflarePage(html, {})) { + if (isCloudflarePage(html, mainResponse.headers)) { return { tier: 4, status: "blocked", @@ -163,13 +155,13 @@ export async function runTier4( } } - if (isBlocked(statusCode, html)) { - return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + if (isBlocked(mainResponse.status, html)) { + return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } } const cookies: Cookie[] = toCookies(await proxyContext.cookies()) - const captured = await captureResponse(mainResponseHolder.value) + const captured = await captureResponse(mainResponse.response) return { tier: 4, @@ -180,7 +172,7 @@ export async function runTier4( ...captured, cookies, userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), - statusCode, + statusCode: mainResponse.status, captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, } } catch (err) { diff --git a/packages/tiers/src/utils/challengeWait.ts b/packages/tiers/src/utils/challengeWait.ts index 704529c..53a21a6 100644 --- a/packages/tiers/src/utils/challengeWait.ts +++ b/packages/tiers/src/utils/challengeWait.ts @@ -13,10 +13,13 @@ export async function waitForChallengeResolution( page: Page, timeoutMs: number, originalUrl?: string, + responseHeaders: () => Record = () => ({}), ): Promise<"ok" | "ip-blocked" | "timeout"> { const deadline = Date.now() + Math.max(timeoutMs, 30_000) let lastClickAttempt = 0 let cfClearanceAt: number | undefined + let challengeSeen = false + let inactiveSamples = 0 // Only count cf_clearance for the current domain — warm browser may have cookies from prior domains const targetHost = (() => { @@ -27,25 +30,36 @@ export async function waitForChallengeResolution( } })() - const earlyTitle = await page.title().catch(() => "") - if (earlyTitle && !CF_CHALLENGE_TITLE.test(earlyTitle)) { - await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {}) - return "ok" - } - // Let CF's challenge JS boot up before we start polling - await new Promise((r) => setTimeout(r, 1000)) + await new Promise((r) => setTimeout(r, 300)) while (Date.now() < deadline) { try { - const title = await page.title().catch(() => "just a moment") - if (!CF_CHALLENGE_TITLE.test(title)) { + const title = await page.title().catch(() => "") + const html = await page.content().catch(() => "") + const url = page.url() + const hasChallengeFrame = page.frames().some(isChallengeFrame) + const active = + CF_CHALLENGE_TITLE.test(title) || + isCloudflarePage(html, responseHeaders()) || + /\/cdn-cgi\/challenge-platform|\/cdn-cgi\/challenge\//i.test(url) || + hasChallengeFrame + + if (!active) { + inactiveSamples++ + } else { + challengeSeen = true + inactiveSamples = 0 + } + + // Two observations prevent a transient normal title/DOM during navigation from + // declaring a challenge solved. This also applies before the first active sample. + if (inactiveSamples >= 2) { // 'load' not 'networkidle' — networkidle stalls indefinitely on JS-heavy pages await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {}) return "ok" } - const url = page.url() if (/\/cdn-cgi\/error\/|error=1020|error=1015/.test(url)) return "ip-blocked" const cookies: Array<{ name: string; domain: string }> = await page @@ -60,7 +74,7 @@ export async function waitForChallengeResolution( c.domain === `.${targetHost}` || targetHost.endsWith(c.domain.replace(/^\./, ""))), ) - if (hasDomainClearance) { + if (hasDomainClearance && challengeSeen) { if (cfClearanceAt === undefined) { cfClearanceAt = Date.now() console.log("[challenge] cf_clearance obtained") diff --git a/packages/tiers/src/utils/detect.ts b/packages/tiers/src/utils/detect.ts index 65051a0..9a6206d 100644 --- a/packages/tiers/src/utils/detect.ts +++ b/packages/tiers/src/utils/detect.ts @@ -9,7 +9,8 @@ export type ChallengeType = | "none" export function isCloudflarePage(html: string, headers: Record): boolean { - if (headers["cf-mitigated"]) return true + const cfMitigated = Object.entries(headers).find(([name]) => name.toLowerCase() === "cf-mitigated")?.[1] + if (cfMitigated?.toLowerCase() === "challenge") return true if (/[^<]*(just a moment|ddos-guard|please wait|checking|attention required)[^<]*<\/title>/i.test(html)) return true if (/checking your browser/i.test(html)) return true @@ -20,6 +21,11 @@ export function isCloudflarePage(html: string, headers: Record<string, string>): if (/id="cf-challenge-running"/i.test(html)) return true // CF Turnstile interstitial wrapper if (/id="turnstile-wrapper"/i.test(html)) return true + // Active challenge orchestration markers. Unlike the passive telemetry markers + // below, these only occur while Cloudflare is serving an interstitial. + if (/_cf_chl_opt/i.test(html)) return true + if (/id=["']challenge-form["']/i.test(html)) return true + if (/orchestrate\/chl_page/i.test(html)) return true // DDoS-Guard if (/ddos-guard\.net|\.ddos-guard\.net/i.test(html)) return true // CF firewall/WAF deny page (error 1020 and friends) — static "blocked" page, not a @@ -35,7 +41,8 @@ export function isCloudflarePage(html: string, headers: Record<string, string>): // bot-management telemetry. Matching on the marker alone flags real pages as blocked. // The actual challenge stub is always near-empty (nothing else can render before the // challenge resolves), so gate on page size too. - if (html.length < 3000 && /__CF\$cv\$params/i.test(html)) return true + if (html.length < 3000 && /__CF\$cv\$params|\/cdn-cgi\/challenge-platform\/[^"']*jsd\/main\.js/i.test(html)) + return true return false } diff --git a/packages/tiers/src/utils/mainResponse.ts b/packages/tiers/src/utils/mainResponse.ts new file mode 100644 index 0000000..3515f17 --- /dev/null +++ b/packages/tiers/src/utils/mainResponse.ts @@ -0,0 +1,40 @@ +import type { Page, Response } from "patchright" + +import type { MinimalResponse } from "./response" + +type NavigationResponse = MinimalResponse & Pick<Response, "request"> + +/** Tracks the latest top-level document response across redirects. */ +export class MainDocumentResponseTracker { + private latest?: NavigationResponse + + constructor(private readonly page: Pick<Page, "mainFrame">) {} + + observe(response: NavigationResponse): void { + try { + const request = response.request() + if (!request.isNavigationRequest() || request.frame() !== this.page.mainFrame()) return + this.latest = response + } catch { + // A response can disappear while Firefox is replacing a challenge document. + } + } + + get response(): MinimalResponse | undefined { + return this.latest + } + + get status(): number { + return this.latest?.status() ?? 200 + } + + get headers(): Record<string, string> { + return this.latest?.headers() ?? {} + } +} + +export function trackMainDocumentResponses(page: Page): MainDocumentResponseTracker { + const tracker = new MainDocumentResponseTracker(page) + page.on("response", (response) => tracker.observe(response)) + return tracker +} diff --git a/packages/tiers/tests/cloudflareDetection.test.ts b/packages/tiers/tests/cloudflareDetection.test.ts new file mode 100644 index 0000000..0a92dd9 --- /dev/null +++ b/packages/tiers/tests/cloudflareDetection.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test" +import { isCloudflarePage } from "../src/utils/detect" + +describe("Cloudflare challenge detection", () => { + test("treats cf-mitigated challenge as authoritative regardless of case or page title", () => { + const html = `<html><title>Welcome${"content ".repeat(1000)}` + expect(isCloudflarePage(html, { "CF-Mitigated": "Challenge" })).toBe(true) + }) + + test("detects active orchestration markers on large pages", () => { + const padding = "content ".repeat(1000) + expect(isCloudflarePage(`${padding}`, {})).toBe(true) + expect(isCloudflarePage(`
${padding}`, {})).toBe(true) + expect( + isCloudflarePage(`${padding}`, {}), + ).toBe(true) + }) + + test("allows passive Cloudflare telemetry on an ordinary large page", () => { + const html = `Real page${"content ".repeat(1000)}` + expect(isCloudflarePage(html, {})).toBe(false) + }) +}) diff --git a/packages/tiers/tests/mainResponse.test.ts b/packages/tiers/tests/mainResponse.test.ts new file mode 100644 index 0000000..8db1654 --- /dev/null +++ b/packages/tiers/tests/mainResponse.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from "bun:test" +import { MainDocumentResponseTracker } from "../src/utils/mainResponse" +import { captureResponse } from "../src/utils/response" + +const mainFrame = {} +const iframe = {} +const page = { mainFrame: () => mainFrame } + +function response( + url: string, + status: number, + headers: Record, + body: string, + options: { navigation?: boolean; frame?: object } = {}, +) { + return { + url: () => url, + status: () => status, + headers: () => headers, + body: async () => Buffer.from(body), + request: () => ({ + isNavigationRequest: () => options.navigation ?? true, + frame: () => options.frame ?? mainFrame, + }), + } +} + +describe("MainDocumentResponseTracker", () => { + test("keeps the final response across cross-origin redirects with coherent metadata", async () => { + const tracker = new MainDocumentResponseTracker(page as never) + tracker.observe( + response("https://a.example/start", 302, { location: "https://b.example/final" }, "redirect") as never, + ) + tracker.observe( + response("https://b.example/final", 200, { "content-type": "text/html", "x-final": "yes" }, "origin") as never, + ) + + expect(tracker.status).toBe(200) + expect(tracker.headers["x-final"]).toBe("yes") + const captured = await captureResponse(tracker.response) + expect(new TextDecoder().decode(captured.body)).toBe("origin") + expect(captured.contentType).toBe("text/html") + }) + + test("ignores subresources and iframe navigations", () => { + const tracker = new MainDocumentResponseTracker(page as never) + tracker.observe(response("https://example.com/", 200, {}, "main") as never) + tracker.observe(response("https://example.com/app.js", 404, {}, "script", { navigation: false }) as never) + tracker.observe(response("https://challenge.example/frame", 403, {}, "frame", { frame: iframe }) as never) + + expect(tracker.status).toBe(200) + expect(tracker.response?.url()).toBe("https://example.com/") + }) + + test("retains a terminal external-scheme redirect", () => { + const tracker = new MainDocumentResponseTracker(page as never) + tracker.observe(response("https://example.com/download", 301, { location: "magnet:?xt=urn:test" }, "") as never) + + expect(tracker.status).toBe(301) + expect(tracker.headers.location).toBe("magnet:?xt=urn:test") + }) +})