fix(tiers): prevent stale challenge responses

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Fixed
+- Keep browser-tier status, headers, content type, and raw body aligned with the latest main-frame navigation response across redirects, and prevent persistent Cloudflare challenges from being returned as successful rendered pages (#53).
 - Translate Prowlarr's serialized `headers.contentType` metadata at the FlareSolverr `/v1` compatibility boundary and discard `contentLength`, allowing form POST requests to enter the scraper pipeline (#50).
 - Bound Camoufox memory growth by counting every Tier 3/4 temporary context and rolling-replacing browsers at `BROWSER_RECYCLE_AFTER_CONTEXTS`, while keeping existing capacity available during warm-up. Replacement launches are serialized, cleanup is timeout-bounded, and failed launches retain the usable browser (#52).
 

packages/tiers/src/tiers/2.ts

@@ -4,7 +4,8 @@ import { solvePageCaptchas } from "../solvers"
 import { normalizeSameSite, toCookies } from "../utils/cookies"
 import { hasAkamaiChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect"
 import { normalizeHtml } from "../utils/html"
-import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response"
+import { trackMainDocumentResponses } from "../utils/mainResponse"
+import { captureResponse, isTextContentType } from "../utils/response"
 import type { RouteLike } from "../utils/sanitize"
 import { routeContinueOverrides } from "../utils/sanitize"
 
@@ -60,14 +61,7 @@ export async function runTier2(
       })
     }
 
-    let statusCode = 200
-    const mainResponseHolder: { value?: MinimalResponse } = {}
-    page.on("response", (res: MinimalResponse) => {
-      if (res.url() === url) {
-        statusCode = res.status()
-        if (!mainResponseHolder.value) mainResponseHolder.value = res
-      }
-    })
+    const mainResponse = trackMainDocumentResponses(page)
 
     await page.goto(url, { waitUntil: "domcontentloaded", timeout: maxTimeout })
     await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {})
@@ -83,7 +77,7 @@ export async function runTier2(
       }
     }
 
-    if (isCloudflarePage(html, {})) {
+    if (isCloudflarePage(html, mainResponse.headers)) {
       return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" }
     }
 
@@ -93,8 +87,8 @@ export async function runTier2(
       return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" }
     }
 
-    if (isBlocked(statusCode, html)) {
-      return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` }
+    if (isBlocked(mainResponse.status, html)) {
+      return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` }
     }
 
     // Attempt to solve any embedded captcha widgets (Turnstile, reCAPTCHA, hCaptcha).
@@ -106,9 +100,14 @@ export async function runTier2(
       captchasSolved = result.solved
     }
 
+    const finalHtml = await page.content()
+    if (isCloudflarePage(finalHtml, mainResponse.headers)) {
+      return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" }
+    }
+
     const cookies: Cookie[] = toCookies(await activeContext.cookies())
 
-    const captured = await captureResponse(mainResponseHolder.value)
+    const captured = await captureResponse(mainResponse.response)
 
     return {
       tier: 2,
@@ -117,10 +116,10 @@ export async function runTier2(
       effectiveUrl: page.url(),
       // For HTML/text content-types, `html` is the rendered DOM. For binary, leave
       // empty so /scrape consumers know to use `body`/`contentType`.
-      html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(html) : "",
+      html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(finalHtml) : "",
       ...captured,
       cookies,
-      statusCode,
+      statusCode: mainResponse.status,
       captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined,
     }
   } catch (err) {

packages/tiers/src/tiers/3.ts

@@ -15,8 +15,9 @@ import {
 } from "../utils/detect"
 import { normalizeHtml } from "../utils/html"
 import { waitForImpervaResolution } from "../utils/impervaWait"
+import { trackMainDocumentResponses } from "../utils/mainResponse"
 import { isHardNetworkFailure } from "../utils/network"
-import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response"
+import { captureResponse, isTextContentType } from "../utils/response"
 import type { RouteLike } from "../utils/sanitize"
 import { routeContinueOverrides } from "../utils/sanitize"
 
@@ -64,17 +65,7 @@ export async function runTier3(
       })
     }
 
-    let statusCode = 200
-    const mainResponseHolder: { value?: MinimalResponse } = {}
-    page.on("response", (res: MinimalResponse) => {
-      try {
-        const resUrl = res.url()
-        if (resUrl === url || resUrl.startsWith(url.replace(/\/$/, ""))) {
-          statusCode = res.status()
-          if (!mainResponseHolder.value) mainResponseHolder.value = res
-        }
-      } catch {}
-    })
+    const mainResponse = trackMainDocumentResponses(page)
 
     // CF challenges can trigger sub-navigations that throw "navigation interrupted" —
     // we catch those so we can continue. Hard failures (DNS, connection refused) are
@@ -100,7 +91,7 @@ export async function runTier3(
         ? await waitForImpervaResolution(page, remaining, url)
         : challengeType === "akamai"
           ? await waitForAkamaiResolution(page, remaining, url)
-          : await waitForChallengeResolution(page, remaining, url)
+          : await waitForChallengeResolution(page, remaining, url, () => mainResponse.headers)
 
     if (resolution !== "ok") {
       return {
@@ -150,7 +141,7 @@ export async function runTier3(
       return { tier: 3, status: "error", durationMs: Date.now() - start, reason: errMsg }
     }
 
-    if (isCloudflarePage(html, {})) {
+    if (isCloudflarePage(html, mainResponse.headers)) {
       const pageTitle = await page.title().catch(() => "?")
       const pageUrl = page.url()
       console.log(`[tier3] cloudflare-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`)
@@ -171,13 +162,13 @@ export async function runTier3(
       return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" }
     }
 
-    if (isBlocked(statusCode, html)) {
-      return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` }
+    if (isBlocked(mainResponse.status, html)) {
+      return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` }
     }
 
     const cookies: Cookie[] = toCookies(await freshCtx.cookies())
 
-    const captured = await captureResponse(mainResponseHolder.value)
+    const captured = await captureResponse(mainResponse.response)
 
     return {
       tier: 3,
@@ -188,7 +179,7 @@ export async function runTier3(
       ...captured,
       cookies,
       userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent),
-      statusCode,
+      statusCode: mainResponse.status,
       captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined,
     }
   } catch (err) {

packages/tiers/src/tiers/4.ts

@@ -15,8 +15,9 @@ import {
 } from "../utils/detect"
 import { normalizeHtml } from "../utils/html"
 import { waitForImpervaResolution } from "../utils/impervaWait"
+import { trackMainDocumentResponses } from "../utils/mainResponse"
 import { isHardNetworkFailure } from "../utils/network"
-import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response"
+import { captureResponse, isTextContentType } from "../utils/response"
 import type { RouteLike } from "../utils/sanitize"
 import { routeContinueOverrides } from "../utils/sanitize"
 
@@ -66,16 +67,7 @@ export async function runTier4(
       })
     }
 
-    let statusCode = 200
-    const mainResponseHolder: { value?: MinimalResponse } = {}
-    page.on("response", (res: MinimalResponse) => {
-      try {
-        if (res.url() === url || res.url().startsWith(url.replace(/\/$/, ""))) {
-          statusCode = res.status()
-          if (!mainResponseHolder.value) mainResponseHolder.value = res
-        }
-      } catch {}
-    })
+    const mainResponse = trackMainDocumentResponses(page)
 
     const gotoErr = await page
       .goto(url, {
@@ -96,7 +88,7 @@ export async function runTier4(
         ? await waitForImpervaResolution(page, remaining, url)
         : challengeType === "akamai"
           ? await waitForAkamaiResolution(page, remaining, url)
-          : await waitForChallengeResolution(page, remaining, url)
+          : await waitForChallengeResolution(page, remaining, url, () => mainResponse.headers)
 
     if (resolution !== "ok") {
       return {
@@ -136,7 +128,7 @@ export async function runTier4(
       }
     }
 
-    if (isCloudflarePage(html, {})) {
+    if (isCloudflarePage(html, mainResponse.headers)) {
       return {
         tier: 4,
         status: "blocked",
@@ -163,13 +155,13 @@ export async function runTier4(
       }
     }
 
-    if (isBlocked(statusCode, html)) {
-      return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` }
+    if (isBlocked(mainResponse.status, html)) {
+      return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` }
     }
 
     const cookies: Cookie[] = toCookies(await proxyContext.cookies())
 
-    const captured = await captureResponse(mainResponseHolder.value)
+    const captured = await captureResponse(mainResponse.response)
 
     return {
       tier: 4,
@@ -180,7 +172,7 @@ export async function runTier4(
       ...captured,
       cookies,
       userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent),
-      statusCode,
+      statusCode: mainResponse.status,
       captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined,
     }
   } catch (err) {

packages/tiers/src/utils/challengeWait.ts

@@ -13,10 +13,13 @@ export async function waitForChallengeResolution(
   page: Page,
   timeoutMs: number,
   originalUrl?: string,
+  responseHeaders: () => Record<string, string> = () => ({}),
 ): Promise<"ok" | "ip-blocked" | "timeout"> {
   const deadline = Date.now() + Math.max(timeoutMs, 30_000)
   let lastClickAttempt = 0
   let cfClearanceAt: number | undefined
+  let challengeSeen = false
+  let inactiveSamples = 0
 
   // Only count cf_clearance for the current domain — warm browser may have cookies from prior domains
   const targetHost = (() => {
@@ -27,25 +30,36 @@ export async function waitForChallengeResolution(
     }
   })()
 
-  const earlyTitle = await page.title().catch(() => "")
-  if (earlyTitle && !CF_CHALLENGE_TITLE.test(earlyTitle)) {
-    await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {})
-    return "ok"
-  }
-
   // Let CF's challenge JS boot up before we start polling
-  await new Promise((r) => setTimeout(r, 1000))
+  await new Promise((r) => setTimeout(r, 300))
 
   while (Date.now() < deadline) {
     try {
-      const title = await page.title().catch(() => "just a moment")
-      if (!CF_CHALLENGE_TITLE.test(title)) {
+      const title = await page.title().catch(() => "")
+      const html = await page.content().catch(() => "")
+      const url = page.url()
+      const hasChallengeFrame = page.frames().some(isChallengeFrame)
+      const active =
+        CF_CHALLENGE_TITLE.test(title) ||
+        isCloudflarePage(html, responseHeaders()) ||
+        /\/cdn-cgi\/challenge-platform|\/cdn-cgi\/challenge\//i.test(url) ||
+        hasChallengeFrame
+
+      if (!active) {
+        inactiveSamples++
+      } else {
+        challengeSeen = true
+        inactiveSamples = 0
+      }
+
+      // Two observations prevent a transient normal title/DOM during navigation from
+      // declaring a challenge solved. This also applies before the first active sample.
+      if (inactiveSamples >= 2) {
         // 'load' not 'networkidle' — networkidle stalls indefinitely on JS-heavy pages
         await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {})
         return "ok"
       }
 
-      const url = page.url()
       if (/\/cdn-cgi\/error\/|error=1020|error=1015/.test(url)) return "ip-blocked"
 
       const cookies: Array<{ name: string; domain: string }> = await page
@@ -60,7 +74,7 @@ export async function waitForChallengeResolution(
             c.domain === `.${targetHost}` ||
             targetHost.endsWith(c.domain.replace(/^\./, ""))),
       )
-      if (hasDomainClearance) {
+      if (hasDomainClearance && challengeSeen) {
         if (cfClearanceAt === undefined) {
           cfClearanceAt = Date.now()
           console.log("[challenge] cf_clearance obtained")

packages/tiers/src/utils/detect.ts

@@ -9,7 +9,8 @@ export type ChallengeType =
   | "none"
 
 export function isCloudflarePage(html: string, headers: Record<string, string>): boolean {
-  if (headers["cf-mitigated"]) return true
+  const cfMitigated = Object.entries(headers).find(([name]) => name.toLowerCase() === "cf-mitigated")?.[1]
+  if (cfMitigated?.toLowerCase() === "challenge") return true
   if (/<title>[^<]*(just a moment|ddos-guard|please wait|checking|attention required)[^<]*<\/title>/i.test(html))
     return true
   if (/checking your browser/i.test(html)) return true
@@ -20,6 +21,11 @@ export function isCloudflarePage(html: string, headers: Record<string, string>):
   if (/id="cf-challenge-running"/i.test(html)) return true
   // CF Turnstile interstitial wrapper
   if (/id="turnstile-wrapper"/i.test(html)) return true
+  // Active challenge orchestration markers. Unlike the passive telemetry markers
+  // below, these only occur while Cloudflare is serving an interstitial.
+  if (/_cf_chl_opt/i.test(html)) return true
+  if (/id=["']challenge-form["']/i.test(html)) return true
+  if (/orchestrate\/chl_page/i.test(html)) return true
   // DDoS-Guard
   if (/ddos-guard\.net|\.ddos-guard\.net/i.test(html)) return true
   // CF firewall/WAF deny page (error 1020 and friends) — static "blocked" page, not a
@@ -35,7 +41,8 @@ export function isCloudflarePage(html: string, headers: Record<string, string>):
   // bot-management telemetry. Matching on the marker alone flags real pages as blocked.
   // The actual challenge stub is always near-empty (nothing else can render before the
   // challenge resolves), so gate on page size too.
-  if (html.length < 3000 && /__CF\$cv\$params/i.test(html)) return true
+  if (html.length < 3000 && /__CF\$cv\$params|\/cdn-cgi\/challenge-platform\/[^"']*jsd\/main\.js/i.test(html))
+    return true
   return false
 }
 

packages/tiers/src/utils/mainResponse.ts

@@ -0,0 +1,40 @@
+import type { Page, Response } from "patchright"
+
+import type { MinimalResponse } from "./response"
+
+type NavigationResponse = MinimalResponse & Pick<Response, "request">
+
+/** Tracks the latest top-level document response across redirects. */
+export class MainDocumentResponseTracker {
+  private latest?: NavigationResponse
+
+  constructor(private readonly page: Pick<Page, "mainFrame">) {}
+
+  observe(response: NavigationResponse): void {
+    try {
+      const request = response.request()
+      if (!request.isNavigationRequest() || request.frame() !== this.page.mainFrame()) return
+      this.latest = response
+    } catch {
+      // A response can disappear while Firefox is replacing a challenge document.
+    }
+  }
+
+  get response(): MinimalResponse | undefined {
+    return this.latest
+  }
+
+  get status(): number {
+    return this.latest?.status() ?? 200
+  }
+
+  get headers(): Record<string, string> {
+    return this.latest?.headers() ?? {}
+  }
+}
+
+export function trackMainDocumentResponses(page: Page): MainDocumentResponseTracker {
+  const tracker = new MainDocumentResponseTracker(page)
+  page.on("response", (response) => tracker.observe(response))
+  return tracker
+}

packages/tiers/tests/cloudflareDetection.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, test } from "bun:test"
+import { isCloudflarePage } from "../src/utils/detect"
+
+describe("Cloudflare challenge detection", () => {
+  test("treats cf-mitigated challenge as authoritative regardless of case or page title", () => {
+    const html = `<html><title>Welcome</title><body>${"content ".repeat(1000)}</body></html>`
+    expect(isCloudflarePage(html, { "CF-Mitigated": "Challenge" })).toBe(true)
+  })
+
+  test("detects active orchestration markers on large pages", () => {
+    const padding = "content ".repeat(1000)
+    expect(isCloudflarePage(`<script>window._cf_chl_opt={}</script>${padding}`, {})).toBe(true)
+    expect(isCloudflarePage(`<form id="challenge-form"></form>${padding}`, {})).toBe(true)
+    expect(
+      isCloudflarePage(`<script src="/cdn-cgi/challenge-platform/orchestrate/chl_page/v1"></script>${padding}`, {}),
+    ).toBe(true)
+  })
+
+  test("allows passive Cloudflare telemetry on an ordinary large page", () => {
+    const html = `<html><title>Real page</title><body>${"content ".repeat(1000)}<script>__CF$cv$params={}</script><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></body></html>`
+    expect(isCloudflarePage(html, {})).toBe(false)
+  })
+})

packages/tiers/tests/mainResponse.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, test } from "bun:test"
+import { MainDocumentResponseTracker } from "../src/utils/mainResponse"
+import { captureResponse } from "../src/utils/response"
+
+const mainFrame = {}
+const iframe = {}
+const page = { mainFrame: () => mainFrame }
+
+function response(
+  url: string,
+  status: number,
+  headers: Record<string, string>,
+  body: string,
+  options: { navigation?: boolean; frame?: object } = {},
+) {
+  return {
+    url: () => url,
+    status: () => status,
+    headers: () => headers,
+    body: async () => Buffer.from(body),
+    request: () => ({
+      isNavigationRequest: () => options.navigation ?? true,
+      frame: () => options.frame ?? mainFrame,
+    }),
+  }
+}
+
+describe("MainDocumentResponseTracker", () => {
+  test("keeps the final response across cross-origin redirects with coherent metadata", async () => {
+    const tracker = new MainDocumentResponseTracker(page as never)
+    tracker.observe(
+      response("https://a.example/start", 302, { location: "https://b.example/final" }, "redirect") as never,
+    )
+    tracker.observe(
+      response("https://b.example/final", 200, { "content-type": "text/html", "x-final": "yes" }, "origin") as never,
+    )
+
+    expect(tracker.status).toBe(200)
+    expect(tracker.headers["x-final"]).toBe("yes")
+    const captured = await captureResponse(tracker.response)
+    expect(new TextDecoder().decode(captured.body)).toBe("origin")
+    expect(captured.contentType).toBe("text/html")
+  })
+
+  test("ignores subresources and iframe navigations", () => {
+    const tracker = new MainDocumentResponseTracker(page as never)
+    tracker.observe(response("https://example.com/", 200, {}, "main") as never)
+    tracker.observe(response("https://example.com/app.js", 404, {}, "script", { navigation: false }) as never)
+    tracker.observe(response("https://challenge.example/frame", 403, {}, "frame", { frame: iframe }) as never)
+
+    expect(tracker.status).toBe(200)
+    expect(tracker.response?.url()).toBe("https://example.com/")
+  })
+
+  test("retains a terminal external-scheme redirect", () => {
+    const tracker = new MainDocumentResponseTracker(page as never)
+    tracker.observe(response("https://example.com/download", 301, { location: "magnet:?xt=urn:test" }, "") as never)
+
+    expect(tracker.status).toBe(301)
+    expect(tracker.headers.location).toBe("magnet:?xt=urn:test")
+  })
+})