From 8e2802fd9741a0211fdb034be6aaa332f6058579 Mon Sep 17 00:00:00 2001 From: germondai Date: Thu, 4 Jun 2026 09:05:00 +0200 Subject: [PATCH] feat: implement tier 3 stealth browser with fingerprinting --- packages/tiers/src/tier3.ts | 154 ++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 packages/tiers/src/tier3.ts diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts new file mode 100644 index 0000000..2fee53b --- /dev/null +++ b/packages/tiers/src/tier3.ts @@ -0,0 +1,154 @@ +import type { BrowserHandle } from "@trawl/browser" +import { FINGERPRINT, newFreshContext } from "@trawl/browser" +import type { Cookie, TierResult } from "@trawl/types" +import { waitForChallengeResolution } from "./challengeWait" +import { isCloudflarePage } from "./detect" +import { normalizeHtml } from "./html" +import { solvePageCaptchas } from "./solvers" + +export interface Tier3Result extends TierResult { + tier: 3 + html?: string + cookies?: Cookie[] + userAgent?: string + statusCode?: number + captchasSolved?: string[] +} + +export async function runTier3( + url: string, + handle: BrowserHandle, + maxTimeout: number, + _proxyUrl?: string, +): Promise { + const start = Date.now() + + // CRITICAL: Use a fresh browser context for CF challenge solving. + // A warm/reused context carries accumulated state (localStorage, service workers, JS + // engine state) that CF's behavioral analysis scores as suspicious — resulting in 40s + // challenge evaluation. A fresh context with no prior state gets managed-mode treatment: + // CF evaluates in under 1s and the challenge resolves in 3-4s total. + const freshCtx = await newFreshContext(handle.browser) + const page = await freshCtx.newPage() + + try { + let statusCode = 200 + page.on("response", (res: { url(): string; status(): number }) => { + try { + const resUrl = res.url() + if (resUrl === url || resUrl.startsWith(url.replace(/\/$/, ""))) { + statusCode = res.status() + } + } catch {} + }) + + // CF challenges can trigger sub-navigations that throw "navigation interrupted" — + // we catch those so we can continue. Hard failures (DNS, connection refused) are + // rethrown so they surface as proper errors. + const gotoErr = await page + .goto(url, { + waitUntil: "domcontentloaded", + timeout: Math.min(maxTimeout, 30_000), + }) + .catch((e: Error) => e) + + // Abort early on hard network failures — no point running challenge wait + if (gotoErr instanceof Error) { + const msg = gotoErr.message + const isHardFail = + /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED/i.test(msg) + if (isHardFail) { + return { tier: 3, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] } + } + // Otherwise (navigation interrupted by CF redirect) — fall through and keep going + } + + const remaining = maxTimeout - (Date.now() - start) + const resolution = await waitForChallengeResolution(page, remaining, url) + + if (resolution !== "ok") { + return { + tier: 3, + status: resolution === "ip-blocked" ? "blocked" : "timeout", + durationMs: Date.now() - start, + reason: + resolution === "ip-blocked" + ? "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" + : "cloudflare-challenge-timeout", + } + } + + // challengeWait calls waitForLoadState('load') but the CF interstitial iframe can + // linger in page.frames() briefly after navigation. Give it 600ms to clear so the + // captcha solver doesn't mistake the just-solved interstitial for an in-page widget. + await new Promise((r) => setTimeout(r, 600)) + + // Attempt to solve any embedded captcha widgets on the page (Turnstile, reCaptcha, hCaptcha). + // This handles sites where the page itself loads fine but has an in-page challenge widget. + const solveRemaining = maxTimeout - (Date.now() - start) + let captchasSolved: string[] = [] + if (solveRemaining > 5000) { + const solveResult = await solvePageCaptchas(page, solveRemaining).catch(() => ({ attempted: [], solved: [] })) + captchasSolved = solveResult.solved + } + + const html = await page.content() + + // Empty shell means the browser got nothing — treat as a load failure + if (html.length < 100) { + const errMsg = gotoErr instanceof Error ? gotoErr.message.split("\n")[0] : "page returned empty content" + return { tier: 3, status: "error", durationMs: Date.now() - start, reason: errMsg } + } + + if (isCloudflarePage(html, {})) { + const pageTitle = await page.title().catch(() => "?") + const pageUrl = page.url() + console.log(`[tier3] cloudflare-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" } + } + + const rawCookies = await freshCtx.cookies() + const cookies: Cookie[] = rawCookies.map( + (c: { + name: string + value: string + domain: string + path: string + expires: number + httpOnly: boolean + secure: boolean + sameSite?: string + }) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + expires: c.expires ?? -1, + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: c.sameSite, + }), + ) + + return { + tier: 3, + status: "success", + durationMs: Date.now() - start, + html: normalizeHtml(html), + cookies, + userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), + statusCode, + captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, + } + } catch (err) { + return { + tier: 3, + status: "error", + durationMs: Date.now() - start, + reason: err instanceof Error ? err.message : String(err), + } + } finally { + await page.close().catch(() => {}) + await freshCtx.close().catch(() => {}) + } +}