diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index d22c26e..b2f1f2d 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import { waitForImpervaResolution } from "./impervaWait" import { solvePageCaptchas } from "./solvers" export interface Tier3Result extends TierResult { @@ -75,7 +76,12 @@ export async function runTier3( } const remaining = maxTimeout - (Date.now() - start) - const resolution = await waitForChallengeResolution(page, remaining, url) + const peekHtml = await page.content().catch(() => "") + const challengeType = detectChallengeType(peekHtml) + const resolution = + challengeType === "imperva" + ? await waitForImpervaResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -84,8 +90,12 @@ export async function runTier3( durationMs: Date.now() - start, reason: resolution === "ip-blocked" - ? "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" - : "cloudflare-challenge-timeout", + ? challengeType === "imperva" + ? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)" + : "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" + : challengeType === "imperva" + ? "imperva-challenge-timeout" + : "cloudflare-challenge-timeout", } } @@ -118,6 +128,13 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" } } + if (hasImpervaChallenge(html)) { + const pageTitle = await page.title().catch(() => "?") + const pageUrl = page.url() + console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } + } + const rawCookies = await freshCtx.cookies() const cookies: Cookie[] = rawCookies.map( (c: { diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index 3e72d0a..941d7c8 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" +import { waitForImpervaResolution } from "./impervaWait" export interface Tier4Result extends TierResult { tier: 4 @@ -91,14 +92,24 @@ export async function runTier4( } const remaining = maxTimeout - (Date.now() - start) - const resolution = await waitForChallengeResolution(page, remaining, url) + const peekHtml = await page.content().catch(() => "") + const challengeType = detectChallengeType(peekHtml) + const resolution = + challengeType === "imperva" + ? await waitForImpervaResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { tier: 4, status: resolution === "ip-blocked" ? "blocked" : "timeout", durationMs: Date.now() - start, - reason: resolution === "ip-blocked" ? "proxy-ip-blocked" : "cloudflare-challenge-timeout", + reason: + resolution === "ip-blocked" + ? "proxy-ip-blocked" + : challengeType === "imperva" + ? "imperva-challenge-timeout" + : "cloudflare-challenge-timeout", } } @@ -119,6 +130,15 @@ export async function runTier4( } } + if (hasImpervaChallenge(html)) { + return { + tier: 4, + status: "blocked", + durationMs: Date.now() - start, + reason: "imperva-persistent", + } + } + const rawCookies = await proxyContext.cookies() const cookies: Cookie[] = rawCookies.map( (c: {