External/trawl
1
0
Fork 0
mirror of https://github.com/germondai/trawl.git synced 2026-08-17 12:11:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(tiers): solve imperva challenges in tier3 and tier4

Browse Source
This commit is contained in:
germondai
2026-07-05 16:40:17 +02:00
parent 15cae0ff30
commit 2d8769b4f5
2 changed files with 44 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/tiers/src/tier3.ts
+21 -4
View File
@@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser"
import { FINGERPRINT, newFreshContext } from "@trawl/browser"
import type { Cookie, TierResult } from "@trawl/types"
import { waitForChallengeResolution } from "./challengeWait"
import { isCloudflarePage } from "./detect"
import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
import { normalizeHtml } from "./html"
import { waitForImpervaResolution } from "./impervaWait"
import { solvePageCaptchas } from "./solvers"
export interface Tier3Result extends TierResult {
@@ -75,7 +76,12 @@ export async function runTier3(
}
const remaining = maxTimeout - (Date.now() - start)
const resolution = await waitForChallengeResolution(page, remaining, url)
const peekHtml = await page.content().catch(() => "")
const challengeType = detectChallengeType(peekHtml)
const resolution =
challengeType === "imperva"
? await waitForImpervaResolution(page, remaining, url)
: await waitForChallengeResolution(page, remaining, url)
if (resolution !== "ok") {
return {
@@ -84,8 +90,12 @@ export async function runTier3(
durationMs: Date.now() - start,
reason:
resolution === "ip-blocked"
? "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)"
: "cloudflare-challenge-timeout",
? challengeType === "imperva"
? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)"
: "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)"
: challengeType === "imperva"
? "imperva-challenge-timeout"
: "cloudflare-challenge-timeout",
}
}
@@ -118,6 +128,13 @@ export async function runTier3(
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" }
}
if (hasImpervaChallenge(html)) {
const pageTitle = await page.title().catch(() => "?")
const pageUrl = page.url()
console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`)
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" }
}
const rawCookies = await freshCtx.cookies()
const cookies: Cookie[] = rawCookies.map(
(c: {
packages/tiers/src/tier4.ts
+23 -3
View File
@@ -2,8 +2,9 @@ import type { BrowserHandle } from "@trawl/browser"
import { FINGERPRINT } from "@trawl/browser"
import type { Cookie, TierResult } from "@trawl/types"
import { waitForChallengeResolution } from "./challengeWait"
import { isCloudflarePage } from "./detect"
import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect"
import { normalizeHtml } from "./html"
import { waitForImpervaResolution } from "./impervaWait"
export interface Tier4Result extends TierResult {
tier: 4
@@ -91,14 +92,24 @@ export async function runTier4(
}
const remaining = maxTimeout - (Date.now() - start)
const resolution = await waitForChallengeResolution(page, remaining, url)
const peekHtml = await page.content().catch(() => "")
const challengeType = detectChallengeType(peekHtml)
const resolution =
challengeType === "imperva"
? await waitForImpervaResolution(page, remaining, url)
: await waitForChallengeResolution(page, remaining, url)
if (resolution !== "ok") {
return {
tier: 4,
status: resolution === "ip-blocked" ? "blocked" : "timeout",
durationMs: Date.now() - start,
reason: resolution === "ip-blocked" ? "proxy-ip-blocked" : "cloudflare-challenge-timeout",
reason:
resolution === "ip-blocked"
? "proxy-ip-blocked"
: challengeType === "imperva"
? "imperva-challenge-timeout"
: "cloudflare-challenge-timeout",
}
}
@@ -119,6 +130,15 @@ export async function runTier4(
}
}
if (hasImpervaChallenge(html)) {
return {
tier: 4,
status: "blocked",
durationMs: Date.now() - start,
reason: "imperva-persistent",
}
}
const rawCookies = await proxyContext.cookies()
const cookies: Cookie[] = rawCookies.map(
(c: {