From 4ac9e74babffc0c98029f42dcb63b31a38b2a73e Mon Sep 17 00:00:00 2001 From: Erik Dasque Date: Sun, 19 Jul 2026 00:17:17 -0400 Subject: [PATCH] feat(tiers): add Akamai Bot Manager (behavioral / sec-cpt) challenge support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit trawl returned some Akamai-fronted pages as 200 'success' with only the ~2KB sec-cpt behavioral interstitial as content, because tier detection knew Cloudflare/Imperva but not Akamai. - detect.ts: hasAkamaiChallenge() + 'akamai' ChallengeType (sec-if-cpt-container / behavioral-content markers, size-gated sensor fallback); wired into detectChallengeType/isBlocked/needsJs. - akamaiWait.ts (new): Akamai analogue of challengeWait/impervaWait — drives human-like mouse motion, press-and-hold on the behavioral widget, waits for the sensor's location.reload() into real content. - tiers 1-4: escalate the 200 interstitial (needs-js), invalidate a stale cached-session interstitial, dispatch the resolver, report akamai-persistent. Additive; Cloudflare/Imperva paths untouched. Verified against Edmunds. --- packages/tiers/src/tiers/1.ts | 14 ++- packages/tiers/src/tiers/2.ts | 8 +- packages/tiers/src/tiers/3.ts | 17 +++- packages/tiers/src/tiers/4.ts | 19 +++- packages/tiers/src/utils/akamaiWait.ts | 133 +++++++++++++++++++++++++ packages/tiers/src/utils/detect.ts | 22 +++- 6 files changed, 202 insertions(+), 11 deletions(-) create mode 100644 packages/tiers/src/utils/akamaiWait.ts diff --git a/packages/tiers/src/tiers/1.ts b/packages/tiers/src/tiers/1.ts index be9337d..2913c3d 100644 --- a/packages/tiers/src/tiers/1.ts +++ b/packages/tiers/src/tiers/1.ts @@ -1,6 +1,13 @@ import { FINGERPRINT } from "@trawl/browser" import type { TierResult } from "@trawl/types" -import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "../utils/detect" +import { + hasAkamaiChallenge, + hasHcaptcha, + hasRecaptcha, + hasTurnstile, + isBlocked, + isCloudflarePage, +} from "../utils/detect" import { normalizeHtml } from "../utils/html" export interface Tier1Result extends TierResult { @@ -56,6 +63,11 @@ export async function runTier1( if (hasTurnstile(html)) { return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "turnstile-shell" } } + // Akamai's behavioral interstitial is served with HTTP 200; escalate to a browser + // tier so the sensor JS runs and akamaiWait can drive the challenge. + if (hasAkamaiChallenge(html, headers)) { + return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "akamai-interstitial" } + } if (isBlocked(res.status, html)) { return { tier: 1, status: "blocked", durationMs: Date.now() - start, reason: `http-${res.status}` } diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index ece1092..ec3704b 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -2,7 +2,7 @@ import type { BrowserHandle } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" import { normalizeSameSite, toCookies } from "../utils/cookies" -import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" +import { hasAkamaiChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" import { normalizeHtml } from "../utils/html" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" @@ -75,6 +75,12 @@ export async function runTier2( return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } + // A cached session that lands back on Akamai's interstitial is stale — force a + // fresh Tier-3 solve rather than returning the ~2KB challenge stub as content. + if (hasAkamaiChallenge(html)) { + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" } + } + if (isBlocked(statusCode, html)) { return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 58e97b4..01d9508 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -2,10 +2,12 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" +import { waitForAkamaiResolution } from "../utils/akamaiWait" import { waitForChallengeResolution } from "../utils/challengeWait" import { toCookies } from "../utils/cookies" import { detectChallengeType, + hasAkamaiChallenge, hasImpervaChallenge, isBlocked, isBrowserErrorPage, @@ -84,7 +86,9 @@ export async function runTier3( const resolution = challengeType === "imperva" ? await waitForImpervaResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : challengeType === "akamai" + ? await waitForAkamaiResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -96,9 +100,7 @@ export async function runTier3( ? challengeType === "imperva" ? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)" : "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" - : challengeType === "imperva" - ? "imperva-challenge-timeout" - : "cloudflare-challenge-timeout", + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, } } @@ -148,6 +150,13 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } } + if (hasAkamaiChallenge(html)) { + const pageTitle = await page.title().catch(() => "?") + const pageUrl = page.url() + console.log(`[tier3] akamai-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" } + } + if (isBlocked(statusCode, html)) { return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index 1beb36e..6164b25 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -2,10 +2,12 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" +import { waitForAkamaiResolution } from "../utils/akamaiWait" import { waitForChallengeResolution } from "../utils/challengeWait" import { toCookies } from "../utils/cookies" import { detectChallengeType, + hasAkamaiChallenge, hasImpervaChallenge, isBlocked, isBrowserErrorPage, @@ -102,7 +104,9 @@ export async function runTier4( const resolution = challengeType === "imperva" ? await waitForImpervaResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : challengeType === "akamai" + ? await waitForAkamaiResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -112,9 +116,7 @@ export async function runTier4( reason: resolution === "ip-blocked" ? "proxy-ip-blocked" - : challengeType === "imperva" - ? "imperva-challenge-timeout" - : "cloudflare-challenge-timeout", + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, } } @@ -162,6 +164,15 @@ export async function runTier4( } } + if (hasAkamaiChallenge(html)) { + return { + tier: 4, + status: "blocked", + durationMs: Date.now() - start, + reason: "akamai-persistent", + } + } + if (isBlocked(statusCode, html)) { return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/utils/akamaiWait.ts b/packages/tiers/src/utils/akamaiWait.ts new file mode 100644 index 0000000..29bf83f --- /dev/null +++ b/packages/tiers/src/utils/akamaiWait.ts @@ -0,0 +1,133 @@ +// Akamai Bot Manager "Behavioral Detection" (sec-cpt / SBSD) resolver — the Akamai +// analogue of challengeWait.ts (Cloudflare) and impervaWait.ts (Imperva). +// +// Akamai's interstitial is a hidden #sec-if-cpt-container ("behavioral-content") +// driven by an obfuscated sensor script. The sensor collects pointer/timing telemetry +// and POSTs it; an inline hook then location.reload()s into the real page. Some +// variants also surface a press-and-hold "progress button" that must be actuated. +// +// Unlike CF/Imperva (which resolve from JS execution + a cookie alone), Akamai's +// behavioral challenge scores *interaction*, so we drive human-like mouse motion and, +// if the hold button becomes visible, press-and-hold it — then wait for the reload. +// +// Known risk: Akamai's behavioral scoring is adversarial to synthetic input; success +// is not guaranteed even from a real browser. Camoufox emits genuine Firefox-level +// pointer events (not CDP synthetic ones), which is the best available shot. + +import type { Page } from "patchright" +import { hasAkamaiChallenge } from "./detect" + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +export async function waitForAkamaiResolution( + page: Page, + timeoutMs: number, + originalUrl?: string, +): Promise<"ok" | "ip-blocked" | "timeout"> { + const deadline = Date.now() + Math.max(timeoutMs, 35_000) + + const early = await page.content().catch(() => "") + if (early && !hasAkamaiChallenge(early)) { + await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {}) + return "ok" + } + + // Let the sensor script boot and start listening for behavioral telemetry. + await sleep(1500) + await wanderMouse(page, 8) + + let heldOnce = false + let navigatedOnce = false + const sawCookieAt: { t: number | null } = { t: null } + + while (Date.now() < deadline) { + const html = await page.content().catch(() => "") + if (html && !hasAkamaiChallenge(html) && html.length > 3500) { + await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {}) + return "ok" + } + + // Once Akamai has set its clearance cookie but the reload hasn't fired, give it a + // grace period then navigate to the original URL ourselves (mirrors CF/Imperva). + const hasAbck = await pageHasAkamaiCookie(page) + if (hasAbck && sawCookieAt.t === null) { + sawCookieAt.t = Date.now() + console.log("[akamai] _abck cookie present") + } + if (hasAbck && !navigatedOnce && originalUrl && sawCookieAt.t && Date.now() - sawCookieAt.t > 6000) { + navigatedOnce = true + console.log("[akamai] cookie set but still on interstitial — navigating to original URL") + await page.goto(originalUrl, { waitUntil: "domcontentloaded", timeout: 15_000 }).catch(() => {}) + await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {}) + continue + } + + // If the behavioral widget surfaced a press-and-hold button, actuate it once. + if (!heldOnce) { + heldOnce = await pressAndHold(page).catch(() => false) + } + + await wanderMouse(page, 3) + await sleep(600) + } + + return "timeout" +} + +async function pageHasAkamaiCookie(page: Page): Promise { + const cookies: Array<{ name: string; value: string }> = await page + .context() + .cookies() + .catch(() => []) + // A validated _abck has its 2nd '~'-segment != "-1"; presence + validation both help. + const abck = cookies.find((c) => c.name === "_abck") + if (!abck) return false + const seg = abck.value.split("~")[1] + return seg !== undefined && seg !== "-1" +} + +// Move the pointer along a wandering path to produce plausible behavioral telemetry. +async function wanderMouse(page: Page, points: number): Promise { + try { + const vp = page.viewportSize() || { width: 1280, height: 800 } + let x = Math.random() * vp.width + let y = Math.random() * vp.height + for (let i = 0; i < points; i++) { + const nx = Math.max(2, Math.min(vp.width - 2, x + (Math.random() - 0.5) * vp.width * 0.5)) + const ny = Math.max(2, Math.min(vp.height - 2, y + (Math.random() - 0.5) * vp.height * 0.5)) + await page.mouse.move(nx, ny, { steps: 4 + Math.floor(Math.random() * 8) }) + x = nx + y = ny + await sleep(60 + Math.random() * 140) + } + } catch { + // page navigating / closed — ignore + } +} + +// Press-and-hold the behavioral "progress button" if the sensor has made it visible. +async function pressAndHold(page: Page): Promise { + const sel = "#progress-button, .behavioral-button, #sec-if-cpt-container [role='button']" + try { + const el = page.locator(sel).first() + if (!(await el.isVisible({ timeout: 500 }).catch(() => false))) return false + const box = await el.boundingBox().catch(() => null) + if (!box || box.width < 4 || box.height < 4) return false + const cx = box.x + box.width / 2 + const cy = box.y + box.height / 2 + await page.mouse.move(cx - 18, cy - 10, { steps: 6 }) + await page.mouse.move(cx, cy, { steps: 8 }) + await page.mouse.down() + // Hold ~5.5s with micro-jitter so the "progress" bar fills. + const holdUntil = Date.now() + 5500 + while (Date.now() < holdUntil) { + await page.mouse.move(cx + (Math.random() - 0.5) * 3, cy + (Math.random() - 0.5) * 3, { steps: 2 }) + await sleep(220) + } + await page.mouse.up() + console.log("[akamai] press-and-hold performed") + return true + } catch { + return false + } +} diff --git a/packages/tiers/src/utils/detect.ts b/packages/tiers/src/utils/detect.ts index a104617..14bce9e 100644 --- a/packages/tiers/src/utils/detect.ts +++ b/packages/tiers/src/utils/detect.ts @@ -5,6 +5,7 @@ export type ChallengeType = | "recaptcha" | "cap" | "imperva" + | "akamai" | "none" export function isCloudflarePage(html: string, headers: Record): boolean { @@ -83,10 +84,28 @@ export function hasImpervaChallenge(html: string, headers: Record