diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a982a1..15c6177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Detect and resolve Akamai Bot Manager behavioral interstitials across scraper tiers and the HTTP/HTTPS proxy (#33). + ### Fixed - Reject non-object request bodies and missing, non-string, or blank `url` values with HTTP 400 before scraper-tier execution (#34). diff --git a/apps/api/src/proxy/__tests__/directForward.test.ts b/apps/api/src/proxy/__tests__/directForward.test.ts index f1f9484..9a51685 100644 --- a/apps/api/src/proxy/__tests__/directForward.test.ts +++ b/apps/api/src/proxy/__tests__/directForward.test.ts @@ -31,6 +31,10 @@ const fetchFixture = (req: Request): Response => { headers: { "Content-Encoding": "gzip", "Content-Type": "text/html; charset=utf-8" }, }) } + if (pathname === "/akamai-challenge") + return new Response('
', { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }) if (pathname === "/video") return new Response(chunked(Buffer.from([0, 1, 2, 3]), Buffer.from([4, 5, 6, 7])), { headers: { "Content-Type": "video/mp4" }, @@ -172,6 +176,19 @@ describe("directForwardHttp — buffered by default", () => { expect(result.headers["content-encoding"]).toBe("gzip") }) + test("detects a 200 Akamai behavioral interstitial", async () => { + const result = await directForwardHttp({ + url: `${baseUrl}/akamai-challenge`, + method: "GET", + headers: {}, + }) + + expect(result.mode).toBe("buffer") + if (result.mode !== "buffer") return + expect(result.status).toBe(200) + expect(result.challengeDetected).toBe(true) + }) + test("streams explicit video responses", async () => { const result = await directForwardHttp({ url: `${baseUrl}/video`, diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index b3fff87..23aad7d 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -8,6 +8,7 @@ export { runTier4, type Tier4Result } from "./tiers/4" export { type ChallengeType, detectChallengeType, + hasAkamaiChallenge, hasHcaptcha, hasImpervaChallenge, hasRecaptcha, diff --git a/packages/tiers/src/tiers/1.ts b/packages/tiers/src/tiers/1.ts index 4d6fc9d..9252cd0 100644 --- a/packages/tiers/src/tiers/1.ts +++ b/packages/tiers/src/tiers/1.ts @@ -1,6 +1,13 @@ import { FINGERPRINT } from "@trawl/browser" import type { TierResult } from "@trawl/types" -import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "../utils/detect" +import { + hasAkamaiChallenge, + hasHcaptcha, + hasRecaptcha, + hasTurnstile, + isBlocked, + isCloudflarePage, +} from "../utils/detect" import { normalizeHtml } from "../utils/html" import { isTextContentType } from "../utils/response" @@ -111,6 +118,18 @@ export async function runTier1( statusCode: res.status, } } + if (hasAkamaiChallenge(previewText, responseHeaders)) { + return { + tier: 1, + status: "needs-js", + durationMs: Date.now() - start, + reason: "akamai-interstitial", + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } + } if (isBlocked(res.status, previewText)) { return { diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index f8ce3a2..3102deb 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -2,7 +2,7 @@ import type { BrowserHandle, PersistentBrowserContext } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" import { normalizeSameSite, toCookies } from "../utils/cookies" -import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" +import { hasAkamaiChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" import { normalizeHtml } from "../utils/html" import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" import type { RouteLike } from "../utils/sanitize" @@ -111,6 +111,12 @@ export async function runTier2( return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } + // A cached session that lands back on Akamai's interstitial is stale — force a + // fresh Tier-3 solve rather than returning the ~2KB challenge stub as content. + if (hasAkamaiChallenge(html)) { + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" } + } + if (isBlocked(statusCode, html)) { return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 356886d..19234bf 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -2,10 +2,12 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" +import { waitForAkamaiResolution } from "../utils/akamaiWait" import { waitForChallengeResolution } from "../utils/challengeWait" import { toCookies } from "../utils/cookies" import { detectChallengeType, + hasAkamaiChallenge, hasImpervaChallenge, isBlocked, isBrowserErrorPage, @@ -90,7 +92,9 @@ export async function runTier3( const resolution = challengeType === "imperva" ? await waitForImpervaResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : challengeType === "akamai" + ? await waitForAkamaiResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -101,10 +105,10 @@ export async function runTier3( resolution === "ip-blocked" ? challengeType === "imperva" ? "datacenter-ip-blocked (imperva sensor cookie obtained but challenge persisted — needs residential proxy)" - : "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" - : challengeType === "imperva" - ? "imperva-challenge-timeout" - : "cloudflare-challenge-timeout", + : challengeType === "akamai" + ? "datacenter-ip-blocked (Akamai sensor cookie obtained but challenge persisted — needs residential proxy)" + : "datacenter-ip-blocked (cf_clearance obtained but redirect never completed — needs residential proxy)" + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, } } @@ -154,6 +158,13 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } } + if (hasAkamaiChallenge(html)) { + const pageTitle = await page.title().catch(() => "?") + const pageUrl = page.url() + console.log(`[tier3] akamai-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" } + } + if (isBlocked(statusCode, html)) { return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index e83ade0..e0089fb 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -2,10 +2,12 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" +import { waitForAkamaiResolution } from "../utils/akamaiWait" import { waitForChallengeResolution } from "../utils/challengeWait" import { toCookies } from "../utils/cookies" import { detectChallengeType, + hasAkamaiChallenge, hasImpervaChallenge, isBlocked, isBrowserErrorPage, @@ -108,7 +110,9 @@ export async function runTier4( const resolution = challengeType === "imperva" ? await waitForImpervaResolution(page, remaining, url) - : await waitForChallengeResolution(page, remaining, url) + : challengeType === "akamai" + ? await waitForAkamaiResolution(page, remaining, url) + : await waitForChallengeResolution(page, remaining, url) if (resolution !== "ok") { return { @@ -118,9 +122,7 @@ export async function runTier4( reason: resolution === "ip-blocked" ? "proxy-ip-blocked" - : challengeType === "imperva" - ? "imperva-challenge-timeout" - : "cloudflare-challenge-timeout", + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, } } @@ -168,6 +170,15 @@ export async function runTier4( } } + if (hasAkamaiChallenge(html)) { + return { + tier: 4, + status: "blocked", + durationMs: Date.now() - start, + reason: "akamai-persistent", + } + } + if (isBlocked(statusCode, html)) { return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } diff --git a/packages/tiers/src/utils/akamaiWait.ts b/packages/tiers/src/utils/akamaiWait.ts new file mode 100644 index 0000000..6360ece --- /dev/null +++ b/packages/tiers/src/utils/akamaiWait.ts @@ -0,0 +1,129 @@ +import type { Page } from "patchright" +import { hasAkamaiChallenge } from "./detect" + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +const hostnameFor = (url?: string) => { + try { + return url ? new URL(url).hostname : "" + } catch { + return "" + } +} + +export async function waitForAkamaiResolution( + page: Page, + timeoutMs: number, + originalUrl?: string, +): Promise<"ok" | "ip-blocked" | "timeout"> { + const deadline = Date.now() + Math.max(timeoutMs, 0) + const targetHost = hostnameFor(originalUrl ?? page.url()) + + const early = await page.content().catch(() => "") + if (early && !hasAkamaiChallenge(early)) { + await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => {}) + return "ok" + } + if (Date.now() >= deadline) return "timeout" + + await sleep(Math.min(1500, Math.max(deadline - Date.now(), 0))) + await wanderMouse(page, 8, deadline) + + let heldOnce = false + let navigatedOnce = false + let sawCookieAt: number | undefined + + while (Date.now() < deadline) { + const html = await page.content().catch(() => "") + if (html && !hasAkamaiChallenge(html)) { + await page.waitForLoadState("load", { timeout: 5000 }).catch(() => {}) + return "ok" + } + + const hasAbck = await pageHasAkamaiCookie(page, targetHost) + if (hasAbck && sawCookieAt === undefined) { + sawCookieAt = Date.now() + console.log("[akamai] _abck cookie present") + } + if (hasAbck && !navigatedOnce && originalUrl && sawCookieAt && Date.now() - sawCookieAt > 6000) { + navigatedOnce = true + console.log("[akamai] cookie set but still on interstitial — navigating to original URL") + await page.goto(originalUrl, { waitUntil: "domcontentloaded", timeout: 15_000 }).catch(() => {}) + await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => {}) + const resolvedHtml = await page.content().catch(() => "") + return resolvedHtml && !hasAkamaiChallenge(resolvedHtml) ? "ok" : "ip-blocked" + } + + if (!heldOnce && deadline - Date.now() > 6000) heldOnce = await pressAndHold(page) + + await wanderMouse(page, 3, deadline) + await sleep(Math.min(600, Math.max(deadline - Date.now(), 0))) + } + + return "timeout" +} + +async function pageHasAkamaiCookie(page: Page, targetHost: string): Promise { + const cookies: Array<{ name: string; value: string; domain: string }> = await page + .context() + .cookies() + .catch(() => []) + const abck = cookies.find( + (cookie) => + cookie.name === "_abck" && + targetHost && + (cookie.domain === targetHost || + cookie.domain === `.${targetHost}` || + targetHost.endsWith(cookie.domain.replace(/^\./, ""))), + ) + if (!abck) return false + const seg = abck.value.split("~")[1] + return seg !== undefined && seg !== "-1" +} + +async function wanderMouse(page: Page, points: number, deadline: number): Promise { + try { + const vp = page.viewportSize() || { width: 1280, height: 800 } + let x = Math.random() * vp.width + let y = Math.random() * vp.height + for (let i = 0; i < points; i++) { + if (Date.now() >= deadline) break + const nx = Math.max(2, Math.min(vp.width - 2, x + (Math.random() - 0.5) * vp.width * 0.5)) + const ny = Math.max(2, Math.min(vp.height - 2, y + (Math.random() - 0.5) * vp.height * 0.5)) + await page.mouse.move(nx, ny, { steps: 4 + Math.floor(Math.random() * 8) }) + x = nx + y = ny + await sleep(Math.min(60 + Math.random() * 140, Math.max(deadline - Date.now(), 0))) + } + } catch { + // Navigation can temporarily invalidate the input target. + } +} + +async function pressAndHold(page: Page): Promise { + const sel = "#progress-button, .behavioral-button, #sec-if-cpt-container [role='button']" + let mouseDown = false + try { + const el = page.locator(sel).first() + if (!(await el.isVisible({ timeout: 500 }).catch(() => false))) return false + const box = await el.boundingBox().catch(() => undefined) + if (!box || box.width < 4 || box.height < 4) return false + const cx = box.x + box.width / 2 + const cy = box.y + box.height / 2 + await page.mouse.move(cx - 18, cy - 10, { steps: 6 }) + await page.mouse.move(cx, cy, { steps: 8 }) + await page.mouse.down() + mouseDown = true + const holdUntil = Date.now() + 5500 + while (Date.now() < holdUntil) { + await page.mouse.move(cx + (Math.random() - 0.5) * 3, cy + (Math.random() - 0.5) * 3, { steps: 2 }) + await sleep(220) + } + console.log("[akamai] press-and-hold performed") + return true + } catch { + return false + } finally { + if (mouseDown) await page.mouse.up().catch(() => {}) + } +} diff --git a/packages/tiers/src/utils/detect.ts b/packages/tiers/src/utils/detect.ts index 6d50ee4..65051a0 100644 --- a/packages/tiers/src/utils/detect.ts +++ b/packages/tiers/src/utils/detect.ts @@ -5,6 +5,7 @@ export type ChallengeType = | "recaptcha" | "cap" | "imperva" + | "akamai" | "none" export function isCloudflarePage(html: string, headers: Record): boolean { @@ -83,10 +84,28 @@ export function hasImpervaChallenge(html: string, headers: Record