diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 8a5ec65..a5e3e53 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -5,6 +5,7 @@ import { ProxyPool, RequestValidationError, requireContentTypeForBody, + ScrapeError, SUPPORTED_METHODS, type SupportedMethod, sanitizeHeaders, @@ -236,6 +237,9 @@ new Elysia() return flareSolverrError(req.url ?? "", "Browser pool saturated, retry shortly") } set.status = 500 + if (err instanceof ScrapeError) { + return { error: err.message, timings: err.timings } + } return { error: err instanceof Error ? err.message : String(err) } } }) @@ -248,6 +252,20 @@ initPool().catch((err) => { process.exit(1) }) +// Camoufox (Firefox) emits page-error events in a shape playwright-core's dispatcher +// doesn't expect for some target-page JS errors (e.g. missing `error.location`), which +// throws inside the library's own internal event handling — outside any try/catch we +// control, since it fires from a page-level event listener, not from our request path. +// Without this, one target site's malformed error crashes the entire process and drops +// every in-flight request across all clients, not just the one that triggered it. +process.on("uncaughtException", (err) => { + console.error("[api] uncaughtException (continuing):", err instanceof Error ? err.message : err) +}) + +process.on("unhandledRejection", (reason) => { + console.error("[api] unhandledRejection (continuing):", reason instanceof Error ? reason.message : reason) +}) + process.on("SIGTERM", async () => { await pool?.shutdown() process.exit(0) diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index 02c958c..3fd9dfd 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -46,6 +46,8 @@ interface ScrapeResult { sessionCached: boolean // true if a cached session was used timings: TierResult[] // per-tier attempt history totalMs: number + captchasSolved?: string[] // captcha types solved on the page itself (e.g. ['turnstile']) + proxyUsed?: boolean // true if the winning tier routed through a proxy (Tier 3 datacenter pool or Tier 4 residential pool/override) } interface TierResult { @@ -151,4 +153,19 @@ For 429 pool-exhaustion errors, the body is a **FlareSolverr v2 envelope** (same } ``` -For 400 / 503 / 500 the body is the native shape `{ "error": "Human-readable message" }`. +For 400 / 503 the body is the native shape `{ "error": "Human-readable message" }`. + +For 500 errors raised after at least one tier was attempted, the body also includes the +per-tier attempt history, so a failed request is still fully diagnosable from the response +alone — no need to check server logs: + +```json +{ + "error": "All tiers exhausted. Last failure: http-403", + "timings": [ + { "tier": 1, "status": "needs-js", "durationMs": 50, "reason": "cloudflare-challenge" }, + { "tier": 3, "status": "blocked", "durationMs": 2942, "reason": "http-403" }, + { "tier": 4, "status": "blocked", "durationMs": 7890, "reason": "http-403" } + ] +} +``` diff --git a/packages/tiers/src/detect.ts b/packages/tiers/src/detect.ts index 01bc5c1..a104617 100644 --- a/packages/tiers/src/detect.ts +++ b/packages/tiers/src/detect.ts @@ -9,7 +9,8 @@ export type ChallengeType = export function isCloudflarePage(html: string, headers: Record): boolean { if (headers["cf-mitigated"]) return true - if (/[^<]*(just a moment|ddos-guard|please wait|checking)[^<]*<\/title>/i.test(html)) return true + if (/<title>[^<]*(just a moment|ddos-guard|please wait|checking|attention required)[^<]*<\/title>/i.test(html)) + return true if (/checking your browser/i.test(html)) return true if (/enable javascript and cookies to continue/i.test(html)) return true if (/verify you are human/i.test(html)) return true @@ -20,6 +21,31 @@ export function isCloudflarePage(html: string, headers: Record<string, string>): if (/id="turnstile-wrapper"/i.test(html)) return true // DDoS-Guard if (/ddos-guard\.net|\.ddos-guard\.net/i.test(html)) return true + // CF firewall/WAF deny page (error 1020 and friends) — static "blocked" page, not a + // solvable JS challenge, but still needs to be recognized as CF so the orchestrator + // reports tier failure and escalates instead of returning the block page as content + if (/id="cf-error-details"/i.test(html)) return true + if (/you have been blocked/i.test(html)) return true + // Lean CF challenge stub — blank title/body, just the challenge-platform bootstrap + // script. No human-readable text at all, so none of the checks above catch it. + // + // CAUTION: __CF$cv$params is NOT exclusive to active challenges — Cloudflare injects + // the same bootstrap into countless ordinary, fully-rendered pages as passive + // bot-management telemetry. Matching on the marker alone flags real pages as blocked. + // The actual challenge stub is always near-empty (nothing else can render before the + // challenge resolves), so gate on page size too. + if (html.length < 3000 && /__CF\$cv\$params/i.test(html)) return true + return false +} + +// Firefox's own internal about:neterror / about:certerror page — means the browser never +// reached a real server at all (DNS failure, connection refused, TLS error, etc). Distinct +// from a Cloudflare/WAF block: there's no origin response to retry against, so callers +// should treat this the same as a hard network failure, not as scraped content. +export function isBrowserErrorPage(html: string): boolean { + if (/chrome:\/\/global\/skin\/aboutNetError/i.test(html)) return true + if (/data-l10n-id="(neterror|certerror)-page-title"/i.test(html)) return true + if (/<net-error-card>/i.test(html)) return true return false } diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index 0fdbdc8..6f3280f 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -5,11 +5,12 @@ export { hasRecaptcha, hasTurnstile, isBlocked, + isBrowserErrorPage, isCloudflarePage, needsJs, } from "./detect" export type { OrchestratorDeps } from "./orchestrator" -export { scrape } from "./orchestrator" +export { ScrapeError, scrape } from "./orchestrator" export { normalizeProxy, ProxyPool } from "./proxyRotator" export { isValidMethod, diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 29208b4..9960d13 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -13,6 +13,19 @@ import { runTier4 } from "./tier4" // keeps a long proxy list from blowing the request's maxTimeout budget. const MAX_PROXY_ATTEMPTS = 2 +// Carries the per-tier attempt history alongside the failure message, so callers +// (the API layer) can report exactly which tier failed and why instead of just a +// flat string — this data already exists in-memory by the time we throw, it just +// wasn't reaching anyone outside the orchestrator. +export class ScrapeError extends Error { + timings: TierResult[] + constructor(message: string, timings: TierResult[]) { + super(message) + this.name = "ScrapeError" + this.timings = timings + } +} + // True when a Tier 3/4 result indicates the browser's profile was actively rejected // by the upstream (CF / Imperva / etc.). On these outcomes the orchestrator flags the // pool for a future recycle; on every other outcome (success, transient error, timeout) @@ -73,12 +86,13 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis sessionCached: false, timings, totalMs: Date.now() - totalStart, + proxyUsed: false, } } } if (maxTier < 2) { - throw new Error("Max tier reached without success") + throw new ScrapeError("Max tier reached without success", timings) } // Acquire browser for tiers 2-4 @@ -110,6 +124,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis timings, totalMs: Date.now() - totalStart, captchasSolved: t2.captchasSolved, + proxyUsed: false, } } // Session failed — purge it @@ -117,7 +132,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis } if (maxTier < 3) { - throw new Error("Max tier reached without success") + throw new ScrapeError("Max tier reached without success", timings) } // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request @@ -170,19 +185,21 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis timings, totalMs: Date.now() - totalStart, captchasSolved: t3.captchasSolved, + proxyUsed: Boolean(proxy3), } } if (maxTier < 4) { - throw new Error("Max tier reached without success") + throw new ScrapeError("Max tier reached without success", timings) } // Tier 4: residential proxy escalation — requires at least one residential proxy, // supplied either per-request (req.proxy) or via the configured residential pool. let proxy4 = req.proxy ?? deps.residentialProxyPool?.next(domain) if (!proxy4) { - throw new Error( + throw new ScrapeError( `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`, + timings, ) } @@ -225,10 +242,12 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis sessionCached: false, timings, totalMs: Date.now() - totalStart, + captchasSolved: t4.captchasSolved, + proxyUsed: true, } } - throw new Error(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`) + throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings) } finally { deps.releaseBrowser(handle.id) } diff --git a/packages/tiers/src/tier2.ts b/packages/tiers/src/tier2.ts index 6a32451..aabf87a 100644 --- a/packages/tiers/src/tier2.ts +++ b/packages/tiers/src/tier2.ts @@ -1,6 +1,6 @@ import type { BrowserHandle } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" -import { isCloudflarePage } from "./detect" +import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" import type { RouteLike } from "./sanitize" import { routeContinueOverrides } from "./sanitize" @@ -68,10 +68,23 @@ export async function runTier2( const html = await page.content() + if (isBrowserErrorPage(html)) { + return { + tier: 2, + status: "error", + durationMs: Date.now() - start, + reason: "browser network error (about:neterror)", + } + } + if (isCloudflarePage(html, {})) { return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } + if (isBlocked(statusCode, html)) { + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + } + // Attempt to solve any embedded captcha widgets (Turnstile, reCAPTCHA, hCaptcha). // Pages that load cleanly via session cache may still have in-page challenge widgets. const solveRemaining = maxTimeout - (Date.now() - start) diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tier3.ts index 8a4a557..b0826c4 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tier3.ts @@ -2,7 +2,7 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" import { waitForImpervaResolution } from "./impervaWait" import type { RouteLike } from "./sanitize" @@ -123,6 +123,15 @@ export async function runTier3( return { tier: 3, status: "error", durationMs: Date.now() - start, reason: errMsg } } + // Browser never reached a real server (DNS/connection/TLS failure) — the "navigation + // interrupted" tolerance above lets Firefox-specific network errors fall through + // instead of hitting the isHardFail regex (which only matches Chromium ERR_* strings), + // so we still need to catch the resulting about:neterror page here. + if (isBrowserErrorPage(html)) { + const errMsg = gotoErr instanceof Error ? gotoErr.message.split("\n")[0] : "browser network error (about:neterror)" + return { tier: 3, status: "error", durationMs: Date.now() - start, reason: errMsg } + } + if (isCloudflarePage(html, {})) { const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() @@ -137,6 +146,10 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } } + if (isBlocked(statusCode, html)) { + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + } + const rawCookies = await freshCtx.cookies() const cookies: Cookie[] = rawCookies.map( (c: { diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tier4.ts index fc6db08..1564512 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tier4.ts @@ -2,11 +2,12 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" import { waitForChallengeResolution } from "./challengeWait" -import { detectChallengeType, hasImpervaChallenge, isCloudflarePage } from "./detect" +import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" import { normalizeHtml } from "./html" import { waitForImpervaResolution } from "./impervaWait" import type { RouteLike } from "./sanitize" import { routeContinueOverrides } from "./sanitize" +import { solvePageCaptchas } from "./solvers" export interface Tier4Result extends TierResult { tier: 4 @@ -14,6 +15,7 @@ export interface Tier4Result extends TierResult { cookies?: Cookie[] userAgent?: string statusCode?: number + captchasSolved?: string[] } export async function runTier4( @@ -117,12 +119,30 @@ export async function runTier4( await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}) + // Attempt to solve any embedded captcha widgets on the page (Turnstile, reCaptcha, hCaptcha) — + // same as Tier 3. Sites that reach Tier 4 for IP reputation can still have an in-page widget. + const solveRemaining = maxTimeout - (Date.now() - start) + let captchasSolved: string[] = [] + if (solveRemaining > 5000) { + const solveResult = await solvePageCaptchas(page, solveRemaining).catch(() => ({ attempted: [], solved: [] })) + captchasSolved = solveResult.solved + } + const html = await page.content() if (html.length < 100) { return { tier: 4, status: "error", durationMs: Date.now() - start, reason: "page returned empty content" } } + if (isBrowserErrorPage(html)) { + return { + tier: 4, + status: "error", + durationMs: Date.now() - start, + reason: "browser network error (about:neterror)", + } + } + if (isCloudflarePage(html, {})) { return { tier: 4, @@ -141,6 +161,10 @@ export async function runTier4( } } + if (isBlocked(statusCode, html)) { + return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } + } + const rawCookies = await proxyContext.cookies() const cookies: Cookie[] = rawCookies.map( (c: { @@ -172,6 +196,7 @@ export async function runTier4( cookies, userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), statusCode, + captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, } } catch (err) { return { diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 860670f..b9fc8ea 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -43,6 +43,7 @@ export interface ScrapeResult { timings: TierResult[] totalMs: number captchasSolved?: string[] // captcha types solved during this request (e.g. ['turnstile', 'recaptcha-v2']) + proxyUsed?: boolean // true if the winning tier routed through a proxy (Tier 3 datacenter pool or Tier 4 residential pool/override) } export interface SessionData {