diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index 0b41b43..b3fff87 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -1,10 +1,10 @@ export type { OrchestratorDeps } from "./orchestrator" export { ScrapeError, scrape } from "./orchestrator" export { type SolveResult, solvePageCaptchas } from "./solvers" -export { runTier1 } from "./tiers/1" -export { runTier2 } from "./tiers/2" -export { runTier3 } from "./tiers/3" -export { runTier4 } from "./tiers/4" +export { runTier1, type Tier1Result } from "./tiers/1" +export { runTier2, type Tier2Result } from "./tiers/2" +export { runTier3, type Tier3Result } from "./tiers/3" +export { runTier4, type Tier4Result } from "./tiers/4" export { type ChallengeType, detectChallengeType, @@ -21,7 +21,9 @@ export { export { normalizeProxy, ProxyPool } from "./utils/proxyRotator" export { isValidMethod, + proxySanitizeHeaders, RESERVED_HEADER_NAMES, + RESPONSE_HOP_BY_HOP_HEADERS, RequestValidationError, requireContentTypeForBody, routeContinueOverrides, diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index f106b78..93409f5 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -1,4 +1,4 @@ -import type { BrowserHandle } from "@trawl/browser" +import type { BrowserHandle, PersistentBrowserContext } from "@trawl/browser" import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { runTier1 } from "./tiers/1" @@ -38,12 +38,19 @@ export function shouldFlagForRecycle(status: TierResult["status"]): boolean { export interface OrchestratorDeps { acquireBrowser(domain: string): Promise releaseBrowser(id: number): void - loadSession(domain: string): Promise + loadSession(domain: string): Promise saveSession(domain: string, data: SessionData): Promise invalidateSession(domain: string): Promise proxyPool?: ProxyPool residentialProxyPool?: ProxyPool onTierAttempt?: (result: TierResult) => void + // Optional per-host persistent browser context cache. When provided, Tier 2 + // reuses a warm context with cached `cf_clearance` cookies on repeat visits + // — the cookie-loading + Redis round-trip is skipped entirely. + acquireContext?(handleId: number, hostname: string): Promise + saveContext?(handleId: number, hostname: string, context: PersistentBrowserContext): Promise + releaseContext?(handleId: number, hostname: string): void + invalidateContext?(hostname: string): Promise } const extractDomain = (url: string): string => { @@ -54,6 +61,9 @@ const extractDomain = (url: string): string => { } } +const hasUsablePayload = (result: { status: TierResult["status"]; html?: string; body?: Uint8Array }): boolean => + result.status === "success" && (result.body !== undefined || Boolean(result.html)) + export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promise { const totalStart = Date.now() const maxTimeout = req.maxTimeout ?? 60_000 @@ -73,13 +83,13 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis if (!req.skipHttp && maxTier >= 1) { const t1 = await runTier1(req.url, sanitizedHeaders, req.method, req.body) emit(t1) - if (t1.status === "success" && t1.html !== undefined) { + if (hasUsablePayload(t1)) { // Tier 1 doesn't acquire a browser (it's a plain HTTP fetch). Use a random fingerprint // UA from the pool so even Tier 1 requests don't share a single signature. const tier1UA = FINGERPRINT_POOL[Math.floor(Math.random() * FINGERPRINT_POOL.length)].userAgent return { url: req.url, - html: normalizeHtml(t1.html), + html: normalizeHtml(t1.html ?? ""), cookies: [], userAgent: tier1UA, statusCode: t1.statusCode ?? 200, @@ -88,6 +98,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis timings, totalMs: Date.now() - totalStart, proxyUsed: false, + body: t1.body, + responseHeaders: t1.responseHeaders, + contentType: t1.contentType, } } } @@ -104,9 +117,19 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis const session = await deps.loadSession(domain) if (session && maxTier >= 2) { const remaining = maxTimeout - (Date.now() - totalStart) - const t2 = await runTier2(req.url, handle, session, remaining, sanitizedHeaders, req.method, req.body) + const t2 = await runTier2( + req.url, + handle, + session, + remaining, + sanitizedHeaders, + req.method, + req.body, + deps, + domain, + ) emit(t2) - if (t2.status === "success" && t2.html !== undefined) { + if (hasUsablePayload(t2)) { if (t2.cookies && t2.cookies.length > 0) { await deps.saveSession(domain, { cookies: t2.cookies, @@ -116,7 +139,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis } return { url: req.url, - html: normalizeHtml(t2.html), + html: normalizeHtml(t2.html ?? ""), cookies: t2.cookies ?? [], userAgent: session.userAgent, statusCode: t2.statusCode ?? 200, @@ -126,6 +149,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis totalMs: Date.now() - totalStart, captchasSolved: t2.captchasSolved, proxyUsed: false, + body: t2.body, + responseHeaders: t2.responseHeaders, + contentType: t2.contentType, } } // Session failed — purge it @@ -166,7 +192,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis proxy3 = next } emit(t3) - if (t3.status === "success" && t3.html !== undefined) { + if (hasUsablePayload(t3)) { const cookies: Cookie[] = t3.cookies ?? [] if (cookies.length > 0) { await deps.saveSession(domain, { @@ -177,7 +203,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis } return { url: req.url, - html: normalizeHtml(t3.html), + html: normalizeHtml(t3.html ?? ""), cookies, userAgent: t3.userAgent ?? FINGERPRINT.userAgent, statusCode: t3.statusCode ?? 200, @@ -187,6 +213,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis totalMs: Date.now() - totalStart, captchasSolved: t3.captchasSolved, proxyUsed: Boolean(proxy3), + body: t3.body, + responseHeaders: t3.responseHeaders, + contentType: t3.contentType, } } @@ -225,7 +254,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis proxy4 = next } emit(t4) - if (t4.status === "success" && t4.html !== undefined) { + if (hasUsablePayload(t4)) { const cookies: Cookie[] = t4.cookies ?? [] if (cookies.length > 0) { await deps.saveSession(domain, { @@ -236,7 +265,7 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis } return { url: req.url, - html: normalizeHtml(t4.html), + html: normalizeHtml(t4.html ?? ""), cookies, userAgent: t4.userAgent ?? FINGERPRINT.userAgent, statusCode: t4.statusCode ?? 200, @@ -246,6 +275,9 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis totalMs: Date.now() - totalStart, captchasSolved: t4.captchasSolved, proxyUsed: true, + body: t4.body, + responseHeaders: t4.responseHeaders, + contentType: t4.contentType, } } diff --git a/packages/tiers/src/tiers/1.ts b/packages/tiers/src/tiers/1.ts index be9337d..4d6fc9d 100644 --- a/packages/tiers/src/tiers/1.ts +++ b/packages/tiers/src/tiers/1.ts @@ -2,13 +2,21 @@ import { FINGERPRINT } from "@trawl/browser" import type { TierResult } from "@trawl/types" import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "../utils/detect" import { normalizeHtml } from "../utils/html" +import { isTextContentType } from "../utils/response" export interface Tier1Result extends TierResult { tier: 1 html?: string + body?: Uint8Array + responseHeaders?: Record + contentType?: string statusCode?: number } +// Methods that may carry a request body per RFC 7231/9341. CONNECT is excluded +// (tunneling verb), TRACE/GET/HEAD/OPTIONS excluded (no body semantics). +const METHODS_WITH_BODY = new Set(["POST", "PUT", "PATCH", "DELETE", "QUERY"]) + export async function runTier1( url: string, extraHeaders?: Record, @@ -17,9 +25,10 @@ export async function runTier1( ): Promise { const start = Date.now() try { + const m = (method ?? "GET").toUpperCase() const res = await fetch(url, { - method: method ?? "GET", - body: method === "POST" ? body : undefined, + method: m, + body: METHODS_WITH_BODY.has(m) ? body : undefined, headers: { "User-Agent": FINGERPRINT.userAgent, Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", @@ -32,14 +41,33 @@ export async function runTier1( redirect: "follow", }) - const html = await res.text() - const headers: Record = {} - res.headers.forEach((v, k) => { - headers[k] = v - }) + // Preserve raw bytes — required for binary content (.torrent, images, etc.). + // The MITM proxy (:8192) consumes `body`; /scrape still consumes `html`. + const rawBytes = new Uint8Array(await res.arrayBuffer()) - if (isCloudflarePage(html, headers)) { - return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "cloudflare-challenge" } + const responseHeaders: Record = {} + res.headers.forEach((v, k) => { + responseHeaders[k] = v + }) + const contentType = responseHeaders["content-type"] ?? "application/octet-stream" + + // Decode a bounded preview losslessly for challenge detection — keeps the original + // byte buffer untouched. `fatal: false` replaces invalid sequences with U+FFFD + // so detection helpers don't throw on non-UTF8 payloads. + const previewLen = Math.min(rawBytes.length, 4096) + const previewText = new TextDecoder("utf-8", { fatal: false }).decode(rawBytes.subarray(0, previewLen)) + + if (isCloudflarePage(previewText, responseHeaders)) { + return { + tier: 1, + status: "needs-js", + durationMs: Date.now() - start, + reason: "cloudflare-challenge", + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } } // JS-only challenges: the page's static HTML is just a shell that loads the @@ -47,25 +75,67 @@ export async function runTier1( // would otherwise report success — but the real content (including the widget) // only renders after JS executes. Escalate so Tier 3 runs the page in a browser, // executes JS, and the solver can engage the actual widget. - if (hasHcaptcha(html)) { - return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "hcaptcha-shell" } + if (hasHcaptcha(previewText)) { + return { + tier: 1, + status: "needs-js", + durationMs: Date.now() - start, + reason: "hcaptcha-shell", + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } } - if (hasRecaptcha(html)) { - return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "recaptcha-shell" } + if (hasRecaptcha(previewText)) { + return { + tier: 1, + status: "needs-js", + durationMs: Date.now() - start, + reason: "recaptcha-shell", + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } } - if (hasTurnstile(html)) { - return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "turnstile-shell" } + if (hasTurnstile(previewText)) { + return { + tier: 1, + status: "needs-js", + durationMs: Date.now() - start, + reason: "turnstile-shell", + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } } - if (isBlocked(res.status, html)) { - return { tier: 1, status: "blocked", durationMs: Date.now() - start, reason: `http-${res.status}` } + if (isBlocked(res.status, previewText)) { + return { + tier: 1, + status: "blocked", + durationMs: Date.now() - start, + reason: `http-${res.status}`, + responseHeaders, + contentType, + body: rawBytes, + statusCode: res.status, + } } return { tier: 1, status: "success", durationMs: Date.now() - start, - html: normalizeHtml(html), + // `html` is best-effort text view of the body — only meaningful for text-like + // content-types. Empty for binary payloads so /scrape consumers see the body + // is binary via the contentType field. + html: isTextContentType(contentType) ? normalizeHtml(previewText) : "", + body: rawBytes, + responseHeaders, + contentType, statusCode: res.status, } } catch (err) { diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index ece1092..f8ce3a2 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -1,15 +1,19 @@ -import type { BrowserHandle } from "@trawl/browser" +import type { BrowserHandle, PersistentBrowserContext } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" import { solvePageCaptchas } from "../solvers" import { normalizeSameSite, toCookies } from "../utils/cookies" import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" import { normalizeHtml } from "../utils/html" +import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" export interface Tier2Result extends TierResult { tier: 2 html?: string + body?: Uint8Array + responseHeaders?: Record + contentType?: string cookies?: Cookie[] statusCode?: number captchasSolved?: string[] @@ -23,26 +27,54 @@ export async function runTier2( extraHeaders?: Record, method?: string, body?: string, + // Optional — when provided by the orchestrator, Tier 2 reuses a warm per-host + // browser context on repeat visits. Cached contexts already hold cf_clearance + // cookies from the prior solve, so we skip the Redis round-trip and cookie + // re-injection entirely. + deps?: { + acquireContext?(handleId: number, hostname: string): Promise + saveContext?(handleId: number, hostname: string, context: PersistentBrowserContext): Promise + releaseContext?(handleId: number, hostname: string): void + }, + hostname?: string, ): Promise { const start = Date.now() - const page = await handle.context.newPage() + + // Pick the context: persistent cache hit → reuse; otherwise fall back to the + // pool's shared context. The shared context still works (existing behavior) + // — only the persistent cache path is new. + let activeContext = handle.context + let contextFromCache = false + if (deps?.acquireContext && hostname) { + const cached = await deps.acquireContext(handle.id, hostname) + if (cached) { + activeContext = cached + contextFromCache = true + } + } + + const page = await activeContext.newPage() try { // addCookies replaces cookies by name+domain+path, so no need to clearCookies first. // Keeping the context's CF cookies (cf_clearance, __cf_bm) intact means CF sees a // browser with history, which speeds up challenge evaluation on the next Tier 3 run. - await handle.context.addCookies( - session.cookies.map((c) => ({ - name: c.name, - value: c.value, - domain: c.domain, - path: c.path, - expires: c.expires, - httpOnly: c.httpOnly, - secure: c.secure, - sameSite: normalizeSameSite(c.sameSite), - })), - ) + // Skip the Redis→cookie re-injection on cache hits — the persistent context + // already carries cookies from its prior solve. + if (!contextFromCache) { + await activeContext.addCookies( + session.cookies.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + expires: c.expires, + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: normalizeSameSite(c.sameSite), + })), + ) + } await page.setExtraHTTPHeaders({ "User-Agent": session.userAgent }) @@ -53,8 +85,12 @@ export async function runTier2( } let statusCode = 200 - page.on("response", (res: { url(): string; status(): number }) => { - if (res.url() === url) statusCode = res.status() + const mainResponseHolder: { value?: MinimalResponse } = {} + page.on("response", (res: MinimalResponse) => { + if (res.url() === url) { + statusCode = res.status() + if (!mainResponseHolder.value) mainResponseHolder.value = res + } }) await page.goto(url, { waitUntil: "domcontentloaded", timeout: maxTimeout }) @@ -88,13 +124,27 @@ export async function runTier2( captchasSolved = result.solved } - const cookies: Cookie[] = toCookies(await handle.context.cookies()) + const cookies: Cookie[] = toCookies(await activeContext.cookies()) + + const captured = await captureResponse(mainResponseHolder.value) + + // On success, register this context in the persistent cache so the next + // visit to this hostname skips cookie loading entirely. Skip if it was + // already served from the cache (no need to re-register the same context). + if (!contextFromCache && deps?.saveContext && hostname && cookies.length > 0) { + await deps.saveContext(handle.id, hostname, activeContext).catch(() => { + // Caching is best-effort; the request already succeeded. + }) + } return { tier: 2, status: "success", durationMs: Date.now() - start, - html: normalizeHtml(html), + // For HTML/text content-types, `html` is the rendered DOM. For binary, leave + // empty so /scrape consumers know to use `body`/`contentType`. + html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(html) : "", + ...captured, cookies, statusCode, captchasSolved: captchasSolved.length > 0 ? captchasSolved : undefined, diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 58e97b4..356886d 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -14,12 +14,16 @@ import { import { normalizeHtml } from "../utils/html" import { waitForImpervaResolution } from "../utils/impervaWait" import { isHardNetworkFailure } from "../utils/network" +import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" export interface Tier3Result extends TierResult { tier: 3 html?: string + body?: Uint8Array + responseHeaders?: Record + contentType?: string cookies?: Cookie[] userAgent?: string statusCode?: number @@ -53,11 +57,13 @@ export async function runTier3( } let statusCode = 200 - page.on("response", (res: { url(): string; status(): number }) => { + const mainResponseHolder: { value?: MinimalResponse } = {} + page.on("response", (res: MinimalResponse) => { try { const resUrl = res.url() if (resUrl === url || resUrl.startsWith(url.replace(/\/$/, ""))) { statusCode = res.status() + if (!mainResponseHolder.value) mainResponseHolder.value = res } } catch {} }) @@ -154,11 +160,14 @@ export async function runTier3( const cookies: Cookie[] = toCookies(await freshCtx.cookies()) + const captured = await captureResponse(mainResponseHolder.value) + return { tier: 3, status: "success", durationMs: Date.now() - start, - html: normalizeHtml(html), + html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(html) : "", + ...captured, cookies, userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), statusCode, diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index 1beb36e..e83ade0 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -14,12 +14,16 @@ import { import { normalizeHtml } from "../utils/html" import { waitForImpervaResolution } from "../utils/impervaWait" import { isHardNetworkFailure } from "../utils/network" +import { captureResponse, isTextContentType, type MinimalResponse } from "../utils/response" import type { RouteLike } from "../utils/sanitize" import { routeContinueOverrides } from "../utils/sanitize" export interface Tier4Result extends TierResult { tier: 4 html?: string + body?: Uint8Array + responseHeaders?: Record + contentType?: string cookies?: Cookie[] userAgent?: string statusCode?: number @@ -41,15 +45,16 @@ export async function runTier4( // Proxies must be set at context creation time in Playwright — they cannot be // applied per-request. We create a fresh context here and close it when done, // leaving the pool's shared context untouched. - let proxyContext: Awaited> | null = null + const state: { proxyContext?: Awaited> } = {} try { // Camoufox handles fingerprinting at the C++ level — only the proxy needs to // be set at context creation (Playwright requires proxy at context init time). - proxyContext = await handle.browser.newContext({ + const proxyContext = await handle.browser.newContext({ proxy: { server: proxyUrl }, viewport: null, }) + state.proxyContext = proxyContext await proxyContext.addInitScript(() => { window.onerror = () => true window.addEventListener( @@ -62,8 +67,7 @@ export async function runTier4( const _orig = Element.prototype.attachShadow Element.prototype.attachShadow = function (init: ShadowRootInit) { const r = _orig.call(this, init) - // biome-ignore lint/suspicious/noExplicitAny: monkeypatching Element.prototype — 'this' is HTMLElement at runtime, no TS type - ;(this as any).shadowRootUnl = r + Object.defineProperty(this, "shadowRootUnl", { configurable: true, value: r }) return r } }) @@ -77,10 +81,12 @@ export async function runTier4( } let statusCode = 200 - page.on("response", (res: { url(): string; status(): number }) => { + const mainResponseHolder: { value?: MinimalResponse } = {} + page.on("response", (res: MinimalResponse) => { try { if (res.url() === url || res.url().startsWith(url.replace(/\/$/, ""))) { statusCode = res.status() + if (!mainResponseHolder.value) mainResponseHolder.value = res } } catch {} }) @@ -168,11 +174,14 @@ export async function runTier4( const cookies: Cookie[] = toCookies(await proxyContext.cookies()) + const captured = await captureResponse(mainResponseHolder.value) + return { tier: 4, status: "success", durationMs: Date.now() - start, - html: normalizeHtml(html), + html: !captured.contentType || isTextContentType(captured.contentType) ? normalizeHtml(html) : "", + ...captured, cookies, userAgent: await page.evaluate(() => navigator.userAgent).catch(() => FINGERPRINT.userAgent), statusCode, @@ -187,8 +196,8 @@ export async function runTier4( } } finally { // Same timeout-bounded close as tier3 — see comment there. - if (proxyContext) { - await Promise.race([proxyContext.close(), new Promise((resolve) => setTimeout(resolve, 5000))]).catch( + if (state.proxyContext) { + await Promise.race([state.proxyContext.close(), new Promise((resolve) => setTimeout(resolve, 5000))]).catch( () => {}, ) } diff --git a/packages/tiers/src/utils/response.ts b/packages/tiers/src/utils/response.ts new file mode 100644 index 0000000..402a6c7 --- /dev/null +++ b/packages/tiers/src/utils/response.ts @@ -0,0 +1,34 @@ +export interface MinimalResponse { + url(): string + status(): number + headers(): Record + body(): Promise +} + +export interface CapturedResponse { + body?: Uint8Array + responseHeaders?: Record + contentType?: string +} + +const TEXT_CONTENT_MARKERS = ["html", "xml", "json", "javascript", "ecmascript", "x-www-form-urlencoded"] + +export const isTextContentType = (contentType: string): boolean => { + const normalized = contentType.toLowerCase() + return normalized.startsWith("text/") || TEXT_CONTENT_MARKERS.some((marker) => normalized.includes(marker)) +} + +export const captureResponse = async (response?: MinimalResponse): Promise => { + if (!response) return {} + try { + const raw = await response.body() + const responseHeaders = response.headers() + return { + body: raw instanceof Uint8Array ? raw : new Uint8Array(raw), + responseHeaders, + contentType: responseHeaders["content-type"] ?? "application/octet-stream", + } + } catch { + return {} + } +}