From 7a36a6c2c95dd3ba6b772befe4c5de30e5b0900d Mon Sep 17 00:00:00 2001 From: germondai Date: Thu, 9 Jul 2026 03:23:30 +0200 Subject: [PATCH] refactor(tiers): split into tiers/ and utils/, dedupe cookie and network-failure helpers --- packages/tiers/src/index.ts | 20 +++---- packages/tiers/src/orchestrator.ts | 14 ++--- packages/tiers/src/{tier1.ts => tiers/1.ts} | 4 +- packages/tiers/src/{tier2.ts => tiers/2.ts} | 41 +++---------- packages/tiers/src/{tier3.ts => tiers/3.ts} | 58 ++++++------------- packages/tiers/src/{tier4.ts => tiers/4.ts} | 56 ++++++------------ .../tiers/src/{ => utils}/challengeWait.ts | 0 packages/tiers/src/utils/cookies.ts | 34 +++++++++++ packages/tiers/src/{ => utils}/detect.ts | 0 packages/tiers/src/{ => utils}/html.ts | 0 packages/tiers/src/{ => utils}/impervaWait.ts | 0 packages/tiers/src/utils/network.ts | 9 +++ .../tiers/src/{ => utils}/proxyRotator.ts | 0 packages/tiers/src/{ => utils}/sanitize.ts | 12 ++-- packages/tiers/tests/normalizeProxy.test.ts | 2 +- packages/tiers/tests/proxyPool.test.ts | 2 +- packages/tiers/tests/runTier1Post.test.ts | 4 +- packages/tiers/tests/sanitize.test.ts | 2 +- 18 files changed, 119 insertions(+), 139 deletions(-) rename packages/tiers/src/{tier1.ts => tiers/1.ts} (96%) rename packages/tiers/src/{tier2.ts => tiers/2.ts} (74%) rename packages/tiers/src/{tier3.ts => tiers/3.ts} (82%) rename packages/tiers/src/{tier4.ts => tiers/4.ts} (80%) rename packages/tiers/src/{ => utils}/challengeWait.ts (100%) create mode 100644 packages/tiers/src/utils/cookies.ts rename packages/tiers/src/{ => utils}/detect.ts (100%) rename packages/tiers/src/{ => utils}/html.ts (100%) rename packages/tiers/src/{ => utils}/impervaWait.ts (100%) create mode 100644 packages/tiers/src/utils/network.ts rename packages/tiers/src/{ => utils}/proxyRotator.ts (100%) rename packages/tiers/src/{ => utils}/sanitize.ts (91%) diff --git a/packages/tiers/src/index.ts b/packages/tiers/src/index.ts index 6f3280f..03409cb 100644 --- a/packages/tiers/src/index.ts +++ b/packages/tiers/src/index.ts @@ -1,3 +1,10 @@ +export type { OrchestratorDeps } from "./orchestrator" +export { ScrapeError, scrape } from "./orchestrator" +export { solvePageCaptchas } from "./solvers" +export { runTier1 } from "./tiers/1" +export { runTier2 } from "./tiers/2" +export { runTier3 } from "./tiers/3" +export { runTier4 } from "./tiers/4" export { detectChallengeType, hasHcaptcha, @@ -8,10 +15,8 @@ export { isBrowserErrorPage, isCloudflarePage, needsJs, -} from "./detect" -export type { OrchestratorDeps } from "./orchestrator" -export { ScrapeError, scrape } from "./orchestrator" -export { normalizeProxy, ProxyPool } from "./proxyRotator" +} from "./utils/detect" +export { normalizeProxy, ProxyPool } from "./utils/proxyRotator" export { isValidMethod, RESERVED_HEADER_NAMES, @@ -21,9 +26,4 @@ export { SUPPORTED_METHODS, type SupportedMethod, sanitizeHeaders, -} from "./sanitize" -export { solvePageCaptchas } from "./solvers" -export { runTier1 } from "./tier1" -export { runTier2 } from "./tier2" -export { runTier3 } from "./tier3" -export { runTier4 } from "./tier4" +} from "./utils/sanitize" diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 9960d13..86d9584 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -1,13 +1,13 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser" import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" -import { normalizeHtml } from "./html" -import type { ProxyPool } from "./proxyRotator" -import { requireContentTypeForBody, sanitizeHeaders } from "./sanitize" -import { runTier1 } from "./tier1" -import { runTier2 } from "./tier2" -import { runTier3 } from "./tier3" -import { runTier4 } from "./tier4" +import { runTier1 } from "./tiers/1" +import { runTier2 } from "./tiers/2" +import { runTier3 } from "./tiers/3" +import { runTier4 } from "./tiers/4" +import { normalizeHtml } from "./utils/html" +import type { ProxyPool } from "./utils/proxyRotator" +import { requireContentTypeForBody, sanitizeHeaders } from "./utils/sanitize" // Bounds how many distinct proxies a single request will try per tier before giving up — // keeps a long proxy list from blowing the request's maxTimeout budget. diff --git a/packages/tiers/src/tier1.ts b/packages/tiers/src/tiers/1.ts similarity index 96% rename from packages/tiers/src/tier1.ts rename to packages/tiers/src/tiers/1.ts index 2a499f7..be9337d 100644 --- a/packages/tiers/src/tier1.ts +++ b/packages/tiers/src/tiers/1.ts @@ -1,7 +1,7 @@ import { FINGERPRINT } from "@trawl/browser" import type { TierResult } from "@trawl/types" -import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "./detect" -import { normalizeHtml } from "./html" +import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "../utils/detect" +import { normalizeHtml } from "../utils/html" export interface Tier1Result extends TierResult { tier: 1 diff --git a/packages/tiers/src/tier2.ts b/packages/tiers/src/tiers/2.ts similarity index 74% rename from packages/tiers/src/tier2.ts rename to packages/tiers/src/tiers/2.ts index aabf87a..ece1092 100644 --- a/packages/tiers/src/tier2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -1,10 +1,11 @@ import type { BrowserHandle } from "@trawl/browser" import type { Cookie, SessionData, TierResult } from "@trawl/types" -import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" -import { normalizeHtml } from "./html" -import type { RouteLike } from "./sanitize" -import { routeContinueOverrides } from "./sanitize" -import { solvePageCaptchas } from "./solvers" +import { solvePageCaptchas } from "../solvers" +import { normalizeSameSite, toCookies } from "../utils/cookies" +import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect" +import { normalizeHtml } from "../utils/html" +import type { RouteLike } from "../utils/sanitize" +import { routeContinueOverrides } from "../utils/sanitize" export interface Tier2Result extends TierResult { tier: 2 @@ -14,13 +15,6 @@ export interface Tier2Result extends TierResult { captchasSolved?: string[] } -// Playwright's cookie.sameSite is `"Strict" | "Lax" | "None"` but can be undefined when -// the cookie was set without an explicit sameSite. Normalize to the Playwright literal -// union with a default of "Lax" (matches browser default for same-origin cookies). -function normalizeSameSite(s: string | undefined): "Strict" | "Lax" | "None" { - return s === "Strict" || s === "Lax" || s === "None" ? s : "Lax" -} - export async function runTier2( url: string, handle: BrowserHandle, @@ -94,28 +88,7 @@ export async function runTier2( captchasSolved = result.solved } - const rawCookies = await handle.context.cookies() - const cookies: Cookie[] = rawCookies.map( - (c: { - name: string - value: string - domain: string - path: string - expires: number - httpOnly: boolean - secure: boolean - sameSite?: string - }) => ({ - name: c.name, - value: c.value, - domain: c.domain, - path: c.path, - expires: c.expires ?? -1, - httpOnly: c.httpOnly, - secure: c.secure, - sameSite: c.sameSite, - }), - ) + const cookies: Cookie[] = toCookies(await handle.context.cookies()) return { tier: 2, diff --git a/packages/tiers/src/tier3.ts b/packages/tiers/src/tiers/3.ts similarity index 82% rename from packages/tiers/src/tier3.ts rename to packages/tiers/src/tiers/3.ts index c45ae2d..58e97b4 100644 --- a/packages/tiers/src/tier3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -1,13 +1,21 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, newFreshContext } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" -import { waitForChallengeResolution } from "./challengeWait" -import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" -import { normalizeHtml } from "./html" -import { waitForImpervaResolution } from "./impervaWait" -import type { RouteLike } from "./sanitize" -import { routeContinueOverrides } from "./sanitize" -import { solvePageCaptchas } from "./solvers" +import { solvePageCaptchas } from "../solvers" +import { waitForChallengeResolution } from "../utils/challengeWait" +import { toCookies } from "../utils/cookies" +import { + detectChallengeType, + hasImpervaChallenge, + isBlocked, + isBrowserErrorPage, + isCloudflarePage, +} from "../utils/detect" +import { normalizeHtml } from "../utils/html" +import { waitForImpervaResolution } from "../utils/impervaWait" +import { isHardNetworkFailure } from "../utils/network" +import type { RouteLike } from "../utils/sanitize" +import { routeContinueOverrides } from "../utils/sanitize" export interface Tier3Result extends TierResult { tier: 3 @@ -65,17 +73,10 @@ export async function runTier3( .catch((e: Error) => e) // Abort early on hard network failures — no point running challenge wait - if (gotoErr instanceof Error) { - const msg = gotoErr.message - const isHardFail = - /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test( - msg, - ) - if (isHardFail) { - return { tier: 3, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] } - } - // Otherwise (navigation interrupted by CF redirect) — fall through and keep going + if (isHardNetworkFailure(gotoErr)) { + return { tier: 3, status: "error", durationMs: Date.now() - start, reason: gotoErr.message.split("\n")[0] } } + // Otherwise (navigation interrupted by CF redirect) — fall through and keep going const remaining = maxTimeout - (Date.now() - start) const peekHtml = await page.content().catch(() => "") @@ -151,28 +152,7 @@ export async function runTier3( return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } - const rawCookies = await freshCtx.cookies() - const cookies: Cookie[] = rawCookies.map( - (c: { - name: string - value: string - domain: string - path: string - expires: number - httpOnly: boolean - secure: boolean - sameSite?: string - }) => ({ - name: c.name, - value: c.value, - domain: c.domain, - path: c.path, - expires: c.expires ?? -1, - httpOnly: c.httpOnly, - secure: c.secure, - sameSite: c.sameSite, - }), - ) + const cookies: Cookie[] = toCookies(await freshCtx.cookies()) return { tier: 3, diff --git a/packages/tiers/src/tier4.ts b/packages/tiers/src/tiers/4.ts similarity index 80% rename from packages/tiers/src/tier4.ts rename to packages/tiers/src/tiers/4.ts index 1564512..1beb36e 100644 --- a/packages/tiers/src/tier4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -1,13 +1,21 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT } from "@trawl/browser" import type { Cookie, TierResult } from "@trawl/types" -import { waitForChallengeResolution } from "./challengeWait" -import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect" -import { normalizeHtml } from "./html" -import { waitForImpervaResolution } from "./impervaWait" -import type { RouteLike } from "./sanitize" -import { routeContinueOverrides } from "./sanitize" -import { solvePageCaptchas } from "./solvers" +import { solvePageCaptchas } from "../solvers" +import { waitForChallengeResolution } from "../utils/challengeWait" +import { toCookies } from "../utils/cookies" +import { + detectChallengeType, + hasImpervaChallenge, + isBlocked, + isBrowserErrorPage, + isCloudflarePage, +} from "../utils/detect" +import { normalizeHtml } from "../utils/html" +import { waitForImpervaResolution } from "../utils/impervaWait" +import { isHardNetworkFailure } from "../utils/network" +import type { RouteLike } from "../utils/sanitize" +import { routeContinueOverrides } from "../utils/sanitize" export interface Tier4Result extends TierResult { tier: 4 @@ -84,15 +92,8 @@ export async function runTier4( }) .catch((e: Error) => e) - if (gotoErr instanceof Error) { - const msg = gotoErr.message - const isHardFail = - /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test( - msg, - ) - if (isHardFail) { - return { tier: 4, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] } - } + if (isHardNetworkFailure(gotoErr)) { + return { tier: 4, status: "error", durationMs: Date.now() - start, reason: gotoErr.message.split("\n")[0] } } const remaining = maxTimeout - (Date.now() - start) @@ -165,28 +166,7 @@ export async function runTier4( return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` } } - const rawCookies = await proxyContext.cookies() - const cookies: Cookie[] = rawCookies.map( - (c: { - name: string - value: string - domain: string - path: string - expires: number - httpOnly: boolean - secure: boolean - sameSite?: string - }) => ({ - name: c.name, - value: c.value, - domain: c.domain, - path: c.path, - expires: c.expires ?? -1, - httpOnly: c.httpOnly, - secure: c.secure, - sameSite: c.sameSite, - }), - ) + const cookies: Cookie[] = toCookies(await proxyContext.cookies()) return { tier: 4, diff --git a/packages/tiers/src/challengeWait.ts b/packages/tiers/src/utils/challengeWait.ts similarity index 100% rename from packages/tiers/src/challengeWait.ts rename to packages/tiers/src/utils/challengeWait.ts diff --git a/packages/tiers/src/utils/cookies.ts b/packages/tiers/src/utils/cookies.ts new file mode 100644 index 0000000..3f5d487 --- /dev/null +++ b/packages/tiers/src/utils/cookies.ts @@ -0,0 +1,34 @@ +import type { Cookie } from "@trawl/types" + +interface RawCookie { + name: string + value: string + domain: string + path: string + expires: number + httpOnly: boolean + secure: boolean + sameSite?: string +} + +// Playwright's cookie.sameSite is `"Strict" | "Lax" | "None"` but can be undefined when +// the cookie was set without an explicit sameSite. Normalize to the Playwright literal +// union with a default of "Lax" (matches browser default for same-origin cookies). +export function normalizeSameSite(s: string | undefined): "Strict" | "Lax" | "None" { + return s === "Strict" || s === "Lax" || s === "None" ? s : "Lax" +} + +// Maps Playwright's raw context.cookies() shape to TRAWL's Cookie type — shared by +// tiers 2-4, which each read cookies back off the browser context after a successful load. +export function toCookies(rawCookies: RawCookie[]): Cookie[] { + return rawCookies.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain, + path: c.path, + expires: c.expires ?? -1, + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: c.sameSite, + })) +} diff --git a/packages/tiers/src/detect.ts b/packages/tiers/src/utils/detect.ts similarity index 100% rename from packages/tiers/src/detect.ts rename to packages/tiers/src/utils/detect.ts diff --git a/packages/tiers/src/html.ts b/packages/tiers/src/utils/html.ts similarity index 100% rename from packages/tiers/src/html.ts rename to packages/tiers/src/utils/html.ts diff --git a/packages/tiers/src/impervaWait.ts b/packages/tiers/src/utils/impervaWait.ts similarity index 100% rename from packages/tiers/src/impervaWait.ts rename to packages/tiers/src/utils/impervaWait.ts diff --git a/packages/tiers/src/utils/network.ts b/packages/tiers/src/utils/network.ts new file mode 100644 index 0000000..e4bae74 --- /dev/null +++ b/packages/tiers/src/utils/network.ts @@ -0,0 +1,9 @@ +// Shared "hard network failure" check — duplicated verbatim in tier3/tier4's goto-error +// handling before this extraction. These are Chromium/Playwright ERR_* strings that mean +// the browser never reached a server, so there's no point running challenge-wait logic. +const HARD_NETWORK_FAILURE = + /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i + +export function isHardNetworkFailure(err: unknown): err is Error { + return err instanceof Error && HARD_NETWORK_FAILURE.test(err.message) +} diff --git a/packages/tiers/src/proxyRotator.ts b/packages/tiers/src/utils/proxyRotator.ts similarity index 100% rename from packages/tiers/src/proxyRotator.ts rename to packages/tiers/src/utils/proxyRotator.ts diff --git a/packages/tiers/src/sanitize.ts b/packages/tiers/src/utils/sanitize.ts similarity index 91% rename from packages/tiers/src/sanitize.ts rename to packages/tiers/src/utils/sanitize.ts index c69a39d..557751a 100644 --- a/packages/tiers/src/sanitize.ts +++ b/packages/tiers/src/utils/sanitize.ts @@ -42,7 +42,13 @@ export function sanitizeHeaders(headers?: Record): Record = new Set(SUPPORTED_METHODS) diff --git a/packages/tiers/tests/normalizeProxy.test.ts b/packages/tiers/tests/normalizeProxy.test.ts index 4cba13a..f5047c9 100644 --- a/packages/tiers/tests/normalizeProxy.test.ts +++ b/packages/tiers/tests/normalizeProxy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test" -import { normalizeProxy } from "../src/proxyRotator" +import { normalizeProxy } from "../src/utils/proxyRotator" // Covers issue #12: Prowlarr's Cardigann flow serializes `proxy` as an object // {url, username, password}; other callers send a plain URL string. The API diff --git a/packages/tiers/tests/proxyPool.test.ts b/packages/tiers/tests/proxyPool.test.ts index 47cc8cc..3bc5d02 100644 --- a/packages/tiers/tests/proxyPool.test.ts +++ b/packages/tiers/tests/proxyPool.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test" import { mkdtempSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { ProxyPool } from "../src/proxyRotator" +import { ProxyPool } from "../src/utils/proxyRotator" describe("ProxyPool", () => { test("round-robins across proxies for different domains", () => { diff --git a/packages/tiers/tests/runTier1Post.test.ts b/packages/tiers/tests/runTier1Post.test.ts index 49e1322..ae40df6 100644 --- a/packages/tiers/tests/runTier1Post.test.ts +++ b/packages/tiers/tests/runTier1Post.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" -import { runTier1 } from "../src/tier1" +import { runTier1 } from "../src/tiers/1" interface RecordedCall { url: string @@ -89,7 +89,7 @@ describe("runTier1 — POST support", () => { // This demonstrates the tier's pass-through behaviour: a non-reserved // header does override. Reserved headers are stripped upstream by // sanitizeHeaders(); this test pins both halves of the contract. - const { sanitizeHeaders } = await import("../src/sanitize") + const { sanitizeHeaders } = await import("../src/utils/sanitize") const cleaned = sanitizeHeaders({ "User-Agent": "evil-spider/1.0", Accept: "application/json" }) expect(cleaned).toEqual({ Accept: "application/json" }) // UA was reserved, dropped diff --git a/packages/tiers/tests/sanitize.test.ts b/packages/tiers/tests/sanitize.test.ts index 1ef5498..1dff93e 100644 --- a/packages/tiers/tests/sanitize.test.ts +++ b/packages/tiers/tests/sanitize.test.ts @@ -5,7 +5,7 @@ import { RequestValidationError, requireContentTypeForBody, sanitizeHeaders, -} from "../src/sanitize" +} from "../src/utils/sanitize" describe("sanitizeHeaders", () => { test("returns undefined for undefined / empty input", () => {