refactor(tiers): split into tiers/ and utils/, dedupe cookie and network-failure helpers

packages/tiers/src/index.ts

@@ -1,3 +1,10 @@
+export type { OrchestratorDeps } from "./orchestrator"
+export { ScrapeError, scrape } from "./orchestrator"
+export { solvePageCaptchas } from "./solvers"
+export { runTier1 } from "./tiers/1"
+export { runTier2 } from "./tiers/2"
+export { runTier3 } from "./tiers/3"
+export { runTier4 } from "./tiers/4"
 export {
   detectChallengeType,
   hasHcaptcha,
@@ -8,10 +15,8 @@ export {
   isBrowserErrorPage,
   isCloudflarePage,
   needsJs,
-} from "./detect"
-export type { OrchestratorDeps } from "./orchestrator"
-export { ScrapeError, scrape } from "./orchestrator"
-export { normalizeProxy, ProxyPool } from "./proxyRotator"
+} from "./utils/detect"
+export { normalizeProxy, ProxyPool } from "./utils/proxyRotator"
 export {
   isValidMethod,
   RESERVED_HEADER_NAMES,
@@ -21,9 +26,4 @@ export {
   SUPPORTED_METHODS,
   type SupportedMethod,
   sanitizeHeaders,
-} from "./sanitize"
-export { solvePageCaptchas } from "./solvers"
-export { runTier1 } from "./tier1"
-export { runTier2 } from "./tier2"
-export { runTier3 } from "./tier3"
-export { runTier4 } from "./tier4"
+} from "./utils/sanitize"

packages/tiers/src/orchestrator.ts

@@ -1,13 +1,13 @@
 import type { BrowserHandle } from "@trawl/browser"
 import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser"
 import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types"
-import { normalizeHtml } from "./html"
-import type { ProxyPool } from "./proxyRotator"
-import { requireContentTypeForBody, sanitizeHeaders } from "./sanitize"
-import { runTier1 } from "./tier1"
-import { runTier2 } from "./tier2"
-import { runTier3 } from "./tier3"
-import { runTier4 } from "./tier4"
+import { runTier1 } from "./tiers/1"
+import { runTier2 } from "./tiers/2"
+import { runTier3 } from "./tiers/3"
+import { runTier4 } from "./tiers/4"
+import { normalizeHtml } from "./utils/html"
+import type { ProxyPool } from "./utils/proxyRotator"
+import { requireContentTypeForBody, sanitizeHeaders } from "./utils/sanitize"
 
 // Bounds how many distinct proxies a single request will try per tier before giving up —
 // keeps a long proxy list from blowing the request's maxTimeout budget.

packages/tiers/src/tiers/1.ts

@@ -1,7 +1,7 @@
 import { FINGERPRINT } from "@trawl/browser"
 import type { TierResult } from "@trawl/types"
-import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "./detect"
-import { normalizeHtml } from "./html"
+import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "../utils/detect"
+import { normalizeHtml } from "../utils/html"
 
 export interface Tier1Result extends TierResult {
   tier: 1

packages/tiers/src/tiers/2.ts

@@ -1,10 +1,11 @@
 import type { BrowserHandle } from "@trawl/browser"
 import type { Cookie, SessionData, TierResult } from "@trawl/types"
-import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect"
-import { normalizeHtml } from "./html"
-import type { RouteLike } from "./sanitize"
-import { routeContinueOverrides } from "./sanitize"
-import { solvePageCaptchas } from "./solvers"
+import { solvePageCaptchas } from "../solvers"
+import { normalizeSameSite, toCookies } from "../utils/cookies"
+import { isBlocked, isBrowserErrorPage, isCloudflarePage } from "../utils/detect"
+import { normalizeHtml } from "../utils/html"
+import type { RouteLike } from "../utils/sanitize"
+import { routeContinueOverrides } from "../utils/sanitize"
 
 export interface Tier2Result extends TierResult {
   tier: 2
@@ -14,13 +15,6 @@ export interface Tier2Result extends TierResult {
   captchasSolved?: string[]
 }
 
-// Playwright's cookie.sameSite is `"Strict" | "Lax" | "None"` but can be undefined when
-// the cookie was set without an explicit sameSite. Normalize to the Playwright literal
-// union with a default of "Lax" (matches browser default for same-origin cookies).
-function normalizeSameSite(s: string | undefined): "Strict" | "Lax" | "None" {
-  return s === "Strict" || s === "Lax" || s === "None" ? s : "Lax"
-}
-
 export async function runTier2(
   url: string,
   handle: BrowserHandle,
@@ -94,28 +88,7 @@ export async function runTier2(
       captchasSolved = result.solved
     }
 
-    const rawCookies = await handle.context.cookies()
-    const cookies: Cookie[] = rawCookies.map(
-      (c: {
-        name: string
-        value: string
-        domain: string
-        path: string
-        expires: number
-        httpOnly: boolean
-        secure: boolean
-        sameSite?: string
-      }) => ({
-        name: c.name,
-        value: c.value,
-        domain: c.domain,
-        path: c.path,
-        expires: c.expires ?? -1,
-        httpOnly: c.httpOnly,
-        secure: c.secure,
-        sameSite: c.sameSite,
-      }),
-    )
+    const cookies: Cookie[] = toCookies(await handle.context.cookies())
 
     return {
       tier: 2,

packages/tiers/src/tiers/3.ts

@@ -1,13 +1,21 @@
 import type { BrowserHandle } from "@trawl/browser"
 import { FINGERPRINT, newFreshContext } from "@trawl/browser"
 import type { Cookie, TierResult } from "@trawl/types"
-import { waitForChallengeResolution } from "./challengeWait"
-import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect"
-import { normalizeHtml } from "./html"
-import { waitForImpervaResolution } from "./impervaWait"
-import type { RouteLike } from "./sanitize"
-import { routeContinueOverrides } from "./sanitize"
-import { solvePageCaptchas } from "./solvers"
+import { solvePageCaptchas } from "../solvers"
+import { waitForChallengeResolution } from "../utils/challengeWait"
+import { toCookies } from "../utils/cookies"
+import {
+  detectChallengeType,
+  hasImpervaChallenge,
+  isBlocked,
+  isBrowserErrorPage,
+  isCloudflarePage,
+} from "../utils/detect"
+import { normalizeHtml } from "../utils/html"
+import { waitForImpervaResolution } from "../utils/impervaWait"
+import { isHardNetworkFailure } from "../utils/network"
+import type { RouteLike } from "../utils/sanitize"
+import { routeContinueOverrides } from "../utils/sanitize"
 
 export interface Tier3Result extends TierResult {
   tier: 3
@@ -65,17 +73,10 @@ export async function runTier3(
       .catch((e: Error) => e)
 
     // Abort early on hard network failures — no point running challenge wait
-    if (gotoErr instanceof Error) {
-      const msg = gotoErr.message
-      const isHardFail =
-        /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test(
-          msg,
-        )
-      if (isHardFail) {
-        return { tier: 3, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] }
-      }
-      // Otherwise (navigation interrupted by CF redirect) — fall through and keep going
+    if (isHardNetworkFailure(gotoErr)) {
+      return { tier: 3, status: "error", durationMs: Date.now() - start, reason: gotoErr.message.split("\n")[0] }
     }
+    // Otherwise (navigation interrupted by CF redirect) — fall through and keep going
 
     const remaining = maxTimeout - (Date.now() - start)
     const peekHtml = await page.content().catch(() => "")
@@ -151,28 +152,7 @@ export async function runTier3(
       return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` }
     }
 
-    const rawCookies = await freshCtx.cookies()
-    const cookies: Cookie[] = rawCookies.map(
-      (c: {
-        name: string
-        value: string
-        domain: string
-        path: string
-        expires: number
-        httpOnly: boolean
-        secure: boolean
-        sameSite?: string
-      }) => ({
-        name: c.name,
-        value: c.value,
-        domain: c.domain,
-        path: c.path,
-        expires: c.expires ?? -1,
-        httpOnly: c.httpOnly,
-        secure: c.secure,
-        sameSite: c.sameSite,
-      }),
-    )
+    const cookies: Cookie[] = toCookies(await freshCtx.cookies())
 
     return {
       tier: 3,

packages/tiers/src/tiers/4.ts

@@ -1,13 +1,21 @@
 import type { BrowserHandle } from "@trawl/browser"
 import { FINGERPRINT } from "@trawl/browser"
 import type { Cookie, TierResult } from "@trawl/types"
-import { waitForChallengeResolution } from "./challengeWait"
-import { detectChallengeType, hasImpervaChallenge, isBlocked, isBrowserErrorPage, isCloudflarePage } from "./detect"
-import { normalizeHtml } from "./html"
-import { waitForImpervaResolution } from "./impervaWait"
-import type { RouteLike } from "./sanitize"
-import { routeContinueOverrides } from "./sanitize"
-import { solvePageCaptchas } from "./solvers"
+import { solvePageCaptchas } from "../solvers"
+import { waitForChallengeResolution } from "../utils/challengeWait"
+import { toCookies } from "../utils/cookies"
+import {
+  detectChallengeType,
+  hasImpervaChallenge,
+  isBlocked,
+  isBrowserErrorPage,
+  isCloudflarePage,
+} from "../utils/detect"
+import { normalizeHtml } from "../utils/html"
+import { waitForImpervaResolution } from "../utils/impervaWait"
+import { isHardNetworkFailure } from "../utils/network"
+import type { RouteLike } from "../utils/sanitize"
+import { routeContinueOverrides } from "../utils/sanitize"
 
 export interface Tier4Result extends TierResult {
   tier: 4
@@ -84,15 +92,8 @@ export async function runTier4(
       })
       .catch((e: Error) => e)
 
-    if (gotoErr instanceof Error) {
-      const msg = gotoErr.message
-      const isHardFail =
-        /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i.test(
-          msg,
-        )
-      if (isHardFail) {
-        return { tier: 4, status: "error", durationMs: Date.now() - start, reason: msg.split("\n")[0] }
-      }
+    if (isHardNetworkFailure(gotoErr)) {
+      return { tier: 4, status: "error", durationMs: Date.now() - start, reason: gotoErr.message.split("\n")[0] }
     }
 
     const remaining = maxTimeout - (Date.now() - start)
@@ -165,28 +166,7 @@ export async function runTier4(
       return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${statusCode}` }
     }
 
-    const rawCookies = await proxyContext.cookies()
-    const cookies: Cookie[] = rawCookies.map(
-      (c: {
-        name: string
-        value: string
-        domain: string
-        path: string
-        expires: number
-        httpOnly: boolean
-        secure: boolean
-        sameSite?: string
-      }) => ({
-        name: c.name,
-        value: c.value,
-        domain: c.domain,
-        path: c.path,
-        expires: c.expires ?? -1,
-        httpOnly: c.httpOnly,
-        secure: c.secure,
-        sameSite: c.sameSite,
-      }),
-    )
+    const cookies: Cookie[] = toCookies(await proxyContext.cookies())
 
     return {
       tier: 4,

packages/tiers/src/utils/cookies.ts

@@ -0,0 +1,34 @@
+import type { Cookie } from "@trawl/types"
+
+interface RawCookie {
+  name: string
+  value: string
+  domain: string
+  path: string
+  expires: number
+  httpOnly: boolean
+  secure: boolean
+  sameSite?: string
+}
+
+// Playwright's cookie.sameSite is `"Strict" | "Lax" | "None"` but can be undefined when
+// the cookie was set without an explicit sameSite. Normalize to the Playwright literal
+// union with a default of "Lax" (matches browser default for same-origin cookies).
+export function normalizeSameSite(s: string | undefined): "Strict" | "Lax" | "None" {
+  return s === "Strict" || s === "Lax" || s === "None" ? s : "Lax"
+}
+
+// Maps Playwright's raw context.cookies() shape to TRAWL's Cookie type — shared by
+// tiers 2-4, which each read cookies back off the browser context after a successful load.
+export function toCookies(rawCookies: RawCookie[]): Cookie[] {
+  return rawCookies.map((c) => ({
+    name: c.name,
+    value: c.value,
+    domain: c.domain,
+    path: c.path,
+    expires: c.expires ?? -1,
+    httpOnly: c.httpOnly,
+    secure: c.secure,
+    sameSite: c.sameSite,
+  }))
+}

packages/tiers/src/utils/network.ts

@@ -0,0 +1,9 @@
+// Shared "hard network failure" check — duplicated verbatim in tier3/tier4's goto-error
+// handling before this extraction. These are Chromium/Playwright ERR_* strings that mean
+// the browser never reached a server, so there's no point running challenge-wait logic.
+const HARD_NETWORK_FAILURE =
+  /ERR_NAME_NOT_RESOLVED|ERR_CONNECTION_REFUSED|ERR_CONNECTION_TIMED_OUT|ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED/i
+
+export function isHardNetworkFailure(err: unknown): err is Error {
+  return err instanceof Error && HARD_NETWORK_FAILURE.test(err.message)
+}

packages/tiers/src/utils/sanitize.ts

@@ -42,7 +42,13 @@ export function sanitizeHeaders(headers?: Record<string, string>): Record<string
   return Object.keys(out).length ? out : undefined
 }
 
-export const SUPPORTED_METHODS = [
+import type { SupportedMethod } from "@trawl/types"
+
+// SupportedMethod now lives in @trawl/types (single source of truth, shared with
+// ScrapeRequest.method); re-exported here for backward compat.
+export type { SupportedMethod } from "@trawl/types"
+
+export const SUPPORTED_METHODS: readonly SupportedMethod[] = [
   "GET",
   "POST",
   "PUT",
@@ -53,9 +59,7 @@ export const SUPPORTED_METHODS = [
   "TRACE",
   // RFC 9341 — safe verb that carries the query in the request body.
   "QUERY",
-] as const
-
-export type SupportedMethod = (typeof SUPPORTED_METHODS)[number]
+]
 
 const SUPPORTED_METHOD_SET: ReadonlySet<string> = new Set(SUPPORTED_METHODS)
 

packages/tiers/tests/normalizeProxy.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test"
-import { normalizeProxy } from "../src/proxyRotator"
+import { normalizeProxy } from "../src/utils/proxyRotator"
 
 // Covers issue #12: Prowlarr's Cardigann flow serializes `proxy` as an object
 // {url, username, password}; other callers send a plain URL string. The API

packages/tiers/tests/proxyPool.test.ts

@@ -2,7 +2,7 @@ import { afterEach, describe, expect, test } from "bun:test"
 import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
-import { ProxyPool } from "../src/proxyRotator"
+import { ProxyPool } from "../src/utils/proxyRotator"
 
 describe("ProxyPool", () => {
   test("round-robins across proxies for different domains", () => {

packages/tiers/tests/runTier1Post.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test"
-import { runTier1 } from "../src/tier1"
+import { runTier1 } from "../src/tiers/1"
 
 interface RecordedCall {
   url: string
@@ -89,7 +89,7 @@ describe("runTier1 — POST support", () => {
       // This demonstrates the tier's pass-through behaviour: a non-reserved
       // header does override. Reserved headers are stripped upstream by
       // sanitizeHeaders(); this test pins both halves of the contract.
-      const { sanitizeHeaders } = await import("../src/sanitize")
+      const { sanitizeHeaders } = await import("../src/utils/sanitize")
       const cleaned = sanitizeHeaders({ "User-Agent": "evil-spider/1.0", Accept: "application/json" })
       expect(cleaned).toEqual({ Accept: "application/json" }) // UA was reserved, dropped
 

packages/tiers/tests/sanitize.test.ts

@@ -5,7 +5,7 @@ import {
   RequestValidationError,
   requireContentTypeForBody,
   sanitizeHeaders,
-} from "../src/sanitize"
+} from "../src/utils/sanitize"
 
 describe("sanitizeHeaders", () => {
   test("returns undefined for undefined / empty input", () => {