From eff5760ef5feb49392e7fa95ace85f4b5e3de445 Mon Sep 17 00:00:00 2001 From: germondai Date: Sat, 25 Jul 2026 17:42:11 +0200 Subject: [PATCH] feat(api): wire proxy lifecycle and context cache --- apps/api/src/config.ts | 18 ++++++------ apps/api/src/deps.ts | 58 +++++++++++++++++++++++++-------------- apps/api/src/index.ts | 14 +++++++--- apps/api/src/lifecycle.ts | 23 +++++++++++----- 4 files changed, 71 insertions(+), 42 deletions(-) diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index ee5faaa..af37836 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -17,13 +17,11 @@ export const CONTENT_PROCESSES = Number(process.env.BROWSER_CONTENT_PROCESSES ?? // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source // (one proxy per line) for lists too large for a single env var. -export const proxyPool = - ProxyPool.fromEnv(process.env.PROXY_URL || undefined, process.env.PROXY_LIST_FILE || undefined) ?? undefined -export const residentialProxyPool = - ProxyPool.fromEnv( - process.env.RESIDENTIAL_PROXY_URL || undefined, - process.env.RESIDENTIAL_PROXY_LIST_FILE || undefined, - ) ?? undefined +export const proxyPool = ProxyPool.fromEnv(process.env.PROXY_URL, process.env.PROXY_LIST_FILE) +export const residentialProxyPool = ProxyPool.fromEnv( + process.env.RESIDENTIAL_PROXY_URL, + process.env.RESIDENTIAL_PROXY_LIST_FILE, +) // ── MITM forward-proxy mode ──────────────────────────────────────────────────── // Optional browser-backed HTTP(S) forward proxy (apps/api/src/proxy). Off by default. @@ -41,9 +39,9 @@ export const MITM_PROXY_HOST = process.env.MITM_PROXY_HOST ?? "0.0.0.0" // CA cert + key live here (persist across restarts so the CA is installed once). export const MITM_PROXY_CA_DIR = process.env.MITM_PROXY_CA_DIR ?? "/data/proxy-ca" // Cap the tier the proxy will escalate to (e.g. keep it off residential Tier 4). -export const MITM_PROXY_MAX_TIER = process.env.MITM_PROXY_MAX_TIER - ? (Number(process.env.MITM_PROXY_MAX_TIER) as 1 | 2 | 3 | 4) - : undefined +const configuredMaxTier = Number(process.env.MITM_PROXY_MAX_TIER) +const isTier = (tier: number): tier is 1 | 2 | 3 | 4 => tier === 1 || tier === 2 || tier === 3 || tier === 4 +export const MITM_PROXY_MAX_TIER = isTier(configuredMaxTier) ? configuredMaxTier : undefined // Log one line per proxied request (method, url, status, content-type, bytes). Off by // default — proxied clients can be chatty. Errors are always logged. export const MITM_PROXY_DEBUG = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_DEBUG ?? "") diff --git a/apps/api/src/deps.ts b/apps/api/src/deps.ts index 76d4399..e8d6385 100644 --- a/apps/api/src/deps.ts +++ b/apps/api/src/deps.ts @@ -1,4 +1,6 @@ -import { BrowserPool, SessionCache } from "@trawl/browser" +import { BrowserPool, type PersistentBrowserContext, PersistentContextCache, SessionCache } from "@trawl/browser" +import type { OrchestratorDeps } from "@trawl/tiers" +import type { SessionData } from "@trawl/types" import { ACQUIRE_TIMEOUT_MS, CONTENT_PROCESSES, @@ -10,19 +12,17 @@ import { SESSION_TTL, } from "./config" -// Single embedded pool — no BullMQ / worker process required. -// Redis is optional: without it, session caching (Tier 2 fast path) is disabled -// but scraping still works via Tier 1 / Tier 3. -let pool: BrowserPool | null = null -let sessionCache: SessionCache | null = null +const state: { + pool?: BrowserPool + sessionCache?: SessionCache + persistentContextCache?: PersistentContextCache +} = {} -export function getPool(): BrowserPool | null { - return pool -} +export const getPool = () => state.pool -export async function initPool() { +export const initPool = async (): Promise => { try { - sessionCache = new SessionCache({ + state.sessionCache = new SessionCache({ redisUrl: REDIS_URL, ttlSeconds: SESSION_TTL, }) @@ -31,29 +31,45 @@ export async function initPool() { console.warn("[api] session cache unavailable — Tier 2 disabled:", err instanceof Error ? err.message : err) } - pool = new BrowserPool({ + state.pool = new BrowserPool({ poolSize: POOL_SIZE, acquireTimeoutMs: ACQUIRE_TIMEOUT_MS, recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS, contentProcesses: CONTENT_PROCESSES, }) - await pool.init() - pool.startHealthCheck() + await state.pool.init() + state.pool.startHealthCheck() + + state.persistentContextCache = new PersistentContextCache({ + maxEntries: 20, + ttlMs: 10 * 60 * 1000, + }) + console.log(`[api] ready — all ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"} warm`) } -export function getDeps() { - if (!pool) throw new Error("pool not ready") - const p = pool - const sc = sessionCache +export const getDeps = (): OrchestratorDeps => { + if (!state.pool) throw new Error("pool not ready") + const p = state.pool + const sc = state.sessionCache + const pcc = state.persistentContextCache return { acquireBrowser: (d: string) => p.acquire(d), releaseBrowser: (id: number) => p.release(id), - // Session cache ops are no-ops when Redis is unavailable - loadSession: (d: string) => (sc ? sc.load(d).catch(() => null) : Promise.resolve(null)), - saveSession: (d: string, data: unknown) => (sc ? sc.save(d, data as never).catch(() => {}) : Promise.resolve()), + loadSession: (d: string) => (sc ? sc.load(d).catch(() => undefined) : Promise.resolve(undefined)), + saveSession: (d: string, data: SessionData) => (sc ? sc.save(d, data).catch(() => {}) : Promise.resolve()), invalidateSession: (d: string) => (sc ? sc.invalidate(d).catch(() => {}) : Promise.resolve()), proxyPool, residentialProxyPool, + acquireContext: async (_handleId: number, hostname: string) => pcc?.get(hostname), + saveContext: async (handleId: number, hostname: string, context: PersistentBrowserContext) => { + if (pcc) pcc.set(hostname, context, handleId) + }, + releaseContext: (_handleId: number, hostname: string) => { + pcc?.get(hostname) + }, + invalidateContext: async (hostname: string) => { + pcc?.evict(hostname) + }, } } diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 93acdad..513c5ba 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -11,7 +11,7 @@ import { } from "./config" import { getDeps, initPool } from "./deps" import { registerLifecycleHandlers } from "./lifecycle" -import { startMitmProxy } from "./proxy/server" +import { type MitmProxyHandle, shutdownMitmProxy, startMitmProxy } from "./proxy/server" import { healthRoute } from "./routes/health" import { indexRoute } from "./routes/index" import { proxyCaRoute } from "./routes/proxy-ca" @@ -29,11 +29,13 @@ new Elysia() .listen(PORT) console.log(`[api] TRAWL starting on :${PORT} (pool: ${POOL_SIZE} browser${POOL_SIZE === 1 ? "" : "s"})`) + +const state: { proxyHandle?: MitmProxyHandle } = {} + initPool() .then(() => { - // Proxy needs a ready pool — start it only after the browsers are warm. if (MITM_PROXY_ENABLED) { - startMitmProxy({ + state.proxyHandle = startMitmProxy({ port: MITM_PROXY_PORT, host: MITM_PROXY_HOST, caDir: MITM_PROXY_CA_DIR, @@ -48,4 +50,8 @@ initPool() process.exit(1) }) -registerLifecycleHandlers() +registerLifecycleHandlers({ + onShutdown: async () => { + if (state.proxyHandle) await shutdownMitmProxy(state.proxyHandle) + }, +}) diff --git a/apps/api/src/lifecycle.ts b/apps/api/src/lifecycle.ts index 007ba84..dc734e8 100644 --- a/apps/api/src/lifecycle.ts +++ b/apps/api/src/lifecycle.ts @@ -1,6 +1,10 @@ import { getPool } from "./deps" -export function registerLifecycleHandlers(): void { +export interface LifecycleOptions { + onShutdown?: () => Promise +} + +export const registerLifecycleHandlers = (opts: LifecycleOptions = {}): void => { // Camoufox (Firefox) emits page-error events in a shape playwright-core's dispatcher // doesn't expect for some target-page JS errors (e.g. missing `error.location`), which // throws inside the library's own internal event handling — outside any try/catch we @@ -15,13 +19,18 @@ export function registerLifecycleHandlers(): void { console.error("[api] unhandledRejection (continuing):", reason instanceof Error ? reason.message : reason) }) - process.on("SIGTERM", async () => { + const shutdown = async () => { + if (opts.onShutdown) { + try { + await opts.onShutdown() + } catch (err) { + console.error("[api] onShutdown error:", err instanceof Error ? err.message : err) + } + } await getPool()?.shutdown() process.exit(0) - }) + } - process.on("SIGINT", async () => { - await getPool()?.shutdown() - process.exit(0) - }) + process.on("SIGTERM", shutdown) + process.on("SIGINT", shutdown) }