refactor(proxy): normalize proxy pool fallbacks

This commit is contained in:
germondai
2026-07-25 08:34:12 +02:00
parent 1d752aee92
commit 748d89dfc5
3 changed files with 30 additions and 26 deletions
+20 -16
View File
@@ -3,7 +3,6 @@
// fetches or trusts any third-party proxy list.
import { readFileSync } from "node:fs"
import type { ProxyEndpointInput } from "@trawl/types"
const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxed cooldown"
@@ -16,25 +15,30 @@ const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxe
// Returns undefined for null/undefined/empty/non-string-non-object inputs.
// Credentials (if present in the object form) are URL-encoded and embedded into the URL
// so downstream code keeps treating proxy as a single string.
export function normalizeProxy(input: ProxyEndpointInput | null | undefined): string | undefined {
if (input == null) return undefined
export function normalizeProxy(input?: unknown) {
if (input == null) return
if (typeof input === "string") {
const trimmed = input.trim()
return trimmed ? trimmed : undefined
}
if (typeof input === "object") {
if (typeof input === "object" && input) {
const endpoint = input as Record<string, unknown>
const server =
typeof input.url === "string" ? input.url : typeof input.server === "string" ? input.server : undefined
if (!server) return undefined
const user = typeof input.username === "string" && input.username.length > 0 ? input.username : undefined
const pass = typeof input.password === "string" && input.password.length > 0 ? input.password : undefined
typeof endpoint.url === "string"
? endpoint.url
: typeof endpoint.server === "string"
? endpoint.server
: undefined
if (!server) return
const user = typeof endpoint.username === "string" && endpoint.username.length > 0 ? endpoint.username : undefined
const pass = typeof endpoint.password === "string" && endpoint.password.length > 0 ? endpoint.password : undefined
if (!user && !pass) return server
const schemeMatch = server.match(/^([a-z][a-z0-9+\-.]*:\/\/)(.*)$/i)
if (!schemeMatch) return server
const creds = `${encodeURIComponent(user ?? "")}:${encodeURIComponent(pass ?? "")}`
return `${schemeMatch[1]}${creds}@${schemeMatch[2]}`
}
return undefined
return
}
interface ProxyState {
@@ -53,9 +57,9 @@ export class ProxyPool {
// Builds a pool from a comma-separated env var and/or a line-delimited file (one proxy
// per line, '#' comments allowed). A single URL still works — it's just a 1-element list.
// Returns null if neither source yields any proxies, so callers can treat "no proxy
// Returns undefined if neither source yields any proxies, so callers can treat "no proxy
// configured" the same way they did with the old single-string PROXY_URL/RESIDENTIAL_PROXY_URL.
static fromEnv(urlListEnv?: string, fileEnv?: string): ProxyPool | null {
static fromEnv(urlListEnv?: string, fileEnv?: string) {
const urls: string[] = []
if (urlListEnv) {
urls.push(
@@ -76,7 +80,7 @@ export class ProxyPool {
console.warn(`[proxy] failed to read proxy list file ${fileEnv}:`, err instanceof Error ? err.message : err)
}
}
return urls.length > 0 ? new ProxyPool(urls) : null
return urls.length > 0 ? new ProxyPool(urls) : undefined
}
get size(): number {
@@ -91,9 +95,9 @@ export class ProxyPool {
// Sticky-per-domain: reuse the same proxy for repeat requests to a domain (consistency
// helps avoid re-triggering challenges); falls back to round-robin across available
// proxies for new domains or once the sticky proxy has been marked bad.
next(domain?: string): string | null {
next(domain?: string) {
const available = this.available()
if (available.length === 0) return null
if (available.length === 0) return
if (domain) {
const sticky = this.stickyByDomain.get(domain)
@@ -106,9 +110,9 @@ export class ProxyPool {
return proxy.url
}
random(): string | null {
random() {
const available = this.available()
if (available.length === 0) return null
if (available.length === 0) return
return available[Math.floor(Math.random() * available.length)].url
}
+1 -1
View File
@@ -127,7 +127,7 @@ describe("normalizeProxy", () => {
})
test("non-string url field → undefined", () => {
expect(normalizeProxy({ url: 42 as unknown as string })).toBeUndefined()
expect(normalizeProxy({ url: 42 })).toBeUndefined()
})
})
})
+9 -9
View File
@@ -33,12 +33,12 @@ describe("ProxyPool", () => {
expect(after).toBeTruthy()
})
test("returns null once every proxy is marked bad", () => {
test("returns undefined once every proxy is marked bad", () => {
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"])
pool.markBad("http://p1:8080")
pool.markBad("http://p2:8080")
expect(pool.next("example.com")).toBeNull()
expect(pool.random()).toBeNull()
expect(pool.next("example.com")).toBeUndefined()
expect(pool.random()).toBeUndefined()
})
test("random() returns one of the configured proxies", () => {
@@ -49,10 +49,10 @@ describe("ProxyPool", () => {
}
})
test("returns null for an empty pool", () => {
test("returns undefined for an empty pool", () => {
const pool = new ProxyPool([])
expect(pool.next()).toBeNull()
expect(pool.random()).toBeNull()
expect(pool.next()).toBeUndefined()
expect(pool.random()).toBeUndefined()
expect(pool.size).toBe(0)
})
@@ -74,9 +74,9 @@ describe("ProxyPool", () => {
expect(pool?.size).toBe(3)
})
test("returns null when neither source has any proxies", () => {
expect(ProxyPool.fromEnv(undefined, undefined)).toBeNull()
expect(ProxyPool.fromEnv("", "")).toBeNull()
test("returns undefined when neither source has any proxies", () => {
expect(ProxyPool.fromEnv()).toBeUndefined()
expect(ProxyPool.fromEnv("", "")).toBeUndefined()
})
test("reads proxies from a line-delimited file, ignoring comments and blank lines", () => {