mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
refactor(proxy): normalize proxy pool fallbacks
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
// fetches or trusts any third-party proxy list.
|
||||
|
||||
import { readFileSync } from "node:fs"
|
||||
import type { ProxyEndpointInput } from "@trawl/types"
|
||||
|
||||
const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxed cooldown"
|
||||
|
||||
@@ -16,25 +15,30 @@ const COOLDOWN_MS = 5 * 60 * 1000 // 5 minutes — matches the plan's "time-boxe
|
||||
// Returns undefined for null/undefined/empty/non-string-non-object inputs.
|
||||
// Credentials (if present in the object form) are URL-encoded and embedded into the URL
|
||||
// so downstream code keeps treating proxy as a single string.
|
||||
export function normalizeProxy(input: ProxyEndpointInput | null | undefined): string | undefined {
|
||||
if (input == null) return undefined
|
||||
export function normalizeProxy(input?: unknown) {
|
||||
if (input == null) return
|
||||
if (typeof input === "string") {
|
||||
const trimmed = input.trim()
|
||||
return trimmed ? trimmed : undefined
|
||||
}
|
||||
if (typeof input === "object") {
|
||||
if (typeof input === "object" && input) {
|
||||
const endpoint = input as Record<string, unknown>
|
||||
const server =
|
||||
typeof input.url === "string" ? input.url : typeof input.server === "string" ? input.server : undefined
|
||||
if (!server) return undefined
|
||||
const user = typeof input.username === "string" && input.username.length > 0 ? input.username : undefined
|
||||
const pass = typeof input.password === "string" && input.password.length > 0 ? input.password : undefined
|
||||
typeof endpoint.url === "string"
|
||||
? endpoint.url
|
||||
: typeof endpoint.server === "string"
|
||||
? endpoint.server
|
||||
: undefined
|
||||
if (!server) return
|
||||
const user = typeof endpoint.username === "string" && endpoint.username.length > 0 ? endpoint.username : undefined
|
||||
const pass = typeof endpoint.password === "string" && endpoint.password.length > 0 ? endpoint.password : undefined
|
||||
if (!user && !pass) return server
|
||||
const schemeMatch = server.match(/^([a-z][a-z0-9+\-.]*:\/\/)(.*)$/i)
|
||||
if (!schemeMatch) return server
|
||||
const creds = `${encodeURIComponent(user ?? "")}:${encodeURIComponent(pass ?? "")}`
|
||||
return `${schemeMatch[1]}${creds}@${schemeMatch[2]}`
|
||||
}
|
||||
return undefined
|
||||
return
|
||||
}
|
||||
|
||||
interface ProxyState {
|
||||
@@ -53,9 +57,9 @@ export class ProxyPool {
|
||||
|
||||
// Builds a pool from a comma-separated env var and/or a line-delimited file (one proxy
|
||||
// per line, '#' comments allowed). A single URL still works — it's just a 1-element list.
|
||||
// Returns null if neither source yields any proxies, so callers can treat "no proxy
|
||||
// Returns undefined if neither source yields any proxies, so callers can treat "no proxy
|
||||
// configured" the same way they did with the old single-string PROXY_URL/RESIDENTIAL_PROXY_URL.
|
||||
static fromEnv(urlListEnv?: string, fileEnv?: string): ProxyPool | null {
|
||||
static fromEnv(urlListEnv?: string, fileEnv?: string) {
|
||||
const urls: string[] = []
|
||||
if (urlListEnv) {
|
||||
urls.push(
|
||||
@@ -76,7 +80,7 @@ export class ProxyPool {
|
||||
console.warn(`[proxy] failed to read proxy list file ${fileEnv}:`, err instanceof Error ? err.message : err)
|
||||
}
|
||||
}
|
||||
return urls.length > 0 ? new ProxyPool(urls) : null
|
||||
return urls.length > 0 ? new ProxyPool(urls) : undefined
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
@@ -91,9 +95,9 @@ export class ProxyPool {
|
||||
// Sticky-per-domain: reuse the same proxy for repeat requests to a domain (consistency
|
||||
// helps avoid re-triggering challenges); falls back to round-robin across available
|
||||
// proxies for new domains or once the sticky proxy has been marked bad.
|
||||
next(domain?: string): string | null {
|
||||
next(domain?: string) {
|
||||
const available = this.available()
|
||||
if (available.length === 0) return null
|
||||
if (available.length === 0) return
|
||||
|
||||
if (domain) {
|
||||
const sticky = this.stickyByDomain.get(domain)
|
||||
@@ -106,9 +110,9 @@ export class ProxyPool {
|
||||
return proxy.url
|
||||
}
|
||||
|
||||
random(): string | null {
|
||||
random() {
|
||||
const available = this.available()
|
||||
if (available.length === 0) return null
|
||||
if (available.length === 0) return
|
||||
return available[Math.floor(Math.random() * available.length)].url
|
||||
}
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ describe("normalizeProxy", () => {
|
||||
})
|
||||
|
||||
test("non-string url field → undefined", () => {
|
||||
expect(normalizeProxy({ url: 42 as unknown as string })).toBeUndefined()
|
||||
expect(normalizeProxy({ url: 42 })).toBeUndefined()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -33,12 +33,12 @@ describe("ProxyPool", () => {
|
||||
expect(after).toBeTruthy()
|
||||
})
|
||||
|
||||
test("returns null once every proxy is marked bad", () => {
|
||||
test("returns undefined once every proxy is marked bad", () => {
|
||||
const pool = new ProxyPool(["http://p1:8080", "http://p2:8080"])
|
||||
pool.markBad("http://p1:8080")
|
||||
pool.markBad("http://p2:8080")
|
||||
expect(pool.next("example.com")).toBeNull()
|
||||
expect(pool.random()).toBeNull()
|
||||
expect(pool.next("example.com")).toBeUndefined()
|
||||
expect(pool.random()).toBeUndefined()
|
||||
})
|
||||
|
||||
test("random() returns one of the configured proxies", () => {
|
||||
@@ -49,10 +49,10 @@ describe("ProxyPool", () => {
|
||||
}
|
||||
})
|
||||
|
||||
test("returns null for an empty pool", () => {
|
||||
test("returns undefined for an empty pool", () => {
|
||||
const pool = new ProxyPool([])
|
||||
expect(pool.next()).toBeNull()
|
||||
expect(pool.random()).toBeNull()
|
||||
expect(pool.next()).toBeUndefined()
|
||||
expect(pool.random()).toBeUndefined()
|
||||
expect(pool.size).toBe(0)
|
||||
})
|
||||
|
||||
@@ -74,9 +74,9 @@ describe("ProxyPool", () => {
|
||||
expect(pool?.size).toBe(3)
|
||||
})
|
||||
|
||||
test("returns null when neither source has any proxies", () => {
|
||||
expect(ProxyPool.fromEnv(undefined, undefined)).toBeNull()
|
||||
expect(ProxyPool.fromEnv("", "")).toBeNull()
|
||||
test("returns undefined when neither source has any proxies", () => {
|
||||
expect(ProxyPool.fromEnv()).toBeUndefined()
|
||||
expect(ProxyPool.fromEnv("", "")).toBeUndefined()
|
||||
})
|
||||
|
||||
test("reads proxies from a line-delimited file, ignoring comments and blank lines", () => {
|
||||
|
||||
Reference in New Issue
Block a user