From 75983c323d3ee5e36a3841ed98e56f5d48f8ce92 Mon Sep 17 00:00:00 2001 From: RonCodes88 Date: Sun, 23 Aug 2026 23:50:07 +0800 Subject: [PATCH] feat: route outbound fetches through HTTP_PROXY, HTTPS_PROXY, and NO_PROXY --- src/fetch.js | 236 +++++++++++++++++++++++++++++++++++++++++-- tests/fetch.test.js | 238 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 464 insertions(+), 10 deletions(-) diff --git a/src/fetch.js b/src/fetch.js index 0e2ab88..94b587b 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -8,7 +8,10 @@ */ import dns from 'node:dns/promises'; +import http from 'node:http'; +import https from 'node:https'; import net from 'node:net'; +import tls from 'node:tls'; // The fetch fallback can't fake a TLS fingerprint like impers does, but it // should at least send the same Chrome identity in its headers. @@ -146,6 +149,214 @@ async function assertSafeTarget(urlStr) { } } +function envFirst(env, ...names) { + for (const name of names) { + const value = env[name]; + if (value) return value; + } +} + +function normalizeProxy(value) { + if (value == null) return null; + const trimmed = String(value).trim(); + if (!trimmed) return null; + return /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `http://${trimmed}`; +} + +// Split host[:port], including [IPv6]:port. A trailing :digits is a port; +// anything else stays part of the hostname so NO_PROXY=example.com matches. +function splitHostPort(entry) { + if (entry.startsWith('[')) { + const end = entry.indexOf(']'); + if (end === -1) return { host: entry, port: '' }; + const rest = entry.slice(end + 1); + return { host: entry.slice(1, end), port: rest.startsWith(':') ? rest.slice(1) : '' }; + } + const colon = entry.lastIndexOf(':'); + if (colon !== -1 && /^\d+$/.test(entry.slice(colon + 1))) { + return { host: entry.slice(0, colon), port: entry.slice(colon + 1) }; + } + return { host: entry, port: '' }; +} + +function bypassesProxy(target, noProxy) { + const list = noProxy.trim(); + if (!list) return false; + const hostname = target.hostname.toLowerCase().replace(/^\[|\]$/g, ''); + const port = target.port || (target.protocol === 'https:' ? '443' : '80'); + for (let entry of list.split(',')) { + entry = entry.trim(); + if (!entry) continue; + if (entry === '*') return true; + const { host: rawHost, port: entryPort } = splitHostPort(entry); + if (entryPort && entryPort !== port) continue; + const host = rawHost.toLowerCase().replace(/^\[|\]$/g, '').replace(/^\./, ''); + if (!host) continue; + if (hostname === host || hostname.endsWith(`.${host}`)) return true; + } + return false; +} + +/** + * Pick a proxy for this URL from HTTP_PROXY / HTTPS_PROXY / NO_PROXY (and + * their lowercase forms). The proxy host is not run through assertSafeTarget: + * corporate proxies live on loopback or RFC 1918 addresses, and they are not + * the page being fetched. Redirect hops still go through that check. + * @param {string} url + * @param {NodeJS.ProcessEnv} [env] + * @returns {string | null} proxy URL, or null to connect directly + */ +export function resolveProxy(url, env = process.env) { + const target = new URL(url); + if (bypassesProxy(target, envFirst(env, 'NO_PROXY', 'no_proxy') ?? '')) return null; + const httpsProxy = envFirst(env, 'HTTPS_PROXY', 'https_proxy'); + const httpProxy = envFirst(env, 'HTTP_PROXY', 'http_proxy'); + const chosen = target.protocol === 'https:' ? (httpsProxy || httpProxy) : httpProxy; + return normalizeProxy(chosen); +} + +function proxyAuthHeader(proxy) { + if (!proxy.username) return undefined; + const token = Buffer.from(`${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`).toString('base64'); + return `Basic ${token}`; +} + +function authority(target) { + const host = net.isIP(target.hostname) === 6 ? `[${target.hostname}]` : target.hostname; + const port = target.port || (target.protocol === 'https:' ? '443' : '80'); + return `${host}:${port}`; +} + +function wrapNodeResponse(res, url) { + const headers = { + get(name) { + const v = res.headers[name.toLowerCase()]; + if (v == null) return null; + return Array.isArray(v) ? v.join(', ') : v; + }, + }; + const text = () => new Promise((resolve, reject) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + res.on('error', reject); + }); + const status = res.statusCode ?? 0; + return { + status, + statusText: res.statusMessage || '', + headers, + ok: status >= 200 && status < 300, + url, + text, + }; +} + +function proxyTransport(proxy) { + return proxy.protocol === 'https:' ? https : http; +} + +function proxyPort(proxy) { + return Number(proxy.port) || (proxy.protocol === 'https:' ? 443 : 80); +} + +function httpViaProxy(target, proxy, headers) { + const auth = proxyAuthHeader(proxy); + return new Promise((resolve, reject) => { + const req = proxyTransport(proxy).request({ + hostname: proxy.hostname, + port: proxyPort(proxy), + method: 'GET', + path: target.href, + headers: { + ...headers, + host: target.host, + ...(auth && { 'proxy-authorization': auth }), + }, + }, (res) => resolve(wrapNodeResponse(res, target.href))); + req.on('error', (err) => reject(new Error(`proxy failed: ${err.message} for ${target.href}`))); + req.end(); + }); +} + +function httpsViaConnect(target, proxy, headers, tlsOpts = {}) { + const dest = authority(target); + const auth = proxyAuthHeader(proxy); + return new Promise((resolve, reject) => { + let settled = false; + const fail = (err) => { + if (settled) return; + settled = true; + req.destroy(); + reject(err instanceof Error ? err : new Error(String(err))); + }; + const req = proxyTransport(proxy).request({ + hostname: proxy.hostname, + port: proxyPort(proxy), + method: 'CONNECT', + path: dest, + headers: { + host: dest, + ...(auth && { 'proxy-authorization': auth }), + }, + }); + req.on('connect', (res, socket, head) => { + if (res.statusCode !== 200) { + socket.destroy(); + fail(new Error(`proxy CONNECT failed: ${res.statusCode} for ${target.href}`)); + return; + } + if (head.length) socket.unshift(head); + // tls.connect already opened the tunnel. https.request would wrap TLS + // again, and the origin would see a second ClientHello as garbage. + // SNI is a hostname; an IP literal is only used for the cert check. + const hostname = target.hostname; + const tlsSocket = tls.connect({ + socket, + host: hostname, + ...(net.isIP(hostname) ? {} : { servername: hostname }), + ...tlsOpts, + }, () => { + const tunneled = http.request({ + createConnection: () => tlsSocket, + path: `${target.pathname}${target.search}`, + method: 'GET', + headers: { ...headers, host: target.host }, + }, (httpsRes) => { + if (settled) return; + settled = true; + resolve(wrapNodeResponse(httpsRes, target.href)); + }); + tunneled.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`))); + tunneled.end(); + }); + tlsSocket.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`))); + }); + req.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`))); + req.end(); + }); +} + +/** + * One GET through an HTTP(S) proxy. HTTP targets use the absolute-URI form; + * HTTPS targets open a CONNECT tunnel first. Redirects are not followed: + * followRedirects owns that so each hop still goes through assertSafeTarget. + * @param {string} url + * @param {string} proxy + * @param {Record} [headers] + * @param {import('node:tls').ConnectionOptions} [tlsOpts] + */ +export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) { + const target = new URL(url); + const proxyUrl = new URL(proxy); + if (!/^https?:$/.test(proxyUrl.protocol)) { + throw new Error(`unsupported proxy protocol (${proxyUrl.protocol.slice(0, -1)}), oc honors HTTP and HTTPS proxies`); + } + return target.protocol === 'https:' + ? httpsViaConnect(target, proxyUrl, headers, tlsOpts) + : httpViaProxy(target, proxyUrl, headers); +} + /** * Fetch a page. * @param {string} url - with or without a scheme, https is assumed @@ -190,10 +401,19 @@ export async function followRedirects(get, start) { } } +const FETCH_HEADERS = { + 'user-agent': UA, + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.9', +}; + async function viaImpers(impers, target) { // Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a // blocked first attempt gets one cheap retry with a second identity. - const asking = (impersonate) => (url) => impers.get(url, { impersonate, allowRedirects: false }); + const asking = (impersonate) => (url) => { + const proxy = resolveProxy(url); + return impers.get(url, { impersonate, allowRedirects: false, ...(proxy && { proxy }) }); + }; let via = 'impers:chrome'; let { res } = await followRedirects(asking('chrome'), target); let status = res.status ?? res.statusCode ?? 0; @@ -209,14 +429,12 @@ async function viaImpers(impers, target) { } async function viaFetch(target) { - const { res, url: current } = await followRedirects((url) => fetch(url, { - redirect: 'manual', - headers: { - 'user-agent': UA, - accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'accept-language': 'en-US,en;q=0.9', - }, - }), target); + const { res, url: current } = await followRedirects((url) => { + const proxy = resolveProxy(url); + return proxy + ? proxyGet(url, proxy, FETCH_HEADERS) + : fetch(url, { redirect: 'manual', headers: FETCH_HEADERS }); + }, target); if (!res.ok) { throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`); } diff --git a/tests/fetch.test.js b/tests/fetch.test.js index 90bd1c1..0ac71b3 100644 --- a/tests/fetch.test.js +++ b/tests/fetch.test.js @@ -1,7 +1,10 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; -const { fetchPage, followRedirects } = await import('../src/fetch.js'); +const { fetchPage, followRedirects, resolveProxy, proxyGet } = await import('../src/fetch.js'); const BLOCKED_MESSAGE = 'blocked: private or internal URL'; @@ -135,3 +138,236 @@ test('the readable-type gate accepts text and refuses binary, on either transpor } assert.throws(() => assertReadableType('image/png'), /image\/png/); }); + +test('resolveProxy reads the usual env vars and honors NO_PROXY', () => { + const none = {}; + assert.equal(resolveProxy('https://example.com', none), null); + + assert.equal( + resolveProxy('https://example.com', { HTTPS_PROXY: 'http://proxy.corp:8080' }), + 'http://proxy.corp:8080', + ); + assert.equal( + resolveProxy('https://example.com', { HTTP_PROXY: 'http://proxy.corp:8080' }), + 'http://proxy.corp:8080', + ); + assert.equal( + resolveProxy('http://example.com', { HTTP_PROXY: 'http://proxy.corp:8080' }), + 'http://proxy.corp:8080', + ); + assert.equal( + resolveProxy('http://example.com', { HTTPS_PROXY: 'http://secure-proxy.corp:8080' }), + null, + ); + assert.equal( + resolveProxy('https://example.com', { http_proxy: 'proxy.corp:8080' }), + 'http://proxy.corp:8080', + ); + + assert.equal( + resolveProxy('https://example.com/foo', { HTTPS_PROXY: 'http://proxy.corp:8080', NO_PROXY: 'example.com' }), + null, + ); + assert.equal( + resolveProxy('https://foo.example.com', { HTTPS_PROXY: 'http://proxy.corp:8080', NO_PROXY: '.example.com' }), + null, + ); + assert.equal( + resolveProxy('https://elsewhere.test', { HTTPS_PROXY: 'http://proxy.corp:8080', NO_PROXY: 'example.com' }), + 'http://proxy.corp:8080', + ); + assert.equal( + resolveProxy('https://example.com', { HTTPS_PROXY: 'http://proxy.corp:8080', no_proxy: '*' }), + null, + ); + assert.equal( + resolveProxy('https://example.com:8443', { HTTPS_PROXY: 'http://proxy.corp:8080', NO_PROXY: 'example.com:8443' }), + null, + ); + assert.equal( + resolveProxy('https://example.com:8443', { HTTPS_PROXY: 'http://proxy.corp:8080', NO_PROXY: 'example.com:443' }), + 'http://proxy.corp:8080', + ); +}); + +test('a private page URL is still blocked when a proxy is configured', async () => { + // The proxy itself is often loopback; that must not punch a hole in the + // page-target guard. fetchPage rejects before any socket is opened. + const keys = ['HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy']; + const prev = Object.fromEntries(keys.map((k) => [k, process.env[k]])); + for (const k of keys) delete process.env[k]; + process.env.HTTP_PROXY = 'http://127.0.0.1:8080'; + process.env.HTTPS_PROXY = 'http://127.0.0.1:8080'; + try { + await assert.rejects(() => fetchPage('127.0.0.1'), new RegExp(BLOCKED_MESSAGE)); + await assert.rejects(() => fetchPage('https://192.168.1.1/admin'), new RegExp(BLOCKED_MESSAGE)); + } finally { + for (const k of keys) { + if (prev[k] === undefined) delete process.env[k]; + else process.env[k] = prev[k]; + } + } +}); + +function listen(server) { + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(server.address().port)); + }); +} + +test('proxyGet sends an absolute-URI GET to an HTTP proxy', async () => { + const seen = []; + const proxy = http.createServer((req, res) => { + seen.push({ + method: req.method, + url: req.url, + host: req.headers.host, + ua: req.headers['user-agent'], + auth: req.headers['proxy-authorization'], + }); + res.writeHead(200, { 'content-type': 'text/html' }); + res.end('via proxy'); + }); + const port = await listen(proxy); + try { + const res = await proxyGet( + 'http://example.test/page', + `http://user:secret@127.0.0.1:${port}`, + { 'user-agent': 'oc-test' }, + ); + assert.equal(res.status, 200); + assert.equal(res.headers.get('content-type'), 'text/html'); + assert.equal(await res.text(), 'via proxy'); + assert.equal(seen.length, 1); + assert.equal(seen[0].method, 'GET'); + assert.equal(seen[0].url, 'http://example.test/page'); + assert.equal(seen[0].host, 'example.test'); + assert.equal(seen[0].ua, 'oc-test'); + assert.equal(seen[0].auth, `Basic ${Buffer.from('user:secret').toString('base64')}`); + } finally { + proxy.close(); + } +}); + +test('proxyGet issues CONNECT for an HTTPS target and fails loud on a refused tunnel', async () => { + const seen = []; + const proxy = http.createServer(); + proxy.on('connect', (req, socket) => { + seen.push({ url: req.url, auth: req.headers['proxy-authorization'] }); + socket.write('HTTP/1.1 403 Forbidden\r\n\r\n'); + socket.end(); + }); + const port = await listen(proxy); + try { + await assert.rejects( + () => proxyGet('https://example.test/page', `http://127.0.0.1:${port}`), + /proxy CONNECT failed: 403/, + ); + assert.equal(seen.length, 1); + assert.equal(seen[0].url, 'example.test:443'); + } finally { + proxy.close(); + } +}); + +// Self-signed localhost cert so the HTTPS success path can run offline. The +// client is handed the same cert as `ca`, so verification stays on. +const LOCAL_CERT = `-----BEGIN CERTIFICATE----- +MIIBmDCCAT+gAwIBAgIUMrBi6hKtC1hrvo+Ttf70VHSofLkwCgYIKoZIzj0EAwIw +FDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDgyMzE1MzU1MFoXDTM2MDgyMDE1 +MzU1MFowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAESbtFNb5R3K9iqcJJ6J9HII9DRylOGKutU+uoJ4TTsopcsRz2jMns8UYa ++oABlqC0ef+LAcaTwkPHgTwzfS1GuqNvMG0wHQYDVR0OBBYEFPPKq83hQvf8KTZB +r0bMcGIo18wBMB8GA1UdIwQYMBaAFPPKq83hQvf8KTZBr0bMcGIo18wBMA8GA1Ud +EwEB/wQFMAMBAf8wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAoGCCqGSM49 +BAMCA0cAMEQCIHBWYJSTt1qGyhySr2CY+JYWFdpApMvHVqED54/GivKcAiBchJFW +8FrIy8Paiv8v+us5Ahlpr1QheS5LZX+LUWPUIg== +-----END CERTIFICATE-----`; + +const LOCAL_KEY = `-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgtusH8iEEA2S7nGrF +DrJVWIHwY2v4DoYibYK0wTwAQEuhRANCAARJu0U1vlHcr2Kpwknon0cgj0NHKU4Y +q61T66gnhNOyilyxHPaMyezxRhr6gAGWoLR5/4sBxpPCQ8eBPDN9LUa6 +-----END PRIVATE KEY-----`; + +test('proxyGet returns the origin body through an HTTPS CONNECT tunnel', async () => { + // The 403 test above only proves we open the tunnel. This one proves the + // GET after TLS actually reaches the origin: the old path wrapped TLS twice + // and the origin never saw a request. + const originGot = []; + const origin = https.createServer({ cert: LOCAL_CERT, key: LOCAL_KEY }, (req, res) => { + originGot.push({ method: req.method, url: req.url, host: req.headers.host, ua: req.headers['user-agent'] }); + res.writeHead(200, { 'content-type': 'text/html' }); + res.end('via tunnel'); + }); + const originPort = await listen(origin); + + const connected = []; + const proxy = http.createServer(); + proxy.on('connect', (req, socket) => { + connected.push(req.url); + const { hostname, port } = new URL(`http://${req.url}`); + const dest = net.connect(Number(port), hostname, () => { + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + dest.pipe(socket); + socket.pipe(dest); + }); + dest.on('error', () => socket.destroy()); + }); + const proxyPort = await listen(proxy); + + try { + const res = await proxyGet( + `https://127.0.0.1:${originPort}/page`, + `http://127.0.0.1:${proxyPort}`, + { 'user-agent': 'oc-test' }, + { ca: LOCAL_CERT }, + ); + assert.equal(res.status, 200); + assert.equal(res.headers.get('content-type'), 'text/html'); + assert.equal(await res.text(), 'via tunnel'); + assert.deepEqual(connected, [`127.0.0.1:${originPort}`]); + assert.deepEqual(originGot, [{ + method: 'GET', + url: '/page', + host: `127.0.0.1:${originPort}`, + ua: 'oc-test', + }]); + } finally { + origin.close(); + proxy.close(); + } +}); + +test('followRedirects still blocks a private hop when the transport is a proxy', async () => { + // The page 302s to loopback. The proxy is also loopback, which is allowed; + // the hop is not. Blocked before the second request goes out. + let n = 0; + const proxy = http.createServer((req, res) => { + n += 1; + if (n === 1) { + res.writeHead(302, { location: 'http://127.0.0.1/admin' }); + res.end(); + return; + } + res.writeHead(200); + res.end('should not happen'); + }); + const port = await listen(proxy); + try { + await assert.rejects( + () => followRedirects((url) => proxyGet(url, `http://127.0.0.1:${port}`), 'http://public.example/start'), + new RegExp(BLOCKED_MESSAGE), + ); + assert.equal(n, 1); + } finally { + proxy.close(); + } +}); + +test('proxyGet refuses a non-HTTP proxy scheme', () => { + assert.throws( + () => proxyGet('https://example.test/', 'socks5://127.0.0.1:1080'), + /unsupported proxy protocol \(socks5\)/, + ); +});