diff --git a/src/fetch.js b/src/fetch.js index 54c29de..9318c85 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -7,6 +7,9 @@ * never import a browser here. */ +import dns from 'node:dns/promises'; +import net from 'node:net'; + // The fetch fallback can't fake a TLS fingerprint like impers does, but it // should at least send the same Chrome identity in its headers. const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'; @@ -18,6 +21,112 @@ const loadImpers = () => { return impersPromise; }; +const BLOCKED_MESSAGE = 'blocked: private or internal URL'; +const MAX_REDIRECTS = 20; + +// IPv4 ranges with no business receiving a server-initiated fetch: loopback, +// link-local, the three RFC 1918 private blocks, carrier-grade NAT, the +// unspecified/broadcast addresses, and the documentation/benchmark ranges. +const IPV4_BLOCKED_RANGES = [ + ['0.0.0.0', 8], + ['10.0.0.0', 8], + ['100.64.0.0', 10], + ['127.0.0.0', 8], + ['169.254.0.0', 16], + ['172.16.0.0', 12], + ['192.0.0.0', 24], + ['192.0.2.0', 24], + ['192.168.0.0', 16], + ['198.18.0.0', 15], + ['198.51.100.0', 24], + ['203.0.113.0', 24], + ['224.0.0.0', 4], + ['240.0.0.0', 4], + ['255.255.255.255', 32], +]; + +function ipv4ToInt(ip) { + const parts = ip.split('.').map(Number); + return ((parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]) >>> 0; +} + +function isBlockedIPv4Int(addr) { + return IPV4_BLOCKED_RANGES.some(([base, bits]) => { + const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0; + return (addr & mask) === (ipv4ToInt(base) & mask); + }); +} + +function isBlockedIPv4(ip) { + return isBlockedIPv4Int(ipv4ToInt(ip)); +} + +// Expand a parsed IPv6 literal (as given by URL.hostname or dns.lookup, so +// already bracket-free and lowercase) into its 8 16-bit groups. +function expandIPv6(ip) { + const sides = ip.split('::'); + if (sides.length > 2) return null; + const head = sides[0] ? sides[0].split(':') : []; + const tail = sides.length === 2 && sides[1] ? sides[1].split(':') : []; + const missing = 8 - head.length - tail.length; + if (missing < 0) return null; + const groups = [...head, ...Array(missing).fill('0'), ...tail]; + if (groups.length !== 8) return null; + return groups.map((g) => parseInt(g, 16)); +} + +function isBlockedIPv6(ip) { + const g = expandIPv6(ip); + if (!g) return true; // unparsable - fail closed + if (g.every((x) => x === 0)) return true; // :: (unspecified) + if (g[0] === 0 && g[1] === 0 && g[2] === 0 && g[3] === 0 && g[4] === 0 && g[5] === 0 && g[6] === 0 && g[7] === 1) { + return true; // ::1 (loopback) + } + if (g[0] === 0 && g[1] === 0 && g[2] === 0 && g[3] === 0 && g[4] === 0 && (g[5] === 0xffff || g[5] === 0)) { + // ::ffff:a.b.c.d (IPv4-mapped) or the deprecated ::a.b.c.d (IPv4-compatible) + return isBlockedIPv4Int(((g[6] << 16) | g[7]) >>> 0); + } + if ((g[0] & 0xfe00) === 0xfc00) return true; // fc00::/7 (unique local) + if ((g[0] & 0xffc0) === 0xfe80) return true; // fe80::/10 (link-local) + return false; +} + +// Validates the target a socket is actually about to connect to: the parsed +// IP if the URL is a literal, or every address the hostname resolves to +// otherwise. Resolving before connecting (rather than pattern-matching the +// hostname string) is what closes off IPv4-mapped IPv6 loopback, 0.0.0.0, +// and DNS rebinding through an attacker-controlled domain. It runs again on +// every redirect hop, since a public URL that later 302s to an internal +// address is the same attack one step removed. +// +// It does not pin the resolved address for the connection itself - neither +// impers (curl) nor Node's fetch expose that here - so a name that +// re-resolves to a different address between this check and the actual +// connect is a known residual gap, not one this guard can close without +// deeper transport changes. +async function assertSafeTarget(urlStr) { + const u = new URL(urlStr); + if (!/^https?:$/.test(u.protocol)) throw new Error(BLOCKED_MESSAGE); + // URL.hostname keeps the brackets around an IPv6 literal (e.g. "[::1]"); + // net.isIP and the group-based checks below expect the bare address. + const hostname = u.hostname.toLowerCase().replace(/^\[|\]$/g, ''); + const ipType = net.isIP(hostname); + if (ipType === 4) { + if (isBlockedIPv4(hostname)) throw new Error(BLOCKED_MESSAGE); + return; + } + if (ipType === 6) { + if (isBlockedIPv6(hostname)) throw new Error(BLOCKED_MESSAGE); + return; + } + if (hostname === 'localhost') throw new Error(BLOCKED_MESSAGE); + const addresses = await dns.lookup(hostname, { all: true }).catch(() => []); + for (const { address, family } of addresses) { + if (family === 4 && isBlockedIPv4(address)) throw new Error(BLOCKED_MESSAGE); + if (family === 6 && isBlockedIPv6(address)) throw new Error(BLOCKED_MESSAGE); + } +} + /** * Fetch a page. * @param {string} url - with or without a scheme, https is assumed @@ -27,24 +136,36 @@ const loadImpers = () => { */ export async function fetchPage(url) { const target = /^https?:\/\//i.test(url) ? url : `https://${url}`; - const hostname = new URL(target).hostname.toLowerCase(); - if (/^(localhost|127\.|10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|169\.254\.|::1$|\[::1\])/.test(hostname)) { - throw new Error(`blocked: private or internal URL`); - } + await assertSafeTarget(target); const impers = await loadImpers(); return impers ? viaImpers(impers, target) : viaFetch(target); } +async function followImpersRedirects(impers, startUrl, impersonate) { + let current = startUrl; + for (let i = 0; ; i++) { + if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${startUrl}`); + const res = await impers.get(current, { impersonate, allowRedirects: false }); + const status = res.status ?? res.statusCode ?? 0; + const location = res.headers.get('location'); + if (status >= 300 && status < 400 && location) { + current = new URL(location, current).toString(); + await assertSafeTarget(current); + continue; + } + return res; + } +} + async function viaImpers(impers, target) { // Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a // blocked first attempt gets one cheap retry with a second identity. let via = 'impers:chrome'; - let res = await impers.get(target, { impersonate: 'chrome' }); - // impers mirrors the curl_cffi response shape, not the WHATWG one. + let res = await followImpersRedirects(impers, target, 'chrome'); let status = res.status ?? res.statusCode ?? 0; if (status >= 400) { via = 'impers:firefox'; - res = await impers.get(target, { impersonate: 'firefox' }); + res = await followImpersRedirects(impers, target, 'firefox'); status = res.status ?? res.statusCode ?? 0; } if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`); @@ -53,20 +174,32 @@ async function viaImpers(impers, target) { } async function viaFetch(target) { - const res = await fetch(target, { - redirect: 'follow', - headers: { - 'user-agent': UA, - accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'accept-language': 'en-US,en;q=0.9', - }, - }); + let current = target; + let res; + for (let i = 0; ; i++) { + if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${target}`); + res = await fetch(current, { + redirect: 'manual', + headers: { + 'user-agent': UA, + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.9', + }, + }); + const location = res.headers.get('location'); + if (res.status >= 300 && res.status < 400 && location) { + current = new URL(location, current).toString(); + await assertSafeTarget(current); + continue; + } + break; + } if (!res.ok) { - throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${target}`); + throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`); } const type = res.headers.get('content-type') ?? ''; if (type && !type.includes('html') && !type.includes('xml')) { throw new Error(`not an HTML page (${type.split(';')[0]}), nothing to distill`); } - return { url: res.url, html: await res.text(), status: res.status, via: 'fetch' }; + return { url: res.url || current, html: await res.text(), status: res.status, via: 'fetch' }; } diff --git a/tests/fetch.test.js b/tests/fetch.test.js new file mode 100644 index 0000000..48bceb6 --- /dev/null +++ b/tests/fetch.test.js @@ -0,0 +1,74 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +const { fetchPage } = await import('../src/fetch.js'); + +const BLOCKED_MESSAGE = 'blocked: private or internal URL'; + +test('fetchPage blocks literal loopback and RFC 1918 / link-local hosts', async () => { + const blocked = [ + 'localhost', + 'localhost:8080', + '127.0.0.1', + '127.0.0.1/admin', + '10.0.0.1', + '192.168.1.1/admin', + '172.16.0.5', + '172.31.255.255', + '169.254.169.254/latest/meta-data/', + '0.0.0.0', + '[::1]', + ]; + for (const host of blocked) { + await assert.rejects(() => fetchPage(host), new RegExp(BLOCKED_MESSAGE), `expected ${host} to be blocked`); + } +}); + +test('fetchPage does not block an ordinary public hostname', async () => { + // A live fetch of example.com should succeed outright, or at worst fail for + // a network reason - it must never be rejected by the private-URL guard. + try { + await fetchPage('example.com'); + } catch (err) { + assert.ok(!err.message.includes(BLOCKED_MESSAGE), `unexpected block: ${err.message}`); + } +}); + +test('fetchPage does not false-positive on a public hostname that merely starts with a private-looking numeric label', async () => { + // Regression check: an earlier version of this guard matched the URL's + // hostname STRING against ^-anchored prefixes like "10." and could not + // tell a private IPv4 octet from an ordinary DNS label, so a domain like + // 10.example.com (subdomain "10" of example.com) was wrongly blocked as if + // it were 10.0.0.0/8. Validating the resolved address instead of the + // string fixes this. + try { + await fetchPage('10.example.com'); + } catch (err) { + assert.ok(!err.message.includes(BLOCKED_MESSAGE), `unexpected block: ${err.message}`); + } +}); + +test('fetchPage blocks an IPv4-mapped IPv6 loopback literal', async () => { + // new URL('https://[::ffff:127.0.0.1]/').hostname === '::ffff:7f00:1' + // (compressed hex) - a string blocklist checking for "127." or "::1" never + // matches this form, so it has to be decoded and checked as the IPv4 + // address it embeds. + await assert.rejects(() => fetchPage('https://[::ffff:127.0.0.1]/'), new RegExp(BLOCKED_MESSAGE)); +}); + +test('fetchPage blocks a hostname that merely resolves to a loopback address (DNS rebinding shape)', async () => { + // localtest.me is a public, real-world domain that resolves to 127.0.0.1 / + // ::1. Its hostname string looks nothing like a private address, so this + // can only be caught by resolving it and validating the resulting IP - + // exactly the shape of a DNS-rebinding attack. + await assert.rejects(() => fetchPage('localtest.me'), new RegExp(BLOCKED_MESSAGE)); +}); + +test('fetchPage re-validates every redirect hop, not just the original URL', async () => { + // httpbin.org is a public host with no reason to be blocked itself; its + // /redirect-to endpoint 302s wherever it's told, which is exactly the + // shape of an SSRF that hides the real target behind a public-looking + // first hop. + const redirector = `https://httpbin.org/redirect-to?url=${encodeURIComponent('http://127.0.0.1/admin')}`; + await assert.rejects(() => fetchPage(redirector), new RegExp(BLOCKED_MESSAGE)); +});