fix: repair URL import (DNS-pinned fetch on Node 22 + non-image modalities) (#246)

* fix(ssrf): honor lookup all-option so DNS-pinned fetch works on Node 22

createPinnedAgent's custom lookup always called back in single-address
form. Node 22 invokes the agent lookup with { all: true }, so the address
arrived as undefined and every URL fetch failed with "Invalid IP address:
undefined" -- breaking the URL-import feature entirely. Return the pinned
IP as an array when all is requested; the connection is still pinned to the
SSRF-validated public IP (no DNS-rebinding regression).

* fix(url-import): accept non-image modalities

fetch-urls validated every fetched file as an image and ran a Sharp
preview, so audio/video/document URL imports were rejected. Try image
validation, accept non-image media (typed from the HTTP content-type),
and skip the image preview for non-images. SSRF + size limits unchanged.
This commit is contained in:
SnapOtter
2026-06-16 14:33:30 +08:00
committed by GitHub
parent aa3ae6ec91
commit 8f6312c521
2 changed files with 34 additions and 17 deletions
+18 -5
View File
@@ -108,13 +108,26 @@ export const URL_FETCH_CONCURRENCY = 4;
* (private) IP between our SSRF validation and the actual connection.
*/
function createPinnedAgent(resolvedIp: string, protocol: string): http.Agent | https.Agent {
const family = resolvedIp.includes(":") ? 6 : 4;
// Node's lookup contract: when called with { all: true } the callback must
// return the address(es) as an array. Node 22 invokes the agent's custom
// lookup this way, so a single-arg callback passes `undefined` as the
// address and every fetch fails with "Invalid IP address: undefined".
// Honor both forms while still pinning to the SSRF-validated public IP.
const pinnedLookup: (
hostname: string,
options: object,
callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void,
) => void = (_hostname, _options, callback) => {
const family = resolvedIp.includes(":") ? 6 : 4;
callback(null, resolvedIp, family);
options: { all?: boolean },
callback: (
err: NodeJS.ErrnoException | null,
address: string | Array<{ address: string; family: number }>,
family?: number,
) => void,
) => void = (_hostname, options, callback) => {
if (options?.all) {
callback(null, [{ address: resolvedIp, family }]);
} else {
callback(null, resolvedIp, family);
}
};
if (protocol === "https:") {
+16 -12
View File
@@ -80,8 +80,8 @@ interface SuccessResult {
filename: string;
contentType: string;
size: number;
width: number;
height: number;
width?: number;
height?: number;
downloadUrl: string;
previewUrl: string | null;
}
@@ -236,21 +236,25 @@ async function fetchSingleUrl(
const rawFilename = filenameFromUrl(url);
const filename = getUniqueName(sanitizeFilename(rawFilename), usedFilenames);
// Validate as an image
const validation = await validateImageBuffer(buffer, filename);
if (!validation.valid) {
return { success: false, url, error: validation.reason };
}
// Try image validation; non-image media (audio/video/document/data) is
// accepted and typed from the HTTP response so URL import works for every
// modality. The consuming tool runs modality-specific validation at
// process time, and SSRF + size limits still bound what can be fetched.
const validation = await validateImageBuffer(buffer, filename).catch(() => null);
// Save to object storage uploads prefix (raw fetched files)
await putObject(`uploads/${jobId}/${filename}`, buffer);
const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
const responseContentType = response.headers.get("content-type")?.split(";")[0]?.trim();
const contentType = validation?.valid
? (FORMAT_TO_MIME[validation.format] ?? "application/octet-stream")
: responseContentType || "application/octet-stream";
const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
// Generate preview for non-browser formats
// Generate a webp preview only for non-browser-native image formats.
// Non-image media has no image preview; the UI shows a modality icon.
let previewUrl: string | null = null;
if (!BROWSER_PREVIEWABLE.has(contentType)) {
if (validation?.valid && !BROWSER_PREVIEWABLE.has(contentType)) {
try {
const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
@@ -267,8 +271,8 @@ async function fetchSingleUrl(
filename,
contentType,
size: buffer.length,
width: validation.width,
height: validation.height,
width: validation?.valid ? validation.width : undefined,
height: validation?.valid ? validation.height : undefined,
downloadUrl,
previewUrl,
};