mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: repair URL import (DNS-pinned fetch on Node 22 + non-image modalities) (#246)
* fix(ssrf): honor lookup all-option so DNS-pinned fetch works on Node 22
createPinnedAgent's custom lookup always called back in single-address
form. Node 22 invokes the agent lookup with { all: true }, so the address
arrived as undefined and every URL fetch failed with "Invalid IP address:
undefined" -- breaking the URL-import feature entirely. Return the pinned
IP as an array when all is requested; the connection is still pinned to the
SSRF-validated public IP (no DNS-rebinding regression).
* fix(url-import): accept non-image modalities
fetch-urls validated every fetched file as an image and ran a Sharp
preview, so audio/video/document URL imports were rejected. Try image
validation, accept non-image media (typed from the HTTP content-type),
and skip the image preview for non-images. SSRF + size limits unchanged.
This commit is contained in:
@@ -108,13 +108,26 @@ export const URL_FETCH_CONCURRENCY = 4;
|
|||||||
* (private) IP between our SSRF validation and the actual connection.
|
* (private) IP between our SSRF validation and the actual connection.
|
||||||
*/
|
*/
|
||||||
function createPinnedAgent(resolvedIp: string, protocol: string): http.Agent | https.Agent {
|
function createPinnedAgent(resolvedIp: string, protocol: string): http.Agent | https.Agent {
|
||||||
|
const family = resolvedIp.includes(":") ? 6 : 4;
|
||||||
|
// Node's lookup contract: when called with { all: true } the callback must
|
||||||
|
// return the address(es) as an array. Node 22 invokes the agent's custom
|
||||||
|
// lookup this way, so a single-arg callback passes `undefined` as the
|
||||||
|
// address and every fetch fails with "Invalid IP address: undefined".
|
||||||
|
// Honor both forms while still pinning to the SSRF-validated public IP.
|
||||||
const pinnedLookup: (
|
const pinnedLookup: (
|
||||||
hostname: string,
|
hostname: string,
|
||||||
options: object,
|
options: { all?: boolean },
|
||||||
callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void,
|
callback: (
|
||||||
) => void = (_hostname, _options, callback) => {
|
err: NodeJS.ErrnoException | null,
|
||||||
const family = resolvedIp.includes(":") ? 6 : 4;
|
address: string | Array<{ address: string; family: number }>,
|
||||||
callback(null, resolvedIp, family);
|
family?: number,
|
||||||
|
) => void,
|
||||||
|
) => void = (_hostname, options, callback) => {
|
||||||
|
if (options?.all) {
|
||||||
|
callback(null, [{ address: resolvedIp, family }]);
|
||||||
|
} else {
|
||||||
|
callback(null, resolvedIp, family);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if (protocol === "https:") {
|
if (protocol === "https:") {
|
||||||
|
|||||||
@@ -80,8 +80,8 @@ interface SuccessResult {
|
|||||||
filename: string;
|
filename: string;
|
||||||
contentType: string;
|
contentType: string;
|
||||||
size: number;
|
size: number;
|
||||||
width: number;
|
width?: number;
|
||||||
height: number;
|
height?: number;
|
||||||
downloadUrl: string;
|
downloadUrl: string;
|
||||||
previewUrl: string | null;
|
previewUrl: string | null;
|
||||||
}
|
}
|
||||||
@@ -236,21 +236,25 @@ async function fetchSingleUrl(
|
|||||||
const rawFilename = filenameFromUrl(url);
|
const rawFilename = filenameFromUrl(url);
|
||||||
const filename = getUniqueName(sanitizeFilename(rawFilename), usedFilenames);
|
const filename = getUniqueName(sanitizeFilename(rawFilename), usedFilenames);
|
||||||
|
|
||||||
// Validate as an image
|
// Try image validation; non-image media (audio/video/document/data) is
|
||||||
const validation = await validateImageBuffer(buffer, filename);
|
// accepted and typed from the HTTP response so URL import works for every
|
||||||
if (!validation.valid) {
|
// modality. The consuming tool runs modality-specific validation at
|
||||||
return { success: false, url, error: validation.reason };
|
// process time, and SSRF + size limits still bound what can be fetched.
|
||||||
}
|
const validation = await validateImageBuffer(buffer, filename).catch(() => null);
|
||||||
|
|
||||||
// Save to object storage uploads prefix (raw fetched files)
|
// Save to object storage uploads prefix (raw fetched files)
|
||||||
await putObject(`uploads/${jobId}/${filename}`, buffer);
|
await putObject(`uploads/${jobId}/${filename}`, buffer);
|
||||||
|
|
||||||
const contentType = FORMAT_TO_MIME[validation.format] ?? "application/octet-stream";
|
const responseContentType = response.headers.get("content-type")?.split(";")[0]?.trim();
|
||||||
|
const contentType = validation?.valid
|
||||||
|
? (FORMAT_TO_MIME[validation.format] ?? "application/octet-stream")
|
||||||
|
: responseContentType || "application/octet-stream";
|
||||||
const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
|
const downloadUrl = `/api/v1/download/${jobId}/${encodeURIComponent(filename)}`;
|
||||||
|
|
||||||
// Generate preview for non-browser formats
|
// Generate a webp preview only for non-browser-native image formats.
|
||||||
|
// Non-image media has no image preview; the UI shows a modality icon.
|
||||||
let previewUrl: string | null = null;
|
let previewUrl: string | null = null;
|
||||||
if (!BROWSER_PREVIEWABLE.has(contentType)) {
|
if (validation?.valid && !BROWSER_PREVIEWABLE.has(contentType)) {
|
||||||
try {
|
try {
|
||||||
const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
|
const previewBuffer = await sharp(buffer).webp({ quality: 80 }).toBuffer();
|
||||||
const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
|
const previewFilename = `preview-${filename.replace(/\.[^.]+$/, "")}.webp`;
|
||||||
@@ -267,8 +271,8 @@ async function fetchSingleUrl(
|
|||||||
filename,
|
filename,
|
||||||
contentType,
|
contentType,
|
||||||
size: buffer.length,
|
size: buffer.length,
|
||||||
width: validation.width,
|
width: validation?.valid ? validation.width : undefined,
|
||||||
height: validation.height,
|
height: validation?.valid ? validation.height : undefined,
|
||||||
downloadUrl,
|
downloadUrl,
|
||||||
previewUrl,
|
previewUrl,
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user