feat: add smart URL parser for bulk import

This commit is contained in:
SnapOtter
2026-05-11 21:18:45 +08:00
parent 488ed95686
commit a2be47bd68
2 changed files with 115 additions and 0 deletions
+43
View File
@@ -0,0 +1,43 @@
function isValidHttpUrl(str: string): boolean {
try {
const url = new URL(str);
return url.protocol === "http:" || url.protocol === "https:";
} catch {
return false;
}
}
export function extractUrls(input: string): string[] {
const urls: string[] = [];
for (const rawLine of input.split("\n")) {
let line = rawLine.trim();
if (!line) continue;
// Strip numbered list prefixes: "1. ", "2) ", "3 "
line = line.replace(/^\d+[.)]?\s+/, "");
// Strip bullet prefixes: "- ", "* ", "+ "
line = line.replace(/^[-*+]\s+/, "");
// Extract from markdown links: [text](url)
const mdMatch = line.match(/\[.*?]\((https?:\/\/[^)]+)\)/);
if (mdMatch) {
urls.push(mdMatch[1]);
continue;
}
// Extract from HTML img tags: <img src="url">
const imgMatch = line.match(/<img[^>]+src=["'](https?:\/\/[^"']+)["']/i);
if (imgMatch) {
urls.push(imgMatch[1]);
continue;
}
line = line.trim();
if (isValidHttpUrl(line)) {
urls.push(line);
}
}
return [...new Set(urls)];
}
+72
View File
@@ -0,0 +1,72 @@
import { describe, expect, it } from "vitest";
import { extractUrls } from "../../../apps/web/src/lib/url-parser.js";
describe("extractUrls", () => {
it("extracts plain URLs one per line", () => {
const input = "https://example.com/a.jpg\nhttps://example.com/b.png";
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
});
it("strips numbered list prefixes", () => {
const input =
"1. https://example.com/a.jpg\n2) https://example.com/b.png\n3 https://example.com/c.webp";
expect(extractUrls(input)).toEqual([
"https://example.com/a.jpg",
"https://example.com/b.png",
"https://example.com/c.webp",
]);
});
it("strips bullet prefixes", () => {
const input =
"- https://example.com/a.jpg\n* https://example.com/b.png\n+ https://example.com/c.webp";
expect(extractUrls(input)).toEqual([
"https://example.com/a.jpg",
"https://example.com/b.png",
"https://example.com/c.webp",
]);
});
it("extracts URLs from markdown links", () => {
const input = "[Photo 1](https://example.com/a.jpg)\n[Photo 2](https://example.com/b.png)";
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
});
it("extracts URLs from HTML img tags", () => {
const input = '<img src="https://example.com/a.jpg">\n<img src="https://example.com/b.png" />';
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
});
it("handles mixed formats", () => {
const input = `1. https://example.com/a.jpg
- [Photo](https://example.com/b.png)
<img src="https://example.com/c.webp">
https://example.com/d.avif`;
expect(extractUrls(input)).toEqual([
"https://example.com/a.jpg",
"https://example.com/b.png",
"https://example.com/c.webp",
"https://example.com/d.avif",
]);
});
it("deduplicates URLs", () => {
const input = "https://example.com/a.jpg\nhttps://example.com/a.jpg";
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg"]);
});
it("filters out non-HTTP URLs", () => {
const input = "ftp://example.com/a.jpg\nhttps://example.com/b.png\nnot-a-url";
expect(extractUrls(input)).toEqual(["https://example.com/b.png"]);
});
it("returns empty array for empty input", () => {
expect(extractUrls("")).toEqual([]);
expect(extractUrls(" \n \n ")).toEqual([]);
});
it("preserves URLs with query parameters", () => {
const input = "https://example.com/photo?id=123&size=large";
expect(extractUrls(input)).toEqual(["https://example.com/photo?id=123&size=large"]);
});
});