mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
feat: add smart URL parser for bulk import
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
function isValidHttpUrl(str: string): boolean {
|
||||
try {
|
||||
const url = new URL(str);
|
||||
return url.protocol === "http:" || url.protocol === "https:";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractUrls(input: string): string[] {
|
||||
const urls: string[] = [];
|
||||
|
||||
for (const rawLine of input.split("\n")) {
|
||||
let line = rawLine.trim();
|
||||
if (!line) continue;
|
||||
|
||||
// Strip numbered list prefixes: "1. ", "2) ", "3 "
|
||||
line = line.replace(/^\d+[.)]?\s+/, "");
|
||||
// Strip bullet prefixes: "- ", "* ", "+ "
|
||||
line = line.replace(/^[-*+]\s+/, "");
|
||||
|
||||
// Extract from markdown links: [text](url)
|
||||
const mdMatch = line.match(/\[.*?]\((https?:\/\/[^)]+)\)/);
|
||||
if (mdMatch) {
|
||||
urls.push(mdMatch[1]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract from HTML img tags: <img src="url">
|
||||
const imgMatch = line.match(/<img[^>]+src=["'](https?:\/\/[^"']+)["']/i);
|
||||
if (imgMatch) {
|
||||
urls.push(imgMatch[1]);
|
||||
continue;
|
||||
}
|
||||
|
||||
line = line.trim();
|
||||
if (isValidHttpUrl(line)) {
|
||||
urls.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
return [...new Set(urls)];
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractUrls } from "../../../apps/web/src/lib/url-parser.js";
|
||||
|
||||
describe("extractUrls", () => {
|
||||
it("extracts plain URLs one per line", () => {
|
||||
const input = "https://example.com/a.jpg\nhttps://example.com/b.png";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("strips numbered list prefixes", () => {
|
||||
const input =
|
||||
"1. https://example.com/a.jpg\n2) https://example.com/b.png\n3 https://example.com/c.webp";
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
]);
|
||||
});
|
||||
|
||||
it("strips bullet prefixes", () => {
|
||||
const input =
|
||||
"- https://example.com/a.jpg\n* https://example.com/b.png\n+ https://example.com/c.webp";
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts URLs from markdown links", () => {
|
||||
const input = "[Photo 1](https://example.com/a.jpg)\n[Photo 2](https://example.com/b.png)";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("extracts URLs from HTML img tags", () => {
|
||||
const input = '<img src="https://example.com/a.jpg">\n<img src="https://example.com/b.png" />';
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("handles mixed formats", () => {
|
||||
const input = `1. https://example.com/a.jpg
|
||||
- [Photo](https://example.com/b.png)
|
||||
<img src="https://example.com/c.webp">
|
||||
https://example.com/d.avif`;
|
||||
expect(extractUrls(input)).toEqual([
|
||||
"https://example.com/a.jpg",
|
||||
"https://example.com/b.png",
|
||||
"https://example.com/c.webp",
|
||||
"https://example.com/d.avif",
|
||||
]);
|
||||
});
|
||||
|
||||
it("deduplicates URLs", () => {
|
||||
const input = "https://example.com/a.jpg\nhttps://example.com/a.jpg";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg"]);
|
||||
});
|
||||
|
||||
it("filters out non-HTTP URLs", () => {
|
||||
const input = "ftp://example.com/a.jpg\nhttps://example.com/b.png\nnot-a-url";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/b.png"]);
|
||||
});
|
||||
|
||||
it("returns empty array for empty input", () => {
|
||||
expect(extractUrls("")).toEqual([]);
|
||||
expect(extractUrls(" \n \n ")).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves URLs with query parameters", () => {
|
||||
const input = "https://example.com/photo?id=123&size=large";
|
||||
expect(extractUrls(input)).toEqual(["https://example.com/photo?id=123&size=large"]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user