mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
feat: add smart URL parser for bulk import
This commit is contained in:
@@ -0,0 +1,43 @@
|
|||||||
|
function isValidHttpUrl(str: string): boolean {
|
||||||
|
try {
|
||||||
|
const url = new URL(str);
|
||||||
|
return url.protocol === "http:" || url.protocol === "https:";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractUrls(input: string): string[] {
|
||||||
|
const urls: string[] = [];
|
||||||
|
|
||||||
|
for (const rawLine of input.split("\n")) {
|
||||||
|
let line = rawLine.trim();
|
||||||
|
if (!line) continue;
|
||||||
|
|
||||||
|
// Strip numbered list prefixes: "1. ", "2) ", "3 "
|
||||||
|
line = line.replace(/^\d+[.)]?\s+/, "");
|
||||||
|
// Strip bullet prefixes: "- ", "* ", "+ "
|
||||||
|
line = line.replace(/^[-*+]\s+/, "");
|
||||||
|
|
||||||
|
// Extract from markdown links: [text](url)
|
||||||
|
const mdMatch = line.match(/\[.*?]\((https?:\/\/[^)]+)\)/);
|
||||||
|
if (mdMatch) {
|
||||||
|
urls.push(mdMatch[1]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract from HTML img tags: <img src="url">
|
||||||
|
const imgMatch = line.match(/<img[^>]+src=["'](https?:\/\/[^"']+)["']/i);
|
||||||
|
if (imgMatch) {
|
||||||
|
urls.push(imgMatch[1]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
line = line.trim();
|
||||||
|
if (isValidHttpUrl(line)) {
|
||||||
|
urls.push(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...new Set(urls)];
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { extractUrls } from "../../../apps/web/src/lib/url-parser.js";
|
||||||
|
|
||||||
|
describe("extractUrls", () => {
|
||||||
|
it("extracts plain URLs one per line", () => {
|
||||||
|
const input = "https://example.com/a.jpg\nhttps://example.com/b.png";
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("strips numbered list prefixes", () => {
|
||||||
|
const input =
|
||||||
|
"1. https://example.com/a.jpg\n2) https://example.com/b.png\n3 https://example.com/c.webp";
|
||||||
|
expect(extractUrls(input)).toEqual([
|
||||||
|
"https://example.com/a.jpg",
|
||||||
|
"https://example.com/b.png",
|
||||||
|
"https://example.com/c.webp",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("strips bullet prefixes", () => {
|
||||||
|
const input =
|
||||||
|
"- https://example.com/a.jpg\n* https://example.com/b.png\n+ https://example.com/c.webp";
|
||||||
|
expect(extractUrls(input)).toEqual([
|
||||||
|
"https://example.com/a.jpg",
|
||||||
|
"https://example.com/b.png",
|
||||||
|
"https://example.com/c.webp",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts URLs from markdown links", () => {
|
||||||
|
const input = "[Photo 1](https://example.com/a.jpg)\n[Photo 2](https://example.com/b.png)";
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts URLs from HTML img tags", () => {
|
||||||
|
const input = '<img src="https://example.com/a.jpg">\n<img src="https://example.com/b.png" />';
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg", "https://example.com/b.png"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles mixed formats", () => {
|
||||||
|
const input = `1. https://example.com/a.jpg
|
||||||
|
- [Photo](https://example.com/b.png)
|
||||||
|
<img src="https://example.com/c.webp">
|
||||||
|
https://example.com/d.avif`;
|
||||||
|
expect(extractUrls(input)).toEqual([
|
||||||
|
"https://example.com/a.jpg",
|
||||||
|
"https://example.com/b.png",
|
||||||
|
"https://example.com/c.webp",
|
||||||
|
"https://example.com/d.avif",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deduplicates URLs", () => {
|
||||||
|
const input = "https://example.com/a.jpg\nhttps://example.com/a.jpg";
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/a.jpg"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("filters out non-HTTP URLs", () => {
|
||||||
|
const input = "ftp://example.com/a.jpg\nhttps://example.com/b.png\nnot-a-url";
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/b.png"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty array for empty input", () => {
|
||||||
|
expect(extractUrls("")).toEqual([]);
|
||||||
|
expect(extractUrls(" \n \n ")).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves URLs with query parameters", () => {
|
||||||
|
const input = "https://example.com/photo?id=123&size=large";
|
||||||
|
expect(extractUrls(input)).toEqual(["https://example.com/photo?id=123&size=large"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user