feat: initial project setup with CLI, utils, and tests

tests/media.test.ts

@@ -0,0 +1,37 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { buildMediaUrlRegex, DEFAULT_MEDIA_EXTENSIONS } from "../src/utils/media.js";
+
+test("default regex matches typical WordPress upload URLs", () => {
+  const re = buildMediaUrlRegex(DEFAULT_MEDIA_EXTENSIONS);
+  const text =
+    "see <img src=\"https://site.test/wp-content/uploads/2020/01/photo.jpg\"/> and " +
+    "https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2 and ignore https://site.test/page";
+  const matches = text.match(re) ?? [];
+  assert.deepEqual(matches, [
+    "https://site.test/wp-content/uploads/2020/01/photo.jpg",
+    "https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2",
+  ]);
+});
+
+test("regex is case-insensitive on extension", () => {
+  const re = buildMediaUrlRegex(["jpg"]);
+  assert.ok(re.test("https://x.test/a.JPG"));
+});
+
+test("regex respects custom extensions only", () => {
+  const re = buildMediaUrlRegex(["xyz"]);
+  assert.equal("https://x.test/a.jpg".match(re), null);
+  assert.ok(re.test("https://x.test/a.xyz"));
+});
+
+test("buildMediaUrlRegex throws on empty list", () => {
+  assert.throws(() => buildMediaUrlRegex([]), /at least one extension/);
+});
+
+test("regex stops at quotes and whitespace", () => {
+  const re = buildMediaUrlRegex(["png"]);
+  const text = `'https://x.test/a.png' "https://x.test/b.png"`;
+  const matches = text.match(re) ?? [];
+  assert.deepEqual(matches, ["https://x.test/a.png", "https://x.test/b.png"]);
+});

tests/progress.test.ts

@@ -0,0 +1,62 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { ProgressBar } from "../src/utils/progress.js";
+
+test("format reflects ticks", () => {
+  const bar = new ProgressBar({ total: 4, width: 10, enabled: false });
+  bar.tick("ok");
+  bar.tick("skipped");
+  const line = bar.format();
+  assert.match(line, /50%/);
+  assert.match(line, /2\/4/);
+  assert.match(line, /ok:1/);
+  assert.match(line, /skip:1/);
+  assert.match(line, /fail:0/);
+});
+
+test("snapshot returns running totals", () => {
+  const bar = new ProgressBar({ total: 3, enabled: false });
+  bar.tick("ok");
+  bar.tick("failed");
+  assert.deepEqual(bar.snapshot(), { ok: 1, skipped: 0, failed: 1, done: 2 });
+});
+
+test("format reaches 100% on completion", () => {
+  const bar = new ProgressBar({ total: 2, width: 4, enabled: false });
+  bar.tick("ok");
+  bar.tick("ok");
+  assert.match(bar.format(), /100%/);
+});
+
+test("disabled bar does not write to stream", () => {
+  let written = "";
+  const fakeStream = {
+    isTTY: false,
+    write: (s: string) => {
+      written += s;
+      return true;
+    },
+  } as unknown as NodeJS.WriteStream;
+  const bar = new ProgressBar({ total: 1, stream: fakeStream });
+  bar.tick("ok");
+  bar.finish();
+  assert.equal(written, "");
+});
+
+test("enabled bar writes carriage-return updates", () => {
+  let written = "";
+  const fakeStream = {
+    isTTY: true,
+    write: (s: string) => {
+      written += s;
+      return true;
+    },
+  } as unknown as NodeJS.WriteStream;
+  const bar = new ProgressBar({ total: 2, width: 4, stream: fakeStream });
+  bar.tick("ok");
+  bar.tick("ok");
+  bar.finish();
+  assert.ok(written.startsWith("\r"));
+  assert.ok(written.endsWith("\n"));
+  assert.match(written, /100%/);
+});

tests/slice.test.ts

@@ -0,0 +1,135 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+  filterUrlsByExtensions,
+  parseExtensionsOption,
+  parseIntOption,
+  sliceUrls,
+} from "../src/utils/slice.js";
+
+const sample = [
+  "https://x.test/c.jpg",
+  "https://x.test/a.jpg",
+  "https://x.test/b.jpg",
+  "https://x.test/e.jpg",
+  "https://x.test/d.jpg",
+];
+
+test("sliceUrls sorts deterministically with no slice options", () => {
+  assert.deepEqual(sliceUrls(sample), [
+    "https://x.test/a.jpg",
+    "https://x.test/b.jpg",
+    "https://x.test/c.jpg",
+    "https://x.test/d.jpg",
+    "https://x.test/e.jpg",
+  ]);
+});
+
+test("sliceUrls applies offset only", () => {
+  assert.deepEqual(sliceUrls(sample, { offset: 2 }), [
+    "https://x.test/c.jpg",
+    "https://x.test/d.jpg",
+    "https://x.test/e.jpg",
+  ]);
+});
+
+test("sliceUrls applies limit only", () => {
+  assert.deepEqual(sliceUrls(sample, { limit: 2 }), [
+    "https://x.test/a.jpg",
+    "https://x.test/b.jpg",
+  ]);
+});
+
+test("sliceUrls applies offset and limit together", () => {
+  assert.deepEqual(sliceUrls(sample, { offset: 1, limit: 2 }), [
+    "https://x.test/b.jpg",
+    "https://x.test/c.jpg",
+  ]);
+});
+
+test("sliceUrls returns [] when offset exceeds length", () => {
+  assert.deepEqual(sliceUrls(sample, { offset: 99 }), []);
+});
+
+test("sliceUrls works with a Set", () => {
+  const set = new Set(sample);
+  assert.deepEqual(sliceUrls(set, { limit: 1 }), ["https://x.test/a.jpg"]);
+});
+
+test("sliceUrls clamps negative offset to 0", () => {
+  assert.deepEqual(sliceUrls(sample, { offset: -5, limit: 1 }), [
+    "https://x.test/a.jpg",
+  ]);
+});
+
+test("sliceUrls treats limit=0 as empty", () => {
+  assert.deepEqual(sliceUrls(sample, { limit: 0 }), []);
+});
+
+test("parseIntOption returns undefined when missing", () => {
+  assert.equal(parseIntOption(undefined, "--limit"), undefined);
+  assert.equal(parseIntOption("", "--limit"), undefined);
+});
+
+test("parseIntOption parses integers", () => {
+  assert.equal(parseIntOption("0", "--offset"), 0);
+  assert.equal(parseIntOption("42", "--limit"), 42);
+});
+
+test("parseIntOption rejects non-integers", () => {
+  assert.throws(() => parseIntOption("3.14", "--limit"), /must be an integer/);
+  assert.throws(() => parseIntOption("abc", "--offset"), /must be an integer/);
+});
+
+test("parseIntOption rejects negatives", () => {
+  assert.throws(() => parseIntOption("-1", "--limit"), /must be >= 0/);
+});
+
+test("parseExtensionsOption returns undefined for missing/blank", () => {
+  assert.equal(parseExtensionsOption(undefined), undefined);
+  assert.equal(parseExtensionsOption(""), undefined);
+  assert.equal(parseExtensionsOption(" , , "), undefined);
+});
+
+test("parseExtensionsOption normalizes case, dots, and whitespace", () => {
+  const set = parseExtensionsOption(" .JPG, png ,.WebP");
+  assert.deepEqual([...(set ?? [])].sort(), ["jpg", "png", "webp"]);
+});
+
+test("filterUrlsByExtensions returns input unchanged when no filter", () => {
+  const urls = ["https://x.test/a.jpg", "https://x.test/b.png"];
+  assert.deepEqual(filterUrlsByExtensions(urls, undefined), urls);
+});
+
+test("filterUrlsByExtensions keeps only matching extensions", () => {
+  const urls = [
+    "https://x.test/a.jpg",
+    "https://x.test/b.png",
+    "https://x.test/c.JPG",
+    "https://x.test/d.gif",
+  ];
+  const set = parseExtensionsOption("jpg");
+  assert.deepEqual(filterUrlsByExtensions(urls, set), [
+    "https://x.test/a.jpg",
+    "https://x.test/c.JPG",
+  ]);
+});
+
+test("filterUrlsByExtensions ignores query string and fragment", () => {
+  const urls = [
+    "https://x.test/a.jpg?v=2",
+    "https://x.test/b.png#frag",
+    "https://x.test/c.gif?x=1#y",
+  ];
+  const set = parseExtensionsOption("jpg,png");
+  assert.deepEqual(filterUrlsByExtensions(urls, set), [
+    "https://x.test/a.jpg?v=2",
+    "https://x.test/b.png#frag",
+  ]);
+});
+
+test("filterUrlsByExtensions drops URLs without an extension", () => {
+  const urls = ["https://x.test/no-ext", "https://x.test/a.jpg"];
+  const set = parseExtensionsOption("jpg");
+  assert.deepEqual(filterUrlsByExtensions(urls, set), ["https://x.test/a.jpg"]);
+});

tests/snapshot.test.ts

@@ -0,0 +1,59 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+  buildSnapshotUrl,
+  extractTimestamp,
+  normalizeTimestamp,
+} from "../src/utils/snapshot.js";
+
+test("buildSnapshotUrl uses '2' as default timestamp prefix", () => {
+  assert.equal(
+    buildSnapshotUrl("https://example.com/a.jpg"),
+    "https://web.archive.org/web/2id_/https://example.com/a.jpg",
+  );
+});
+
+test("buildSnapshotUrl honors a full timestamp", () => {
+  assert.equal(
+    buildSnapshotUrl("https://example.com/a.jpg", "20200101000000"),
+    "https://web.archive.org/web/20200101000000id_/https://example.com/a.jpg",
+  );
+});
+
+test("buildSnapshotUrl honors a partial timestamp", () => {
+  assert.equal(
+    buildSnapshotUrl("https://example.com/a.jpg", "2020"),
+    "https://web.archive.org/web/2020id_/https://example.com/a.jpg",
+  );
+});
+
+test("normalizeTimestamp strips non-digits", () => {
+  assert.equal(normalizeTimestamp("2020-01-01"), "20200101");
+  assert.equal(normalizeTimestamp("2020/01"), "202001");
+});
+
+test("normalizeTimestamp falls back to '2' on empty input", () => {
+  assert.equal(normalizeTimestamp(undefined), "2");
+  assert.equal(normalizeTimestamp(""), "2");
+  assert.equal(normalizeTimestamp("---"), "2");
+});
+
+test("extractTimestamp pulls timestamp from a resolved snapshot URL", () => {
+  assert.equal(
+    extractTimestamp(
+      "https://web.archive.org/web/20221020185708id_/https://x.test/a.jpg",
+    ),
+    "20221020185708",
+  );
+});
+
+test("extractTimestamp works without id_ marker", () => {
+  assert.equal(
+    extractTimestamp("https://web.archive.org/web/20221020185708/https://x.test/a.jpg"),
+    "20221020185708",
+  );
+});
+
+test("extractTimestamp returns undefined for non-wayback URLs", () => {
+  assert.equal(extractTimestamp("https://example.com/a.jpg"), undefined);
+});

tests/sql.test.ts

@@ -0,0 +1,47 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { extractAttachmentGuids, splitTuples } from "../src/utils/sql.js";
+
+test("splitTuples splits flat tuples", () => {
+  const line = "INSERT INTO `t` VALUES (1,'a'),(2,'b'),(3,'c');";
+  assert.deepEqual(splitTuples(line), ["1,'a'", "2,'b'", "3,'c'"]);
+});
+
+test("splitTuples ignores parens inside quoted strings", () => {
+  const line = "INSERT INTO `t` VALUES (1,'foo (bar) baz'),(2,'x');";
+  assert.deepEqual(splitTuples(line), ["1,'foo (bar) baz'", "2,'x'"]);
+});
+
+test("splitTuples handles escaped quotes", () => {
+  const line = "INSERT INTO `t` VALUES (1,'it\\'s ok'),(2,'y');";
+  const tuples = splitTuples(line);
+  assert.equal(tuples.length, 2);
+  assert.equal(tuples[0], "1,'it\\'s ok'");
+});
+
+test("extractAttachmentGuids returns URLs from attachment rows", () => {
+  const line =
+    "INSERT INTO `wp_posts` VALUES " +
+    "(1,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/wp-content/uploads/2020/01/foo.jpg',0,'attachment','image/jpeg',0)," +
+    "(2,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);";
+  assert.deepEqual(extractAttachmentGuids(line), [
+    "https://site.test/wp-content/uploads/2020/01/foo.jpg",
+  ]);
+});
+
+test("extractAttachmentGuids returns [] when no attachment rows", () => {
+  const line =
+    "INSERT INTO `wp_posts` VALUES (2,1,'2020-01-01','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);";
+  assert.deepEqual(extractAttachmentGuids(line), []);
+});
+
+test("extractAttachmentGuids handles multiple attachment rows", () => {
+  const line =
+    "INSERT INTO `wp_posts` VALUES " +
+    "(1,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/a.jpg',0,'attachment','image/jpeg',0)," +
+    "(2,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/b.png',0,'attachment','image/png',0);";
+  assert.deepEqual(extractAttachmentGuids(line), [
+    "https://site.test/a.jpg",
+    "https://site.test/b.png",
+  ]);
+});

tests/summary.test.ts

@@ -0,0 +1,90 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { buildSummary, formatBytes, formatDuration } from "../src/utils/summary.js";
+
+test("formatBytes scales units", () => {
+  assert.equal(formatBytes(0), "0 B");
+  assert.equal(formatBytes(512), "512 B");
+  assert.equal(formatBytes(2048), "2.00 KB");
+  assert.equal(formatBytes(5 * 1024 * 1024), "5.00 MB");
+  assert.equal(formatBytes(3 * 1024 ** 3), "3.00 GB");
+});
+
+test("formatDuration formats ranges", () => {
+  assert.equal(formatDuration(250), "250ms");
+  assert.equal(formatDuration(1500), "2s");
+  assert.equal(formatDuration(65_000), "1m5s");
+  assert.equal(formatDuration(3_725_000), "1h2m5s");
+});
+
+test("buildSummary includes counters and rate", () => {
+  const out = buildSummary({
+    totalFound: 100,
+    attachmentGuids: 40,
+    ok: 80,
+    skipped: 10,
+    failed: 10,
+    bytes: 1024,
+    elapsedMs: 5000,
+    outputDir: "/tmp/out",
+  });
+  assert.match(out, /URLs found in dump : 100 \(40 attachment GUIDs\)/);
+  assert.match(out, /✓ Downloaded\s*: 80/);
+  assert.match(out, /↷ Skipped\s*\(exists\)\s*: 10/);
+  assert.match(out, /✗ Failed\s*: 10/);
+  assert.match(out, /Success rate\s*: 80%/);
+  assert.match(out, /1\.00 KB/);
+  assert.match(out, /5s/);
+  assert.match(out, /\/tmp\/out/);
+});
+
+test("buildSummary lists failures with truncation", () => {
+  const failures = Array.from({ length: 12 }, (_, i) => ({
+    url: `https://x.test/f${i}.jpg`,
+    reason: "no_snapshot",
+  }));
+  const out = buildSummary({
+    totalFound: 12,
+    attachmentGuids: 0,
+    ok: 0,
+    skipped: 0,
+    failed: 12,
+    bytes: 0,
+    elapsedMs: 100,
+    outputDir: "/o",
+    failures,
+    maxFailuresShown: 5,
+  });
+  assert.match(out, /f0\.jpg/);
+  assert.match(out, /f4\.jpg/);
+  assert.ok(!out.includes("f5.jpg"));
+  assert.match(out, /and 7 more/);
+});
+
+test("buildSummary omits manifest line when not provided", () => {
+  const out = buildSummary({
+    totalFound: 1,
+    attachmentGuids: 0,
+    ok: 1,
+    skipped: 0,
+    failed: 0,
+    bytes: 100,
+    elapsedMs: 10,
+    outputDir: "/o",
+  });
+  assert.ok(!out.includes("Manifest"));
+});
+
+test("buildSummary handles 0 processed", () => {
+  const out = buildSummary({
+    totalFound: 0,
+    attachmentGuids: 0,
+    ok: 0,
+    skipped: 0,
+    failed: 0,
+    bytes: 0,
+    elapsedMs: 5,
+    outputDir: "/o",
+  });
+  assert.match(out, /Success rate\s*: n\/a/);
+});

tests/url.test.ts

@@ -0,0 +1,40 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { cleanUrl, matchesHost, normalizeHost } from "../src/utils/url.js";
+
+test("cleanUrl strips trailing punctuation", () => {
+  assert.equal(cleanUrl("https://x.com/a.jpg)."), "https://x.com/a.jpg");
+  assert.equal(cleanUrl("https://x.com/a.jpg',"), "https://x.com/a.jpg");
+  assert.equal(cleanUrl("https://x.com/a.jpg"), "https://x.com/a.jpg");
+});
+
+test("cleanUrl preserves query strings", () => {
+  assert.equal(
+    cleanUrl("https://x.com/a.jpg?v=1"),
+    "https://x.com/a.jpg?v=1",
+  );
+});
+
+test("normalizeHost accepts bare hosts and full URLs", () => {
+  assert.equal(normalizeHost("Example.COM"), "example.com");
+  assert.equal(normalizeHost("https://Example.com/path"), "example.com");
+  assert.equal(normalizeHost("http://example.com:8080/"), "example.com:8080");
+});
+
+test("normalizeHost returns null for empty input", () => {
+  assert.equal(normalizeHost(undefined), null);
+  assert.equal(normalizeHost(""), null);
+});
+
+test("matchesHost null filter accepts everything", () => {
+  assert.equal(matchesHost("https://anywhere.test/x.jpg", null), true);
+});
+
+test("matchesHost compares case-insensitively", () => {
+  assert.equal(matchesHost("https://Example.com/x.jpg", "example.com"), true);
+  assert.equal(matchesHost("https://other.com/x.jpg", "example.com"), false);
+});
+
+test("matchesHost rejects malformed URLs", () => {
+  assert.equal(matchesHost("not a url", "example.com"), false);
+});

tests/wayback-url.test.ts

@@ -0,0 +1,43 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { urlToOutputPath } from "../src/utils/wayback-url.js";
+
+test("urlToOutputPath preserves host and path layout", () => {
+  assert.equal(
+    urlToOutputPath(
+      "https://example.com/wp-content/uploads/2020/01/photo.jpg",
+      "/out",
+    ),
+    "/out/example.com/wp-content/uploads/2020/01/photo.jpg",
+  );
+});
+
+test("urlToOutputPath decodes percent-encoded segments", () => {
+  assert.equal(
+    urlToOutputPath("https://example.com/uploads/foto%20test.jpg", "/out"),
+    "/out/example.com/uploads/foto test.jpg",
+  );
+});
+
+test("urlToOutputPath sanitizes path-traversal segments", () => {
+  const out = urlToOutputPath("https://example.com/../etc/passwd", "/out");
+  assert.ok(!out.includes(".."));
+});
+
+test("urlToOutputPath omits host directory when hostPrefix=false", () => {
+  assert.equal(
+    urlToOutputPath(
+      "https://example.com/wp-content/uploads/2020/01/photo.jpg",
+      "/out",
+      { hostPrefix: false },
+    ),
+    "/out/wp-content/uploads/2020/01/photo.jpg",
+  );
+});
+
+test("urlToOutputPath falls back to 'index' for empty paths", () => {
+  assert.equal(
+    urlToOutputPath("https://example.com/", "/out"),
+    "/out/example.com/index",
+  );
+});