commit ed8df811f702b155f637091337b9b3c0e1b313e6 Author: Andrea Debernardi Date: Sat May 9 21:59:41 2026 +0200 feat: initial project setup with CLI, utils, and tests diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..33ccb67 --- /dev/null +++ b/.gitignore @@ -0,0 +1,55 @@ +# Recovered media output +recovered/ + +# Node +node_modules/ +dist/ +build/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +.pnpm-store/ + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ +env/ + +# Data / dumps +*.jsonl +*.sql +*.sqlite +*.sqlite3 +*.db + +# Env / secrets +.env +.env.* +!.env.example + +# Logs +*.log +logs/ + +# Testing / cache +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +coverage/ +.cache/ + +# IDE / editor +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6d9840e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Andrea Debernardi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd24a85 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# wp-media-rewind + +A small CLI for the situation nobody wants to be in: you have a WordPress SQL +dump, the original site is gone, and the `wp-content/uploads` folder vanished +with it. `wp-media-rewind` reads the dump, pulls every media URL it can find, and +tries to fetch each file back from the Wayback Machine — keeping the original +folder structure on disk. + +It's deliberately not clever. No database, no headless browser, no scraping +the live site. Just: parse the dump, ask `archive.org/wayback/available` for +the closest snapshot, download the raw bytes. + +## What it does + +- Streams a `.sql` dump line by line, so 2 GB exports don't blow up memory. +- Pulls media URLs from two places: + - any URL with a media-looking extension found anywhere in the file + (covers stuff embedded in `post_content`, options, postmeta, etc.), + - `guid` values from `wp_posts` rows where `post_type='attachment'`. +- For each URL, asks the Wayback availability API for the closest snapshot + (optionally near a timestamp you pick), then downloads the raw asset using + the `id_` flag so you get original bytes, not a rewritten archive page. +- Writes files under `//` by default, so a URL + like `https://example.com/wp-content/uploads/2020/01/photo.jpg` lands in + `recovered/example.com/wp-content/uploads/2020/01/photo.jpg`. +- Skips files that already exist on disk, so re-runs are cheap. +- Shows a progress bar while it works and prints a summary at the end with + counts, total bytes, elapsed time, and the first failures. + +## Install + +```bash +npm install +npm run build +``` + +## Use + +```bash +# Positional argument +node dist/index.js dump.sql --site example.com --output ./recovered + +# Or with the explicit flag +node dist/index.js --sql ./dump.sql --site example.com --output ./recovered + +# See what would be fetched without downloading anything +node dist/index.js dump.sql --site example.com --dry-run +``` + +### Options + +| Flag | Default | What it does | +| --- | --- | --- | +| `` / `-f, --sql ` | — | Path to the WordPress SQL dump. | +| `-o, --output ` | `./recovered` | Where to write recovered files. | +| `-s, --site ` | — | Only keep URLs on this host (recommended; dumps often quote URLs from other sites in `post_content`). | +| `-t, --timestamp ` | — | Prefer snapshots near this date. Useful if you remember roughly when the site was last alive. | +| `-c, --concurrency ` | `4` | Parallel downloads. Be polite to archive.org. | +| `--dry-run` | off | Print URLs and exit. | +| `--manifest ` | — | Append a JSONL row per URL with status, snapshot, bytes, etc. | +| `--no-host-prefix` | off | Drop the `/` directory and write paths starting at the site root. | + +### Final report + +When the run finishes you get something like: + +``` +─── Recovery summary ───────────────────────── + URLs found in dump : 312 (118 attachment GUIDs) + Processed : 312 + ✓ Downloaded : 287 + ↷ Skipped (exists) : 0 + ✗ Failed : 25 + Success rate : 91% + Total downloaded : 184.21 MB + Elapsed : 4m12s + Output directory : ./recovered + Manifest (JSONL) : ./run.jsonl + + Failures: + - https://example.com/wp-content/uploads/2014/03/old.jpg (no_snapshot) + … +────────────────────────────────────────────── +``` + +The exit code is non-zero if any URL failed, so you can wire it into a +script. + +## Notes & caveats + +- The Wayback Machine doesn't have everything. Older or low-traffic files are + the most common misses. Try a different `--timestamp` if a whole period + looks empty. +- The tool requests one availability lookup per URL. If you're recovering + thousands of files, run with low concurrency and expect it to take a while. +- It only restores binaries (images, PDFs, video, etc.). Posts, comments and + other DB content are already in the dump — you don't need this for those. +- Filenames with characters that are illegal on your filesystem get replaced + with `_`. Path traversal segments are stripped. + +## Development + +```bash +npm test # run unit tests (node:test, no extra deps) +npm run typecheck # strict tsc --noEmit on src + tests +npm run dev # tsx src/index.ts +``` + +The pure helpers live in `src/utils/` and have unit tests in `tests/`: + +- `utils/url.ts` — host normalization and URL cleanup. +- `utils/sql.ts` — splitting `INSERT VALUES (...)` tuples and pulling + attachment GUIDs. +- `utils/media.ts` — building the media-extension regex. +- `utils/wayback-url.ts` — converting snapshot URLs to the raw `id_` form + and mapping URLs to output paths. +- `utils/progress.ts` — terminal progress bar. +- `utils/summary.ts` — final report formatting. + +The HTTP code (`wayback.ts`) and the SQL streaming reader (`parser.ts`) are +intentionally thin wrappers around those utilities. + +## License + +MIT. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..91dc339 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,636 @@ +{ + "name": "waybackmachine-media-recovery", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "waybackmachine-media-recovery", + "version": "0.1.0", + "dependencies": { + "commander": "^12.1.0", + "p-limit": "^6.1.0" + }, + "bin": { + "wbm-recover": "dist/index.js" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.19.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.18.tgz", + "integrity": "sha512-9v00a+dn2yWVsYDEunWC4g/TcRKVq3r8N5FuZp7u0SGrPvdN9c2yXI9bBuf5Fl0hNCb+QTIePTn5pJs2pwBOQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/p-limit": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz", + "integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..581b6d4 --- /dev/null +++ b/package.json @@ -0,0 +1,29 @@ +{ + "name": "wp-media-rewind", + "version": "0.1.0", + "description": "Pull WordPress media files out of a SQL dump using the Wayback Machine.", + "license": "MIT", + "type": "module", + "bin": { + "wp-media-rewind": "dist/index.js" + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "start": "npm run build && node dist/index.js", + "dev": "tsx src/index.ts", + "test": "node --import tsx --test tests/*.test.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "commander": "^12.1.0", + "p-limit": "^6.1.0" + }, + "devDependencies": { + "@types/node": "^22.7.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..1e5d997 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,369 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + commander: + specifier: ^12.1.0 + version: 12.1.0 + p-limit: + specifier: ^6.1.0 + version: 6.2.0 + devDependencies: + '@types/node': + specifier: ^22.7.0 + version: 22.19.18 + tsx: + specifier: ^4.19.0 + version: 4.21.0 + typescript: + specifier: ^5.6.0 + version: 5.9.3 + +packages: + + '@esbuild/aix-ppc64@0.27.7': + resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.7': + resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.7': + resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.7': + resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.7': + resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.7': + resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.7': + resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.7': + resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.7': + resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.7': + resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.7': + resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.7': + resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.7': + resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.7': + resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.7': + resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.7': + resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.7': + resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.7': + resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.7': + resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.7': + resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.7': + resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.7': + resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.7': + resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.7': + resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.7': + resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.7': + resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@types/node@22.19.18': + resolution: {integrity: sha512-9v00a+dn2yWVsYDEunWC4g/TcRKVq3r8N5FuZp7u0SGrPvdN9c2yXI9bBuf5Fl0hNCb+QTIePTn5pJs2pwBOQQ==} + + commander@12.1.0: + resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} + engines: {node: '>=18'} + + esbuild@0.27.7: + resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-tsconfig@4.14.0: + resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + +snapshots: + + '@esbuild/aix-ppc64@0.27.7': + optional: true + + '@esbuild/android-arm64@0.27.7': + optional: true + + '@esbuild/android-arm@0.27.7': + optional: true + + '@esbuild/android-x64@0.27.7': + optional: true + + '@esbuild/darwin-arm64@0.27.7': + optional: true + + '@esbuild/darwin-x64@0.27.7': + optional: true + + '@esbuild/freebsd-arm64@0.27.7': + optional: true + + '@esbuild/freebsd-x64@0.27.7': + optional: true + + '@esbuild/linux-arm64@0.27.7': + optional: true + + '@esbuild/linux-arm@0.27.7': + optional: true + + '@esbuild/linux-ia32@0.27.7': + optional: true + + '@esbuild/linux-loong64@0.27.7': + optional: true + + '@esbuild/linux-mips64el@0.27.7': + optional: true + + '@esbuild/linux-ppc64@0.27.7': + optional: true + + '@esbuild/linux-riscv64@0.27.7': + optional: true + + '@esbuild/linux-s390x@0.27.7': + optional: true + + '@esbuild/linux-x64@0.27.7': + optional: true + + '@esbuild/netbsd-arm64@0.27.7': + optional: true + + '@esbuild/netbsd-x64@0.27.7': + optional: true + + '@esbuild/openbsd-arm64@0.27.7': + optional: true + + '@esbuild/openbsd-x64@0.27.7': + optional: true + + '@esbuild/openharmony-arm64@0.27.7': + optional: true + + '@esbuild/sunos-x64@0.27.7': + optional: true + + '@esbuild/win32-arm64@0.27.7': + optional: true + + '@esbuild/win32-ia32@0.27.7': + optional: true + + '@esbuild/win32-x64@0.27.7': + optional: true + + '@types/node@22.19.18': + dependencies: + undici-types: 6.21.0 + + commander@12.1.0: {} + + esbuild@0.27.7: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.7 + '@esbuild/android-arm': 0.27.7 + '@esbuild/android-arm64': 0.27.7 + '@esbuild/android-x64': 0.27.7 + '@esbuild/darwin-arm64': 0.27.7 + '@esbuild/darwin-x64': 0.27.7 + '@esbuild/freebsd-arm64': 0.27.7 + '@esbuild/freebsd-x64': 0.27.7 + '@esbuild/linux-arm': 0.27.7 + '@esbuild/linux-arm64': 0.27.7 + '@esbuild/linux-ia32': 0.27.7 + '@esbuild/linux-loong64': 0.27.7 + '@esbuild/linux-mips64el': 0.27.7 + '@esbuild/linux-ppc64': 0.27.7 + '@esbuild/linux-riscv64': 0.27.7 + '@esbuild/linux-s390x': 0.27.7 + '@esbuild/linux-x64': 0.27.7 + '@esbuild/netbsd-arm64': 0.27.7 + '@esbuild/netbsd-x64': 0.27.7 + '@esbuild/openbsd-arm64': 0.27.7 + '@esbuild/openbsd-x64': 0.27.7 + '@esbuild/openharmony-arm64': 0.27.7 + '@esbuild/sunos-x64': 0.27.7 + '@esbuild/win32-arm64': 0.27.7 + '@esbuild/win32-ia32': 0.27.7 + '@esbuild/win32-x64': 0.27.7 + + fsevents@2.3.3: + optional: true + + get-tsconfig@4.14.0: + dependencies: + resolve-pkg-maps: 1.0.0 + + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.2 + + resolve-pkg-maps@1.0.0: {} + + tsx@4.21.0: + dependencies: + esbuild: 0.27.7 + get-tsconfig: 4.14.0 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + yocto-queue@1.2.2: {} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..b90dd6e --- /dev/null +++ b/src/index.ts @@ -0,0 +1,168 @@ +#!/usr/bin/env node +import { Command } from "commander"; +import pLimit from "p-limit"; +import { appendFile, writeFile } from "node:fs/promises"; +import { parseSqlDump } from "./parser.js"; +import { fileExists, recover } from "./wayback.js"; +import { urlToOutputPath } from "./utils/wayback-url.js"; +import { ProgressBar } from "./utils/progress.js"; +import { buildSummary, type FailureEntry } from "./utils/summary.js"; +import { + filterUrlsByExtensions, + parseExtensionsOption, + parseIntOption, + sliceUrls, +} from "./utils/slice.js"; + +interface CliOpts { + sql?: string; + output: string; + site?: string; + timestamp?: string; + concurrency: string; + dryRun?: boolean; + manifest?: string; + hostPrefix: boolean; + limit?: string; + offset?: string; + filter?: string; +} + +const program = new Command() + .name("wp-media-rewind") + .description("Recover WordPress media from a SQL dump via the Wayback Machine.") + .argument("[sql-dump]", "Path to the WordPress .sql dump (or use --sql)") + .option("-f, --sql ", "Path to the WordPress .sql dump") + .option("-o, --output ", "Output directory", "./recovered") + .option("-s, --site ", "Restrict to this site host (e.g. example.com)") + .option("-t, --timestamp ", "Preferred snapshot timestamp") + .option("-c, --concurrency ", "Parallel downloads", "4") + .option("--dry-run", "List URLs found and exit") + .option("--manifest ", "Write a JSONL manifest of results") + .option("--no-host-prefix", "Do not prefix output paths with the URL host") + .option("--limit ", "Process at most N URLs (after sorting and offset)") + .option("--offset ", "Skip the first N URLs (after sorting)") + .option( + "--filter ", + "Comma-separated extensions to keep (e.g. jpg,png,webp)", + ) + .addHelpText( + "after", + `\nExamples:\n $ wp-media-rewind dump.sql -s example.com -o ./recovered\n $ wp-media-rewind --sql ./dump.sql --site example.com --dry-run\n $ wp-media-rewind dump.sql --filter jpg,png\n`, + ) + .parse(process.argv); + +const opts = program.opts(); +const sqlPath = opts.sql ?? program.args[0]; + +if (!sqlPath) { + console.error("error: missing SQL dump path. Pass it as positional arg or via --sql .\n"); + program.help({ error: true }); +} + +await main(sqlPath); + +async function main(sqlPath: string) { + const startedAt = Date.now(); + console.error(`Parsing ${sqlPath}…`); + const { urls, attachmentGuids } = await parseSqlDump(sqlPath, { + site: opts.site, + }); + console.error( + `Found ${urls.size} media URLs (${attachmentGuids.size} from wp_posts attachments).`, + ); + + const offset = parseIntOption(opts.offset, "--offset"); + const limitN = parseIntOption(opts.limit, "--limit"); + const extensions = parseExtensionsOption(opts.filter); + const filtered = filterUrlsByExtensions(urls, extensions); + if (extensions) { + console.error( + `Filtering by extensions [${[...extensions].join(", ")}]: ${urls.size} → ${filtered.length} URLs`, + ); + } + const work = sliceUrls(filtered, { offset, limit: limitN }); + if (offset !== undefined || limitN !== undefined) { + console.error( + `Slicing: offset=${offset ?? 0} limit=${limitN ?? "∞"} → ${work.length} URLs`, + ); + } + + if (opts.dryRun) { + for (const u of work) console.log(u); + return; + } + + if (work.length === 0) { + console.error("Nothing to download."); + return; + } + + const limit = pLimit(parseInt(opts.concurrency, 10) || 4); + const manifestPath = opts.manifest; + if (manifestPath) await writeFile(manifestPath, ""); + + const bar = new ProgressBar({ total: work.length }); + const failures: FailureEntry[] = []; + let totalBytes = 0; + + await Promise.all( + work.map((url) => + limit(async () => { + const dest = urlToOutputPath(url, opts.output, { + hostPrefix: opts.hostPrefix, + }); + if (await fileExists(dest)) { + await record(manifestPath, { url, dest, status: "skipped_exists" }); + bar.tick("skipped"); + return; + } + const result = await recover(url, dest, { timestamp: opts.timestamp }); + if (result.status === "downloaded") { + totalBytes += result.bytes; + await record(manifestPath, { + url, + dest, + status: "downloaded", + bytes: result.bytes, + snapshot: result.snapshotUrl, + timestamp: result.timestamp, + }); + bar.tick("ok"); + } else if (result.status === "no_snapshot") { + failures.push({ url, reason: "no_snapshot" }); + await record(manifestPath, { url, status: "no_snapshot" }); + bar.tick("failed"); + } else { + failures.push({ url, reason: result.error }); + await record(manifestPath, { url, status: "error", error: result.error }); + bar.tick("failed"); + } + }), + ), + ); + + bar.finish(); + + const counters = bar.snapshot(); + const summary = buildSummary({ + totalFound: work.length, + attachmentGuids: attachmentGuids.size, + ok: counters.ok, + skipped: counters.skipped, + failed: counters.failed, + bytes: totalBytes, + elapsedMs: Date.now() - startedAt, + outputDir: opts.output, + manifestPath, + failures, + }); + console.error(summary); + + if (counters.failed > 0) process.exitCode = 1; +} + +async function record(path: string | undefined, entry: object): Promise { + if (!path) return; + await appendFile(path, JSON.stringify(entry) + "\n"); +} diff --git a/src/parser.ts b/src/parser.ts new file mode 100644 index 0000000..89598e7 --- /dev/null +++ b/src/parser.ts @@ -0,0 +1,53 @@ +import { createReadStream } from "node:fs"; +import { createInterface } from "node:readline"; +import { cleanUrl, matchesHost, normalizeHost } from "./utils/url.js"; +import { extractAttachmentGuids } from "./utils/sql.js"; +import { buildMediaUrlRegex, DEFAULT_MEDIA_EXTENSIONS } from "./utils/media.js"; + +export interface ParseOptions { + /** Restrict URLs to this site (host or scheme://host). If omitted, accept all. */ + site?: string; + /** Override the default media extension list. */ + extensions?: readonly string[]; +} + +export interface ParseResult { + urls: Set; + attachmentGuids: Set; +} + +export async function parseSqlDump( + filePath: string, + options: ParseOptions = {}, +): Promise { + const urls = new Set(); + const attachmentGuids = new Set(); + + const exts = options.extensions ?? DEFAULT_MEDIA_EXTENSIONS; + const urlRe = buildMediaUrlRegex(exts); + const hostFilter = normalizeHost(options.site); + + const stream = createReadStream(filePath, { encoding: "utf8" }); + const rl = createInterface({ input: stream, crlfDelay: Infinity }); + + for await (const line of rl) { + const matches = line.match(urlRe); + if (matches) { + for (const m of matches) { + const cleaned = cleanUrl(m); + if (matchesHost(cleaned, hostFilter)) urls.add(cleaned); + } + } + + if (line.includes("'attachment'") && line.includes("INSERT INTO")) { + for (const guid of extractAttachmentGuids(line)) { + if (matchesHost(guid, hostFilter)) { + attachmentGuids.add(guid); + urls.add(guid); + } + } + } + } + + return { urls, attachmentGuids }; +} diff --git a/src/utils/media.ts b/src/utils/media.ts new file mode 100644 index 0000000..54009bc --- /dev/null +++ b/src/utils/media.ts @@ -0,0 +1,22 @@ +export const DEFAULT_MEDIA_EXTENSIONS = [ + "jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "ico", + "mp3", "wav", "ogg", "m4a", "flac", + "mp4", "webm", "mov", "avi", "mkv", + "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", + "zip", "rar", "7z", "tar", "gz", +] as const; + +export function buildMediaUrlRegex(extensions: readonly string[]): RegExp { + if (extensions.length === 0) { + throw new Error("at least one extension is required"); + } + const escaped = extensions.map(escapeRegex).join("|"); + return new RegExp( + `https?:\\/\\/[^\\s'"<>()\\\\]+\\.(?:${escaped})(?:\\?[^\\s'"<>()\\\\]*)?`, + "gi", + ); +} + +function escapeRegex(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/utils/progress.ts b/src/utils/progress.ts new file mode 100644 index 0000000..14bd63a --- /dev/null +++ b/src/utils/progress.ts @@ -0,0 +1,63 @@ +export interface ProgressCounters { + ok: number; + skipped: number; + failed: number; +} + +export interface ProgressBarOptions { + total: number; + width?: number; + stream?: NodeJS.WriteStream; + /** Force on/off; defaults to stream.isTTY. */ + enabled?: boolean; +} + +/** + * Minimal terminal progress bar. Designed so the same instance can be used in + * tests by passing a non-TTY stream or `enabled: false` (in which case render + * is a no-op and `format()` returns the string for assertions). + */ +export class ProgressBar { + private done = 0; + private readonly counters: ProgressCounters = { ok: 0, skipped: 0, failed: 0 }; + private readonly stream: NodeJS.WriteStream; + private readonly width: number; + private readonly enabled: boolean; + + constructor(private readonly opts: ProgressBarOptions) { + this.stream = opts.stream ?? process.stderr; + this.width = opts.width ?? 30; + this.enabled = opts.enabled ?? Boolean(this.stream.isTTY); + } + + tick(kind: keyof ProgressCounters): void { + this.counters[kind] += 1; + this.done += 1; + this.render(); + } + + /** Build the displayable line without writing to the stream. */ + format(): string { + const total = Math.max(this.opts.total, 1); + const ratio = Math.min(this.done / total, 1); + const filled = Math.round(this.width * ratio); + const bar = "█".repeat(filled) + "░".repeat(this.width - filled); + const pct = String(Math.round(ratio * 100)).padStart(3, " "); + const { ok, skipped, failed } = this.counters; + return `[${bar}] ${pct}% ${this.done}/${this.opts.total} ok:${ok} skip:${skipped} fail:${failed}`; + } + + finish(): void { + if (!this.enabled) return; + this.stream.write("\n"); + } + + snapshot(): ProgressCounters & { done: number } { + return { ...this.counters, done: this.done }; + } + + private render(): void { + if (!this.enabled) return; + this.stream.write(`\r${this.format()}`); + } +} diff --git a/src/utils/slice.ts b/src/utils/slice.ts new file mode 100644 index 0000000..7b01884 --- /dev/null +++ b/src/utils/slice.ts @@ -0,0 +1,83 @@ +export interface SliceOptions { + offset?: number; + limit?: number; +} + +/** + * Parse a comma-separated list of file extensions into a normalized lowercase + * set without leading dots. Returns undefined for missing/blank input. + */ +export function parseExtensionsOption( + value: string | undefined, +): Set | undefined { + if (value === undefined || value === "") return undefined; + const exts = value + .split(",") + .map((e) => e.trim().toLowerCase().replace(/^\.+/, "")) + .filter((e) => e.length > 0); + if (exts.length === 0) return undefined; + return new Set(exts); +} + +/** + * Keep only URLs whose pathname ends with one of the given extensions + * (case-insensitive, query/fragment stripped). Malformed URLs fall back to + * a string suffix check so we don't silently drop them. + */ +export function filterUrlsByExtensions( + urls: Iterable, + extensions: Set | undefined, +): string[] { + const arr = [...urls]; + if (!extensions || extensions.size === 0) return arr; + return arr.filter((u) => { + let path = u; + try { + path = new URL(u).pathname; + } catch { + const q = u.indexOf("?"); + const h = u.indexOf("#"); + const cut = [q, h].filter((i) => i >= 0).sort((a, b) => a - b)[0]; + if (cut !== undefined) path = u.slice(0, cut); + } + const dot = path.lastIndexOf("."); + if (dot < 0) return false; + return extensions.has(path.slice(dot + 1).toLowerCase()); + }); +} + +/** + * Apply offset/limit to an iterable of URLs. Sorts deterministically first + * so re-runs with the same dump and the same `--offset`/`--limit` always pick + * the same window — important when chunking a large recovery across runs. + * + * `offset` and `limit` are treated as non-negative; negatives are clamped to 0 + * (limit < 0 means "no limit"). + */ +export function sliceUrls( + urls: Iterable, + { offset = 0, limit }: SliceOptions = {}, +): string[] { + const sorted = [...urls].sort(); + const start = Math.max(0, Math.floor(offset)); + if (limit === undefined) return sorted.slice(start); + const n = Math.floor(limit); + if (n < 0) return sorted.slice(start); + return sorted.slice(start, start + n); +} + +/** Parse a CLI integer option, returning undefined for missing/blank input. */ +export function parseIntOption( + value: string | undefined, + optionName: string, +): number | undefined { + if (value === undefined || value === "") return undefined; + const n = Number(value); + if (!Number.isFinite(n) || !Number.isInteger(n)) { + throw new Error(`${optionName} must be an integer, got "${value}"`); + } + if (n < 0) { + throw new Error(`${optionName} must be >= 0, got ${n}`); + } + return n; +} diff --git a/src/utils/snapshot.ts b/src/utils/snapshot.ts new file mode 100644 index 0000000..4d10e5d --- /dev/null +++ b/src/utils/snapshot.ts @@ -0,0 +1,31 @@ +/** + * Build a Wayback "raw asset" URL that resolves to the closest capture of + * `targetUrl` to `timestamp`, server-side. The `id_` flag asks for the original + * bytes instead of the rewritten archive page. + * + * https://web.archive.org/web/id_/ + * + * `timestamp` may be a full 14-digit string (YYYYMMDDhhmmss) or any prefix + * (e.g. "2020", "202003"). If omitted we default to "2", which means "any + * capture from the year 2000 onward" — broad enough to cover every WordPress + * upload while still being a valid prefix (an empty timestamp redirects to + * the archive homepage). + */ +export function buildSnapshotUrl(targetUrl: string, timestamp?: string): string { + const ts = normalizeTimestamp(timestamp); + return `https://web.archive.org/web/${ts}id_/${targetUrl}`; +} + +export function normalizeTimestamp(ts?: string): string { + if (!ts) return "2"; + const cleaned = ts.replace(/[^0-9]/g, ""); + return cleaned.length > 0 ? cleaned : "2"; +} + +const TS_RE = /\/web\/(\d{1,14})(?:id_)?\//; + +/** Pull the resolved 14-digit timestamp out of an effective Wayback URL after redirects. */ +export function extractTimestamp(snapshotUrl: string): string | undefined { + const m = snapshotUrl.match(TS_RE); + return m ? m[1] : undefined; +} diff --git a/src/utils/sql.ts b/src/utils/sql.ts new file mode 100644 index 0000000..7919dee --- /dev/null +++ b/src/utils/sql.ts @@ -0,0 +1,47 @@ +import { cleanUrl } from "./url.js"; + +/** Split an INSERT line into individual VALUES tuples, respecting SQL string escaping. */ +export function splitTuples(line: string): string[] { + const tuples: string[] = []; + let depth = 0; + let start = -1; + let inStr = false; + let escaped = false; + + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (inStr) { + if (escaped) { escaped = false; continue; } + if (c === "\\") { escaped = true; continue; } + if (c === "'") inStr = false; + continue; + } + if (c === "'") { inStr = true; continue; } + if (c === "(") { + if (depth === 0) start = i + 1; + depth++; + } else if (c === ")") { + depth--; + if (depth === 0 && start >= 0) { + tuples.push(line.slice(start, i)); + start = -1; + } + } + } + return tuples; +} + +/** + * Parse an `INSERT INTO wp_posts ...` line and pull guid values for attachment rows. + * Column order can vary across exports, so we look for the first URL-shaped string + * inside any tuple that also contains the literal 'attachment'. + */ +export function extractAttachmentGuids(line: string): string[] { + const out: string[] = []; + for (const tuple of splitTuples(line)) { + if (!/'attachment'/.test(tuple)) continue; + const urlMatch = tuple.match(/'(https?:\/\/[^']+)'/); + if (urlMatch) out.push(cleanUrl(urlMatch[1])); + } + return out; +} diff --git a/src/utils/summary.ts b/src/utils/summary.ts new file mode 100644 index 0000000..d0ad03d --- /dev/null +++ b/src/utils/summary.ts @@ -0,0 +1,81 @@ +export interface FailureEntry { + url: string; + reason: string; +} + +export interface SummaryInput { + totalFound: number; + attachmentGuids: number; + ok: number; + skipped: number; + failed: number; + bytes: number; + elapsedMs: number; + outputDir: string; + manifestPath?: string; + failures?: FailureEntry[]; + /** Cap how many failure URLs to print. Default: 10. */ + maxFailuresShown?: number; +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ["KB", "MB", "GB", "TB"]; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${value.toFixed(2)} ${units[unit]}`; +} + +export function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + const totalSec = Math.round(ms / 1000); + const h = Math.floor(totalSec / 3600); + const m = Math.floor((totalSec % 3600) / 60); + const s = totalSec % 60; + if (h > 0) return `${h}h${m}m${s}s`; + if (m > 0) return `${m}m${s}s`; + return `${s}s`; +} + +export function buildSummary(input: SummaryInput): string { + const max = input.maxFailuresShown ?? 10; + const total = input.ok + input.skipped + input.failed; + const successRate = + total > 0 ? `${Math.round((input.ok / total) * 100)}%` : "n/a"; + + const lines = [ + "", + "─── Recovery summary ─────────────────────────", + ` URLs found in dump : ${input.totalFound} (${input.attachmentGuids} attachment GUIDs)`, + ` Processed : ${total}`, + ` ✓ Downloaded : ${input.ok}`, + ` ↷ Skipped (exists) : ${input.skipped}`, + ` ✗ Failed : ${input.failed}`, + ` Success rate : ${successRate}`, + ` Total downloaded : ${formatBytes(input.bytes)}`, + ` Elapsed : ${formatDuration(input.elapsedMs)}`, + ` Output directory : ${input.outputDir}`, + ]; + + if (input.manifestPath) { + lines.push(` Manifest (JSONL) : ${input.manifestPath}`); + } + + const failures = input.failures ?? []; + if (failures.length > 0) { + lines.push("", " Failures:"); + for (const f of failures.slice(0, max)) { + lines.push(` - ${f.url} (${f.reason})`); + } + if (failures.length > max) { + lines.push(` … and ${failures.length - max} more`); + } + } + + lines.push("──────────────────────────────────────────────", ""); + return lines.join("\n"); +} diff --git a/src/utils/url.ts b/src/utils/url.ts new file mode 100644 index 0000000..1ba2c75 --- /dev/null +++ b/src/utils/url.ts @@ -0,0 +1,25 @@ +/** Strip trailing punctuation that often glues onto URLs in SQL strings. */ +export function cleanUrl(u: string): string { + return u.replace(/[),;.'"\\]+$/g, ""); +} + +/** Normalize a site argument ("example.com", "https://example.com/") to a bare host. */ +export function normalizeHost(site?: string): string | null { + if (!site) return null; + try { + const url = new URL(site.includes("://") ? site : `https://${site}`); + return url.host.toLowerCase(); + } catch { + return site.toLowerCase(); + } +} + +/** Does `url` belong to `host`? When host is null, every URL passes. */ +export function matchesHost(url: string, host: string | null): boolean { + if (!host) return true; + try { + return new URL(url).host.toLowerCase() === host; + } catch { + return false; + } +} diff --git a/src/utils/wayback-url.ts b/src/utils/wayback-url.ts new file mode 100644 index 0000000..eacd1b3 --- /dev/null +++ b/src/utils/wayback-url.ts @@ -0,0 +1,32 @@ +import { join } from "node:path"; + +export interface UrlToPathOptions { + /** Prefix the output path with the URL host. Default: true. */ + hostPrefix?: boolean; +} + +/** + * Map an original URL to a path on disk under outputDir, preserving the + * original site directory structure (e.g. wp-content/uploads/2020/01/foo.jpg). + */ +export function urlToOutputPath( + targetUrl: string, + outputDir: string, + options: UrlToPathOptions = {}, +): string { + const u = new URL(targetUrl); + const hostPrefix = options.hostPrefix ?? true; + const pathname = decodeURIComponent(u.pathname); + const segments = pathname.split("/").map(sanitizeSegment).filter(Boolean); + if (segments.length === 0) segments.push("index"); + const prefix = hostPrefix ? [sanitizeSegment(u.host)] : []; + return join(outputDir, ...prefix, ...segments); +} + +/** Strip path traversal and characters not safe across filesystems. */ +function sanitizeSegment(s: string): string { + return s + .replace(/\\/g, "_") + .replace(/[<>:"|?*\x00-\x1f]/g, "_") + .replace(/^\.+$/, "_"); +} diff --git a/src/wayback.ts b/src/wayback.ts new file mode 100644 index 0000000..1b53c21 --- /dev/null +++ b/src/wayback.ts @@ -0,0 +1,141 @@ +import { mkdir, rename, stat, unlink } from "node:fs/promises"; +import { createWriteStream } from "node:fs"; +import { dirname } from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { buildSnapshotUrl, extractTimestamp } from "./utils/snapshot.js"; + +const UA = "wp-media-rewind/0.1 (+https://archive.org)"; + +export interface RecoverOptions { + timestamp?: string; + signal?: AbortSignal; + /** Number of retry attempts on 5xx / network errors. Default: 2. */ + retries?: number; +} + +export type RecoverResult = + | { status: "downloaded"; bytes: number; snapshotUrl: string; timestamp?: string } + | { status: "no_snapshot" } + | { status: "error"; error: string }; + +/** + * Pull `targetUrl` from the Wayback Machine into `destPath`. Uses the + * `web.archive.org/web/id_/` form so the server picks the closest + * capture and we get raw bytes (not a rewritten archive page). + */ +export async function recover( + targetUrl: string, + destPath: string, + opts: RecoverOptions = {}, +): Promise { + const url = buildSnapshotUrl(targetUrl, opts.timestamp); + const retries = opts.retries ?? 2; + + let lastError = ""; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const res = await fetch(url, { + headers: { "User-Agent": UA }, + redirect: "follow", + signal: opts.signal, + }); + + if (res.status === 404) { + await drain(res); + return { status: "no_snapshot" }; + } + + if (res.status >= 500 || res.status === 429) { + await drain(res); + lastError = `http ${res.status}`; + await backoff(attempt, opts.signal); + continue; + } + + if (!res.ok || !res.body) { + await drain(res); + return { status: "error", error: `http ${res.status}` }; + } + + // Sanity check: the archive sometimes serves an HTML "calendar" page even on + // 200 when it can't find a real capture. Detect that and treat as no_snapshot. + const contentType = res.headers.get("content-type") ?? ""; + const effective = res.url || url; + if (looksLikeCalendar(effective, contentType, targetUrl)) { + await drain(res); + return { status: "no_snapshot" }; + } + + await mkdir(dirname(destPath), { recursive: true }); + const tempPath = `${destPath}.part`; + try { + await pipeline(Readable.fromWeb(res.body as any), createWriteStream(tempPath)); + await rename(tempPath, destPath); + } catch (err) { + await unlink(tempPath).catch(() => {}); + throw err; + } + const s = await stat(destPath); + return { + status: "downloaded", + bytes: s.size, + snapshotUrl: effective, + timestamp: extractTimestamp(effective), + }; + } catch (err) { + lastError = err instanceof Error ? err.message : String(err); + if (attempt < retries) await backoff(attempt, opts.signal); + } + } + + return { status: "error", error: lastError || "unknown error" }; +} + +function looksLikeCalendar( + effectiveUrl: string, + contentType: string, + targetUrl: string, +): boolean { + // Pattern: archive.org redirected to /web// without id_, or to a + // wildcard calendar page like /web/*/, and served HTML. + if (!contentType.toLowerCase().includes("html")) return false; + if (/\/web\/\*\//.test(effectiveUrl)) return true; + // If we asked for image/pdf/etc. and got HTML back, it's the calendar. + if (/\.(jpg|jpeg|png|gif|webp|bmp|svg|ico|mp3|mp4|webm|pdf|zip)(\?|$)/i.test(targetUrl)) { + return true; + } + return false; +} + +async function drain(res: Response): Promise { + try { + if (res.body) await res.body.cancel(); + } catch { /* ignore */ } +} + +async function backoff(attempt: number, signal?: AbortSignal): Promise { + const ms = Math.min(2_000 * 2 ** attempt, 10_000); + await new Promise((resolve, reject) => { + const t = setTimeout(resolve, ms); + if (signal) { + signal.addEventListener( + "abort", + () => { + clearTimeout(t); + reject(new Error("aborted")); + }, + { once: true }, + ); + } + }); +} + +export async function fileExists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} diff --git a/tests/media.test.ts b/tests/media.test.ts new file mode 100644 index 0000000..5e54034 --- /dev/null +++ b/tests/media.test.ts @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildMediaUrlRegex, DEFAULT_MEDIA_EXTENSIONS } from "../src/utils/media.js"; + +test("default regex matches typical WordPress upload URLs", () => { + const re = buildMediaUrlRegex(DEFAULT_MEDIA_EXTENSIONS); + const text = + "see and " + + "https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2 and ignore https://site.test/page"; + const matches = text.match(re) ?? []; + assert.deepEqual(matches, [ + "https://site.test/wp-content/uploads/2020/01/photo.jpg", + "https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2", + ]); +}); + +test("regex is case-insensitive on extension", () => { + const re = buildMediaUrlRegex(["jpg"]); + assert.ok(re.test("https://x.test/a.JPG")); +}); + +test("regex respects custom extensions only", () => { + const re = buildMediaUrlRegex(["xyz"]); + assert.equal("https://x.test/a.jpg".match(re), null); + assert.ok(re.test("https://x.test/a.xyz")); +}); + +test("buildMediaUrlRegex throws on empty list", () => { + assert.throws(() => buildMediaUrlRegex([]), /at least one extension/); +}); + +test("regex stops at quotes and whitespace", () => { + const re = buildMediaUrlRegex(["png"]); + const text = `'https://x.test/a.png' "https://x.test/b.png"`; + const matches = text.match(re) ?? []; + assert.deepEqual(matches, ["https://x.test/a.png", "https://x.test/b.png"]); +}); diff --git a/tests/progress.test.ts b/tests/progress.test.ts new file mode 100644 index 0000000..358a08b --- /dev/null +++ b/tests/progress.test.ts @@ -0,0 +1,62 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { ProgressBar } from "../src/utils/progress.js"; + +test("format reflects ticks", () => { + const bar = new ProgressBar({ total: 4, width: 10, enabled: false }); + bar.tick("ok"); + bar.tick("skipped"); + const line = bar.format(); + assert.match(line, /50%/); + assert.match(line, /2\/4/); + assert.match(line, /ok:1/); + assert.match(line, /skip:1/); + assert.match(line, /fail:0/); +}); + +test("snapshot returns running totals", () => { + const bar = new ProgressBar({ total: 3, enabled: false }); + bar.tick("ok"); + bar.tick("failed"); + assert.deepEqual(bar.snapshot(), { ok: 1, skipped: 0, failed: 1, done: 2 }); +}); + +test("format reaches 100% on completion", () => { + const bar = new ProgressBar({ total: 2, width: 4, enabled: false }); + bar.tick("ok"); + bar.tick("ok"); + assert.match(bar.format(), /100%/); +}); + +test("disabled bar does not write to stream", () => { + let written = ""; + const fakeStream = { + isTTY: false, + write: (s: string) => { + written += s; + return true; + }, + } as unknown as NodeJS.WriteStream; + const bar = new ProgressBar({ total: 1, stream: fakeStream }); + bar.tick("ok"); + bar.finish(); + assert.equal(written, ""); +}); + +test("enabled bar writes carriage-return updates", () => { + let written = ""; + const fakeStream = { + isTTY: true, + write: (s: string) => { + written += s; + return true; + }, + } as unknown as NodeJS.WriteStream; + const bar = new ProgressBar({ total: 2, width: 4, stream: fakeStream }); + bar.tick("ok"); + bar.tick("ok"); + bar.finish(); + assert.ok(written.startsWith("\r")); + assert.ok(written.endsWith("\n")); + assert.match(written, /100%/); +}); diff --git a/tests/slice.test.ts b/tests/slice.test.ts new file mode 100644 index 0000000..44bbc86 --- /dev/null +++ b/tests/slice.test.ts @@ -0,0 +1,135 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + filterUrlsByExtensions, + parseExtensionsOption, + parseIntOption, + sliceUrls, +} from "../src/utils/slice.js"; + +const sample = [ + "https://x.test/c.jpg", + "https://x.test/a.jpg", + "https://x.test/b.jpg", + "https://x.test/e.jpg", + "https://x.test/d.jpg", +]; + +test("sliceUrls sorts deterministically with no slice options", () => { + assert.deepEqual(sliceUrls(sample), [ + "https://x.test/a.jpg", + "https://x.test/b.jpg", + "https://x.test/c.jpg", + "https://x.test/d.jpg", + "https://x.test/e.jpg", + ]); +}); + +test("sliceUrls applies offset only", () => { + assert.deepEqual(sliceUrls(sample, { offset: 2 }), [ + "https://x.test/c.jpg", + "https://x.test/d.jpg", + "https://x.test/e.jpg", + ]); +}); + +test("sliceUrls applies limit only", () => { + assert.deepEqual(sliceUrls(sample, { limit: 2 }), [ + "https://x.test/a.jpg", + "https://x.test/b.jpg", + ]); +}); + +test("sliceUrls applies offset and limit together", () => { + assert.deepEqual(sliceUrls(sample, { offset: 1, limit: 2 }), [ + "https://x.test/b.jpg", + "https://x.test/c.jpg", + ]); +}); + +test("sliceUrls returns [] when offset exceeds length", () => { + assert.deepEqual(sliceUrls(sample, { offset: 99 }), []); +}); + +test("sliceUrls works with a Set", () => { + const set = new Set(sample); + assert.deepEqual(sliceUrls(set, { limit: 1 }), ["https://x.test/a.jpg"]); +}); + +test("sliceUrls clamps negative offset to 0", () => { + assert.deepEqual(sliceUrls(sample, { offset: -5, limit: 1 }), [ + "https://x.test/a.jpg", + ]); +}); + +test("sliceUrls treats limit=0 as empty", () => { + assert.deepEqual(sliceUrls(sample, { limit: 0 }), []); +}); + +test("parseIntOption returns undefined when missing", () => { + assert.equal(parseIntOption(undefined, "--limit"), undefined); + assert.equal(parseIntOption("", "--limit"), undefined); +}); + +test("parseIntOption parses integers", () => { + assert.equal(parseIntOption("0", "--offset"), 0); + assert.equal(parseIntOption("42", "--limit"), 42); +}); + +test("parseIntOption rejects non-integers", () => { + assert.throws(() => parseIntOption("3.14", "--limit"), /must be an integer/); + assert.throws(() => parseIntOption("abc", "--offset"), /must be an integer/); +}); + +test("parseIntOption rejects negatives", () => { + assert.throws(() => parseIntOption("-1", "--limit"), /must be >= 0/); +}); + +test("parseExtensionsOption returns undefined for missing/blank", () => { + assert.equal(parseExtensionsOption(undefined), undefined); + assert.equal(parseExtensionsOption(""), undefined); + assert.equal(parseExtensionsOption(" , , "), undefined); +}); + +test("parseExtensionsOption normalizes case, dots, and whitespace", () => { + const set = parseExtensionsOption(" .JPG, png ,.WebP"); + assert.deepEqual([...(set ?? [])].sort(), ["jpg", "png", "webp"]); +}); + +test("filterUrlsByExtensions returns input unchanged when no filter", () => { + const urls = ["https://x.test/a.jpg", "https://x.test/b.png"]; + assert.deepEqual(filterUrlsByExtensions(urls, undefined), urls); +}); + +test("filterUrlsByExtensions keeps only matching extensions", () => { + const urls = [ + "https://x.test/a.jpg", + "https://x.test/b.png", + "https://x.test/c.JPG", + "https://x.test/d.gif", + ]; + const set = parseExtensionsOption("jpg"); + assert.deepEqual(filterUrlsByExtensions(urls, set), [ + "https://x.test/a.jpg", + "https://x.test/c.JPG", + ]); +}); + +test("filterUrlsByExtensions ignores query string and fragment", () => { + const urls = [ + "https://x.test/a.jpg?v=2", + "https://x.test/b.png#frag", + "https://x.test/c.gif?x=1#y", + ]; + const set = parseExtensionsOption("jpg,png"); + assert.deepEqual(filterUrlsByExtensions(urls, set), [ + "https://x.test/a.jpg?v=2", + "https://x.test/b.png#frag", + ]); +}); + +test("filterUrlsByExtensions drops URLs without an extension", () => { + const urls = ["https://x.test/no-ext", "https://x.test/a.jpg"]; + const set = parseExtensionsOption("jpg"); + assert.deepEqual(filterUrlsByExtensions(urls, set), ["https://x.test/a.jpg"]); +}); diff --git a/tests/snapshot.test.ts b/tests/snapshot.test.ts new file mode 100644 index 0000000..b73109c --- /dev/null +++ b/tests/snapshot.test.ts @@ -0,0 +1,59 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + buildSnapshotUrl, + extractTimestamp, + normalizeTimestamp, +} from "../src/utils/snapshot.js"; + +test("buildSnapshotUrl uses '2' as default timestamp prefix", () => { + assert.equal( + buildSnapshotUrl("https://example.com/a.jpg"), + "https://web.archive.org/web/2id_/https://example.com/a.jpg", + ); +}); + +test("buildSnapshotUrl honors a full timestamp", () => { + assert.equal( + buildSnapshotUrl("https://example.com/a.jpg", "20200101000000"), + "https://web.archive.org/web/20200101000000id_/https://example.com/a.jpg", + ); +}); + +test("buildSnapshotUrl honors a partial timestamp", () => { + assert.equal( + buildSnapshotUrl("https://example.com/a.jpg", "2020"), + "https://web.archive.org/web/2020id_/https://example.com/a.jpg", + ); +}); + +test("normalizeTimestamp strips non-digits", () => { + assert.equal(normalizeTimestamp("2020-01-01"), "20200101"); + assert.equal(normalizeTimestamp("2020/01"), "202001"); +}); + +test("normalizeTimestamp falls back to '2' on empty input", () => { + assert.equal(normalizeTimestamp(undefined), "2"); + assert.equal(normalizeTimestamp(""), "2"); + assert.equal(normalizeTimestamp("---"), "2"); +}); + +test("extractTimestamp pulls timestamp from a resolved snapshot URL", () => { + assert.equal( + extractTimestamp( + "https://web.archive.org/web/20221020185708id_/https://x.test/a.jpg", + ), + "20221020185708", + ); +}); + +test("extractTimestamp works without id_ marker", () => { + assert.equal( + extractTimestamp("https://web.archive.org/web/20221020185708/https://x.test/a.jpg"), + "20221020185708", + ); +}); + +test("extractTimestamp returns undefined for non-wayback URLs", () => { + assert.equal(extractTimestamp("https://example.com/a.jpg"), undefined); +}); diff --git a/tests/sql.test.ts b/tests/sql.test.ts new file mode 100644 index 0000000..73d02c8 --- /dev/null +++ b/tests/sql.test.ts @@ -0,0 +1,47 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { extractAttachmentGuids, splitTuples } from "../src/utils/sql.js"; + +test("splitTuples splits flat tuples", () => { + const line = "INSERT INTO `t` VALUES (1,'a'),(2,'b'),(3,'c');"; + assert.deepEqual(splitTuples(line), ["1,'a'", "2,'b'", "3,'c'"]); +}); + +test("splitTuples ignores parens inside quoted strings", () => { + const line = "INSERT INTO `t` VALUES (1,'foo (bar) baz'),(2,'x');"; + assert.deepEqual(splitTuples(line), ["1,'foo (bar) baz'", "2,'x'"]); +}); + +test("splitTuples handles escaped quotes", () => { + const line = "INSERT INTO `t` VALUES (1,'it\\'s ok'),(2,'y');"; + const tuples = splitTuples(line); + assert.equal(tuples.length, 2); + assert.equal(tuples[0], "1,'it\\'s ok'"); +}); + +test("extractAttachmentGuids returns URLs from attachment rows", () => { + const line = + "INSERT INTO `wp_posts` VALUES " + + "(1,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/wp-content/uploads/2020/01/foo.jpg',0,'attachment','image/jpeg',0)," + + "(2,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);"; + assert.deepEqual(extractAttachmentGuids(line), [ + "https://site.test/wp-content/uploads/2020/01/foo.jpg", + ]); +}); + +test("extractAttachmentGuids returns [] when no attachment rows", () => { + const line = + "INSERT INTO `wp_posts` VALUES (2,1,'2020-01-01','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);"; + assert.deepEqual(extractAttachmentGuids(line), []); +}); + +test("extractAttachmentGuids handles multiple attachment rows", () => { + const line = + "INSERT INTO `wp_posts` VALUES " + + "(1,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/a.jpg',0,'attachment','image/jpeg',0)," + + "(2,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/b.png',0,'attachment','image/png',0);"; + assert.deepEqual(extractAttachmentGuids(line), [ + "https://site.test/a.jpg", + "https://site.test/b.png", + ]); +}); diff --git a/tests/summary.test.ts b/tests/summary.test.ts new file mode 100644 index 0000000..12c31e3 --- /dev/null +++ b/tests/summary.test.ts @@ -0,0 +1,90 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { buildSummary, formatBytes, formatDuration } from "../src/utils/summary.js"; + +test("formatBytes scales units", () => { + assert.equal(formatBytes(0), "0 B"); + assert.equal(formatBytes(512), "512 B"); + assert.equal(formatBytes(2048), "2.00 KB"); + assert.equal(formatBytes(5 * 1024 * 1024), "5.00 MB"); + assert.equal(formatBytes(3 * 1024 ** 3), "3.00 GB"); +}); + +test("formatDuration formats ranges", () => { + assert.equal(formatDuration(250), "250ms"); + assert.equal(formatDuration(1500), "2s"); + assert.equal(formatDuration(65_000), "1m5s"); + assert.equal(formatDuration(3_725_000), "1h2m5s"); +}); + +test("buildSummary includes counters and rate", () => { + const out = buildSummary({ + totalFound: 100, + attachmentGuids: 40, + ok: 80, + skipped: 10, + failed: 10, + bytes: 1024, + elapsedMs: 5000, + outputDir: "/tmp/out", + }); + assert.match(out, /URLs found in dump : 100 \(40 attachment GUIDs\)/); + assert.match(out, /✓ Downloaded\s*: 80/); + assert.match(out, /↷ Skipped\s*\(exists\)\s*: 10/); + assert.match(out, /✗ Failed\s*: 10/); + assert.match(out, /Success rate\s*: 80%/); + assert.match(out, /1\.00 KB/); + assert.match(out, /5s/); + assert.match(out, /\/tmp\/out/); +}); + +test("buildSummary lists failures with truncation", () => { + const failures = Array.from({ length: 12 }, (_, i) => ({ + url: `https://x.test/f${i}.jpg`, + reason: "no_snapshot", + })); + const out = buildSummary({ + totalFound: 12, + attachmentGuids: 0, + ok: 0, + skipped: 0, + failed: 12, + bytes: 0, + elapsedMs: 100, + outputDir: "/o", + failures, + maxFailuresShown: 5, + }); + assert.match(out, /f0\.jpg/); + assert.match(out, /f4\.jpg/); + assert.ok(!out.includes("f5.jpg")); + assert.match(out, /and 7 more/); +}); + +test("buildSummary omits manifest line when not provided", () => { + const out = buildSummary({ + totalFound: 1, + attachmentGuids: 0, + ok: 1, + skipped: 0, + failed: 0, + bytes: 100, + elapsedMs: 10, + outputDir: "/o", + }); + assert.ok(!out.includes("Manifest")); +}); + +test("buildSummary handles 0 processed", () => { + const out = buildSummary({ + totalFound: 0, + attachmentGuids: 0, + ok: 0, + skipped: 0, + failed: 0, + bytes: 0, + elapsedMs: 5, + outputDir: "/o", + }); + assert.match(out, /Success rate\s*: n\/a/); +}); diff --git a/tests/url.test.ts b/tests/url.test.ts new file mode 100644 index 0000000..570e370 --- /dev/null +++ b/tests/url.test.ts @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { cleanUrl, matchesHost, normalizeHost } from "../src/utils/url.js"; + +test("cleanUrl strips trailing punctuation", () => { + assert.equal(cleanUrl("https://x.com/a.jpg)."), "https://x.com/a.jpg"); + assert.equal(cleanUrl("https://x.com/a.jpg',"), "https://x.com/a.jpg"); + assert.equal(cleanUrl("https://x.com/a.jpg"), "https://x.com/a.jpg"); +}); + +test("cleanUrl preserves query strings", () => { + assert.equal( + cleanUrl("https://x.com/a.jpg?v=1"), + "https://x.com/a.jpg?v=1", + ); +}); + +test("normalizeHost accepts bare hosts and full URLs", () => { + assert.equal(normalizeHost("Example.COM"), "example.com"); + assert.equal(normalizeHost("https://Example.com/path"), "example.com"); + assert.equal(normalizeHost("http://example.com:8080/"), "example.com:8080"); +}); + +test("normalizeHost returns null for empty input", () => { + assert.equal(normalizeHost(undefined), null); + assert.equal(normalizeHost(""), null); +}); + +test("matchesHost null filter accepts everything", () => { + assert.equal(matchesHost("https://anywhere.test/x.jpg", null), true); +}); + +test("matchesHost compares case-insensitively", () => { + assert.equal(matchesHost("https://Example.com/x.jpg", "example.com"), true); + assert.equal(matchesHost("https://other.com/x.jpg", "example.com"), false); +}); + +test("matchesHost rejects malformed URLs", () => { + assert.equal(matchesHost("not a url", "example.com"), false); +}); diff --git a/tests/wayback-url.test.ts b/tests/wayback-url.test.ts new file mode 100644 index 0000000..8125fb0 --- /dev/null +++ b/tests/wayback-url.test.ts @@ -0,0 +1,43 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { urlToOutputPath } from "../src/utils/wayback-url.js"; + +test("urlToOutputPath preserves host and path layout", () => { + assert.equal( + urlToOutputPath( + "https://example.com/wp-content/uploads/2020/01/photo.jpg", + "/out", + ), + "/out/example.com/wp-content/uploads/2020/01/photo.jpg", + ); +}); + +test("urlToOutputPath decodes percent-encoded segments", () => { + assert.equal( + urlToOutputPath("https://example.com/uploads/foto%20test.jpg", "/out"), + "/out/example.com/uploads/foto test.jpg", + ); +}); + +test("urlToOutputPath sanitizes path-traversal segments", () => { + const out = urlToOutputPath("https://example.com/../etc/passwd", "/out"); + assert.ok(!out.includes("..")); +}); + +test("urlToOutputPath omits host directory when hostPrefix=false", () => { + assert.equal( + urlToOutputPath( + "https://example.com/wp-content/uploads/2020/01/photo.jpg", + "/out", + { hostPrefix: false }, + ), + "/out/wp-content/uploads/2020/01/photo.jpg", + ); +}); + +test("urlToOutputPath falls back to 'index' for empty paths", () => { + assert.equal( + urlToOutputPath("https://example.com/", "/out"), + "/out/example.com/index", + ); +}); diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 0000000..8cd4c1a --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..d99cd2a --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "outDir": "dist", + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": false, + "sourceMap": false + }, + "include": ["src/**/*", "tests/**/*"] +}