mirror of
https://github.com/debba/wp-media-rewind.git
synced 2026-08-03 07:29:00 +02:00
feat: initial project setup with CLI, utils, and tests
This commit is contained in:
+55
@@ -0,0 +1,55 @@
|
||||
# Recovered media output
|
||||
recovered/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
dist/
|
||||
build/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
.pnpm-store/
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Data / dumps
|
||||
*.jsonl
|
||||
*.sql
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
*.db
|
||||
|
||||
# Env / secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# Testing / cache
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
coverage/
|
||||
.cache/
|
||||
|
||||
# IDE / editor
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Andrea Debernardi
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,125 @@
|
||||
# wp-media-rewind
|
||||
|
||||
A small CLI for the situation nobody wants to be in: you have a WordPress SQL
|
||||
dump, the original site is gone, and the `wp-content/uploads` folder vanished
|
||||
with it. `wp-media-rewind` reads the dump, pulls every media URL it can find, and
|
||||
tries to fetch each file back from the Wayback Machine — keeping the original
|
||||
folder structure on disk.
|
||||
|
||||
It's deliberately not clever. No database, no headless browser, no scraping
|
||||
the live site. Just: parse the dump, ask `archive.org/wayback/available` for
|
||||
the closest snapshot, download the raw bytes.
|
||||
|
||||
## What it does
|
||||
|
||||
- Streams a `.sql` dump line by line, so 2 GB exports don't blow up memory.
|
||||
- Pulls media URLs from two places:
|
||||
- any URL with a media-looking extension found anywhere in the file
|
||||
(covers stuff embedded in `post_content`, options, postmeta, etc.),
|
||||
- `guid` values from `wp_posts` rows where `post_type='attachment'`.
|
||||
- For each URL, asks the Wayback availability API for the closest snapshot
|
||||
(optionally near a timestamp you pick), then downloads the raw asset using
|
||||
the `id_` flag so you get original bytes, not a rewritten archive page.
|
||||
- Writes files under `<output>/<host>/<original-path>` by default, so a URL
|
||||
like `https://example.com/wp-content/uploads/2020/01/photo.jpg` lands in
|
||||
`recovered/example.com/wp-content/uploads/2020/01/photo.jpg`.
|
||||
- Skips files that already exist on disk, so re-runs are cheap.
|
||||
- Shows a progress bar while it works and prints a summary at the end with
|
||||
counts, total bytes, elapsed time, and the first failures.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
## Use
|
||||
|
||||
```bash
|
||||
# Positional argument
|
||||
node dist/index.js dump.sql --site example.com --output ./recovered
|
||||
|
||||
# Or with the explicit flag
|
||||
node dist/index.js --sql ./dump.sql --site example.com --output ./recovered
|
||||
|
||||
# See what would be fetched without downloading anything
|
||||
node dist/index.js dump.sql --site example.com --dry-run
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
| Flag | Default | What it does |
|
||||
| --- | --- | --- |
|
||||
| `<sql-dump>` / `-f, --sql <file>` | — | Path to the WordPress SQL dump. |
|
||||
| `-o, --output <dir>` | `./recovered` | Where to write recovered files. |
|
||||
| `-s, --site <host>` | — | Only keep URLs on this host (recommended; dumps often quote URLs from other sites in `post_content`). |
|
||||
| `-t, --timestamp <YYYYMMDD>` | — | Prefer snapshots near this date. Useful if you remember roughly when the site was last alive. |
|
||||
| `-c, --concurrency <n>` | `4` | Parallel downloads. Be polite to archive.org. |
|
||||
| `--dry-run` | off | Print URLs and exit. |
|
||||
| `--manifest <file>` | — | Append a JSONL row per URL with status, snapshot, bytes, etc. |
|
||||
| `--no-host-prefix` | off | Drop the `<host>/` directory and write paths starting at the site root. |
|
||||
|
||||
### Final report
|
||||
|
||||
When the run finishes you get something like:
|
||||
|
||||
```
|
||||
─── Recovery summary ─────────────────────────
|
||||
URLs found in dump : 312 (118 attachment GUIDs)
|
||||
Processed : 312
|
||||
✓ Downloaded : 287
|
||||
↷ Skipped (exists) : 0
|
||||
✗ Failed : 25
|
||||
Success rate : 91%
|
||||
Total downloaded : 184.21 MB
|
||||
Elapsed : 4m12s
|
||||
Output directory : ./recovered
|
||||
Manifest (JSONL) : ./run.jsonl
|
||||
|
||||
Failures:
|
||||
- https://example.com/wp-content/uploads/2014/03/old.jpg (no_snapshot)
|
||||
…
|
||||
──────────────────────────────────────────────
|
||||
```
|
||||
|
||||
The exit code is non-zero if any URL failed, so you can wire it into a
|
||||
script.
|
||||
|
||||
## Notes & caveats
|
||||
|
||||
- The Wayback Machine doesn't have everything. Older or low-traffic files are
|
||||
the most common misses. Try a different `--timestamp` if a whole period
|
||||
looks empty.
|
||||
- The tool requests one availability lookup per URL. If you're recovering
|
||||
thousands of files, run with low concurrency and expect it to take a while.
|
||||
- It only restores binaries (images, PDFs, video, etc.). Posts, comments and
|
||||
other DB content are already in the dump — you don't need this for those.
|
||||
- Filenames with characters that are illegal on your filesystem get replaced
|
||||
with `_`. Path traversal segments are stripped.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
npm test # run unit tests (node:test, no extra deps)
|
||||
npm run typecheck # strict tsc --noEmit on src + tests
|
||||
npm run dev # tsx src/index.ts <args>
|
||||
```
|
||||
|
||||
The pure helpers live in `src/utils/` and have unit tests in `tests/`:
|
||||
|
||||
- `utils/url.ts` — host normalization and URL cleanup.
|
||||
- `utils/sql.ts` — splitting `INSERT VALUES (...)` tuples and pulling
|
||||
attachment GUIDs.
|
||||
- `utils/media.ts` — building the media-extension regex.
|
||||
- `utils/wayback-url.ts` — converting snapshot URLs to the raw `id_` form
|
||||
and mapping URLs to output paths.
|
||||
- `utils/progress.ts` — terminal progress bar.
|
||||
- `utils/summary.ts` — final report formatting.
|
||||
|
||||
The HTTP code (`wayback.ts`) and the SQL streaming reader (`parser.ts`) are
|
||||
intentionally thin wrappers around those utilities.
|
||||
|
||||
## License
|
||||
|
||||
MIT.
|
||||
Generated
+636
@@ -0,0 +1,636 @@
|
||||
{
|
||||
"name": "waybackmachine-media-recovery",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "waybackmachine-media-recovery",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0",
|
||||
"p-limit": "^6.1.0"
|
||||
},
|
||||
"bin": {
|
||||
"wbm-recover": "dist/index.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.7.0",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz",
|
||||
"integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"aix"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz",
|
||||
"integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz",
|
||||
"integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ia32": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz",
|
||||
"integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-loong64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz",
|
||||
"integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==",
|
||||
"cpu": [
|
||||
"loong64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-mips64el": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz",
|
||||
"integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==",
|
||||
"cpu": [
|
||||
"mips64el"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ppc64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz",
|
||||
"integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-riscv64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz",
|
||||
"integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-s390x": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz",
|
||||
"integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/netbsd-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"netbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/netbsd-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"netbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openbsd-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openbsd-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openharmony-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openharmony"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/sunos-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"sunos"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-arm64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz",
|
||||
"integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-ia32": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz",
|
||||
"integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-x64": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz",
|
||||
"integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.19.18",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.18.tgz",
|
||||
"integrity": "sha512-9v00a+dn2yWVsYDEunWC4g/TcRKVq3r8N5FuZp7u0SGrPvdN9c2yXI9bBuf5Fl0hNCb+QTIePTn5pJs2pwBOQQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "12.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
|
||||
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.27.7",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz",
|
||||
"integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"esbuild": "bin/esbuild"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@esbuild/aix-ppc64": "0.27.7",
|
||||
"@esbuild/android-arm": "0.27.7",
|
||||
"@esbuild/android-arm64": "0.27.7",
|
||||
"@esbuild/android-x64": "0.27.7",
|
||||
"@esbuild/darwin-arm64": "0.27.7",
|
||||
"@esbuild/darwin-x64": "0.27.7",
|
||||
"@esbuild/freebsd-arm64": "0.27.7",
|
||||
"@esbuild/freebsd-x64": "0.27.7",
|
||||
"@esbuild/linux-arm": "0.27.7",
|
||||
"@esbuild/linux-arm64": "0.27.7",
|
||||
"@esbuild/linux-ia32": "0.27.7",
|
||||
"@esbuild/linux-loong64": "0.27.7",
|
||||
"@esbuild/linux-mips64el": "0.27.7",
|
||||
"@esbuild/linux-ppc64": "0.27.7",
|
||||
"@esbuild/linux-riscv64": "0.27.7",
|
||||
"@esbuild/linux-s390x": "0.27.7",
|
||||
"@esbuild/linux-x64": "0.27.7",
|
||||
"@esbuild/netbsd-arm64": "0.27.7",
|
||||
"@esbuild/netbsd-x64": "0.27.7",
|
||||
"@esbuild/openbsd-arm64": "0.27.7",
|
||||
"@esbuild/openbsd-x64": "0.27.7",
|
||||
"@esbuild/openharmony-arm64": "0.27.7",
|
||||
"@esbuild/sunos-x64": "0.27.7",
|
||||
"@esbuild/win32-arm64": "0.27.7",
|
||||
"@esbuild/win32-ia32": "0.27.7",
|
||||
"@esbuild/win32-x64": "0.27.7"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
|
||||
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/get-tsconfig": {
|
||||
"version": "4.14.0",
|
||||
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
|
||||
"integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"resolve-pkg-maps": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/p-limit": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.2.0.tgz",
|
||||
"integrity": "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"yocto-queue": "^1.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/resolve-pkg-maps": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
|
||||
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
|
||||
"integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"esbuild": "~0.27.0",
|
||||
"get-tsconfig": "^4.7.5"
|
||||
},
|
||||
"bin": {
|
||||
"tsx": "dist/cli.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "~2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/yocto-queue": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz",
|
||||
"integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.20"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "wp-media-rewind",
|
||||
"version": "0.1.0",
|
||||
"description": "Pull WordPress media files out of a SQL dump using the Wayback Machine.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"wp-media-rewind": "dist/index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"start": "npm run build && node dist/index.js",
|
||||
"dev": "tsx src/index.ts",
|
||||
"test": "node --import tsx --test tests/*.test.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0",
|
||||
"p-limit": "^6.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.7.0",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
Generated
+369
@@ -0,0 +1,369 @@
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
commander:
|
||||
specifier: ^12.1.0
|
||||
version: 12.1.0
|
||||
p-limit:
|
||||
specifier: ^6.1.0
|
||||
version: 6.2.0
|
||||
devDependencies:
|
||||
'@types/node':
|
||||
specifier: ^22.7.0
|
||||
version: 22.19.18
|
||||
tsx:
|
||||
specifier: ^4.19.0
|
||||
version: 4.21.0
|
||||
typescript:
|
||||
specifier: ^5.6.0
|
||||
version: 5.9.3
|
||||
|
||||
packages:
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.7':
|
||||
resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ppc64]
|
||||
os: [aix]
|
||||
|
||||
'@esbuild/android-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/android-arm@0.27.7':
|
||||
resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/android-x64@0.27.7':
|
||||
resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/darwin-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
'@esbuild/darwin-x64@0.27.7':
|
||||
resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
'@esbuild/freebsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [freebsd]
|
||||
|
||||
'@esbuild/freebsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [freebsd]
|
||||
|
||||
'@esbuild/linux-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-arm@0.27.7':
|
||||
resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-ia32@0.27.7':
|
||||
resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ia32]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-loong64@0.27.7':
|
||||
resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-mips64el@0.27.7':
|
||||
resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [mips64el]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-ppc64@0.27.7':
|
||||
resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-riscv64@0.27.7':
|
||||
resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-s390x@0.27.7':
|
||||
resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-x64@0.27.7':
|
||||
resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/netbsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [netbsd]
|
||||
|
||||
'@esbuild/netbsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [netbsd]
|
||||
|
||||
'@esbuild/openbsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [openbsd]
|
||||
|
||||
'@esbuild/openbsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [openbsd]
|
||||
|
||||
'@esbuild/openharmony-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [openharmony]
|
||||
|
||||
'@esbuild/sunos-x64@0.27.7':
|
||||
resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [sunos]
|
||||
|
||||
'@esbuild/win32-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
|
||||
'@esbuild/win32-ia32@0.27.7':
|
||||
resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
|
||||
'@esbuild/win32-x64@0.27.7':
|
||||
resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
'@types/node@22.19.18':
|
||||
resolution: {integrity: sha512-9v00a+dn2yWVsYDEunWC4g/TcRKVq3r8N5FuZp7u0SGrPvdN9c2yXI9bBuf5Fl0hNCb+QTIePTn5pJs2pwBOQQ==}
|
||||
|
||||
commander@12.1.0:
|
||||
resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
esbuild@0.27.7:
|
||||
resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==}
|
||||
engines: {node: '>=18'}
|
||||
hasBin: true
|
||||
|
||||
fsevents@2.3.3:
|
||||
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
|
||||
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
|
||||
os: [darwin]
|
||||
|
||||
get-tsconfig@4.14.0:
|
||||
resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==}
|
||||
|
||||
p-limit@6.2.0:
|
||||
resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
resolve-pkg-maps@1.0.0:
|
||||
resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
|
||||
|
||||
tsx@4.21.0:
|
||||
resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
hasBin: true
|
||||
|
||||
typescript@5.9.3:
|
||||
resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
|
||||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
yocto-queue@1.2.2:
|
||||
resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==}
|
||||
engines: {node: '>=12.20'}
|
||||
|
||||
snapshots:
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-arm@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/darwin-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/darwin-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/freebsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/freebsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-arm@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-ia32@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-loong64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-mips64el@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-ppc64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-riscv64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-s390x@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/netbsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/netbsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openbsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openbsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openharmony-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/sunos-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-ia32@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@types/node@22.19.18':
|
||||
dependencies:
|
||||
undici-types: 6.21.0
|
||||
|
||||
commander@12.1.0: {}
|
||||
|
||||
esbuild@0.27.7:
|
||||
optionalDependencies:
|
||||
'@esbuild/aix-ppc64': 0.27.7
|
||||
'@esbuild/android-arm': 0.27.7
|
||||
'@esbuild/android-arm64': 0.27.7
|
||||
'@esbuild/android-x64': 0.27.7
|
||||
'@esbuild/darwin-arm64': 0.27.7
|
||||
'@esbuild/darwin-x64': 0.27.7
|
||||
'@esbuild/freebsd-arm64': 0.27.7
|
||||
'@esbuild/freebsd-x64': 0.27.7
|
||||
'@esbuild/linux-arm': 0.27.7
|
||||
'@esbuild/linux-arm64': 0.27.7
|
||||
'@esbuild/linux-ia32': 0.27.7
|
||||
'@esbuild/linux-loong64': 0.27.7
|
||||
'@esbuild/linux-mips64el': 0.27.7
|
||||
'@esbuild/linux-ppc64': 0.27.7
|
||||
'@esbuild/linux-riscv64': 0.27.7
|
||||
'@esbuild/linux-s390x': 0.27.7
|
||||
'@esbuild/linux-x64': 0.27.7
|
||||
'@esbuild/netbsd-arm64': 0.27.7
|
||||
'@esbuild/netbsd-x64': 0.27.7
|
||||
'@esbuild/openbsd-arm64': 0.27.7
|
||||
'@esbuild/openbsd-x64': 0.27.7
|
||||
'@esbuild/openharmony-arm64': 0.27.7
|
||||
'@esbuild/sunos-x64': 0.27.7
|
||||
'@esbuild/win32-arm64': 0.27.7
|
||||
'@esbuild/win32-ia32': 0.27.7
|
||||
'@esbuild/win32-x64': 0.27.7
|
||||
|
||||
fsevents@2.3.3:
|
||||
optional: true
|
||||
|
||||
get-tsconfig@4.14.0:
|
||||
dependencies:
|
||||
resolve-pkg-maps: 1.0.0
|
||||
|
||||
p-limit@6.2.0:
|
||||
dependencies:
|
||||
yocto-queue: 1.2.2
|
||||
|
||||
resolve-pkg-maps@1.0.0: {}
|
||||
|
||||
tsx@4.21.0:
|
||||
dependencies:
|
||||
esbuild: 0.27.7
|
||||
get-tsconfig: 4.14.0
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.3
|
||||
|
||||
typescript@5.9.3: {}
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
yocto-queue@1.2.2: {}
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env node
|
||||
import { Command } from "commander";
|
||||
import pLimit from "p-limit";
|
||||
import { appendFile, writeFile } from "node:fs/promises";
|
||||
import { parseSqlDump } from "./parser.js";
|
||||
import { fileExists, recover } from "./wayback.js";
|
||||
import { urlToOutputPath } from "./utils/wayback-url.js";
|
||||
import { ProgressBar } from "./utils/progress.js";
|
||||
import { buildSummary, type FailureEntry } from "./utils/summary.js";
|
||||
import {
|
||||
filterUrlsByExtensions,
|
||||
parseExtensionsOption,
|
||||
parseIntOption,
|
||||
sliceUrls,
|
||||
} from "./utils/slice.js";
|
||||
|
||||
interface CliOpts {
|
||||
sql?: string;
|
||||
output: string;
|
||||
site?: string;
|
||||
timestamp?: string;
|
||||
concurrency: string;
|
||||
dryRun?: boolean;
|
||||
manifest?: string;
|
||||
hostPrefix: boolean;
|
||||
limit?: string;
|
||||
offset?: string;
|
||||
filter?: string;
|
||||
}
|
||||
|
||||
const program = new Command()
|
||||
.name("wp-media-rewind")
|
||||
.description("Recover WordPress media from a SQL dump via the Wayback Machine.")
|
||||
.argument("[sql-dump]", "Path to the WordPress .sql dump (or use --sql)")
|
||||
.option("-f, --sql <file>", "Path to the WordPress .sql dump")
|
||||
.option("-o, --output <dir>", "Output directory", "./recovered")
|
||||
.option("-s, --site <host>", "Restrict to this site host (e.g. example.com)")
|
||||
.option("-t, --timestamp <YYYYMMDD>", "Preferred snapshot timestamp")
|
||||
.option("-c, --concurrency <n>", "Parallel downloads", "4")
|
||||
.option("--dry-run", "List URLs found and exit")
|
||||
.option("--manifest <file>", "Write a JSONL manifest of results")
|
||||
.option("--no-host-prefix", "Do not prefix output paths with the URL host")
|
||||
.option("--limit <n>", "Process at most N URLs (after sorting and offset)")
|
||||
.option("--offset <n>", "Skip the first N URLs (after sorting)")
|
||||
.option(
|
||||
"--filter <exts>",
|
||||
"Comma-separated extensions to keep (e.g. jpg,png,webp)",
|
||||
)
|
||||
.addHelpText(
|
||||
"after",
|
||||
`\nExamples:\n $ wp-media-rewind dump.sql -s example.com -o ./recovered\n $ wp-media-rewind --sql ./dump.sql --site example.com --dry-run\n $ wp-media-rewind dump.sql --filter jpg,png\n`,
|
||||
)
|
||||
.parse(process.argv);
|
||||
|
||||
const opts = program.opts<CliOpts>();
|
||||
const sqlPath = opts.sql ?? program.args[0];
|
||||
|
||||
if (!sqlPath) {
|
||||
console.error("error: missing SQL dump path. Pass it as positional arg or via --sql <file>.\n");
|
||||
program.help({ error: true });
|
||||
}
|
||||
|
||||
await main(sqlPath);
|
||||
|
||||
async function main(sqlPath: string) {
|
||||
const startedAt = Date.now();
|
||||
console.error(`Parsing ${sqlPath}…`);
|
||||
const { urls, attachmentGuids } = await parseSqlDump(sqlPath, {
|
||||
site: opts.site,
|
||||
});
|
||||
console.error(
|
||||
`Found ${urls.size} media URLs (${attachmentGuids.size} from wp_posts attachments).`,
|
||||
);
|
||||
|
||||
const offset = parseIntOption(opts.offset, "--offset");
|
||||
const limitN = parseIntOption(opts.limit, "--limit");
|
||||
const extensions = parseExtensionsOption(opts.filter);
|
||||
const filtered = filterUrlsByExtensions(urls, extensions);
|
||||
if (extensions) {
|
||||
console.error(
|
||||
`Filtering by extensions [${[...extensions].join(", ")}]: ${urls.size} → ${filtered.length} URLs`,
|
||||
);
|
||||
}
|
||||
const work = sliceUrls(filtered, { offset, limit: limitN });
|
||||
if (offset !== undefined || limitN !== undefined) {
|
||||
console.error(
|
||||
`Slicing: offset=${offset ?? 0} limit=${limitN ?? "∞"} → ${work.length} URLs`,
|
||||
);
|
||||
}
|
||||
|
||||
if (opts.dryRun) {
|
||||
for (const u of work) console.log(u);
|
||||
return;
|
||||
}
|
||||
|
||||
if (work.length === 0) {
|
||||
console.error("Nothing to download.");
|
||||
return;
|
||||
}
|
||||
|
||||
const limit = pLimit(parseInt(opts.concurrency, 10) || 4);
|
||||
const manifestPath = opts.manifest;
|
||||
if (manifestPath) await writeFile(manifestPath, "");
|
||||
|
||||
const bar = new ProgressBar({ total: work.length });
|
||||
const failures: FailureEntry[] = [];
|
||||
let totalBytes = 0;
|
||||
|
||||
await Promise.all(
|
||||
work.map((url) =>
|
||||
limit(async () => {
|
||||
const dest = urlToOutputPath(url, opts.output, {
|
||||
hostPrefix: opts.hostPrefix,
|
||||
});
|
||||
if (await fileExists(dest)) {
|
||||
await record(manifestPath, { url, dest, status: "skipped_exists" });
|
||||
bar.tick("skipped");
|
||||
return;
|
||||
}
|
||||
const result = await recover(url, dest, { timestamp: opts.timestamp });
|
||||
if (result.status === "downloaded") {
|
||||
totalBytes += result.bytes;
|
||||
await record(manifestPath, {
|
||||
url,
|
||||
dest,
|
||||
status: "downloaded",
|
||||
bytes: result.bytes,
|
||||
snapshot: result.snapshotUrl,
|
||||
timestamp: result.timestamp,
|
||||
});
|
||||
bar.tick("ok");
|
||||
} else if (result.status === "no_snapshot") {
|
||||
failures.push({ url, reason: "no_snapshot" });
|
||||
await record(manifestPath, { url, status: "no_snapshot" });
|
||||
bar.tick("failed");
|
||||
} else {
|
||||
failures.push({ url, reason: result.error });
|
||||
await record(manifestPath, { url, status: "error", error: result.error });
|
||||
bar.tick("failed");
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
bar.finish();
|
||||
|
||||
const counters = bar.snapshot();
|
||||
const summary = buildSummary({
|
||||
totalFound: work.length,
|
||||
attachmentGuids: attachmentGuids.size,
|
||||
ok: counters.ok,
|
||||
skipped: counters.skipped,
|
||||
failed: counters.failed,
|
||||
bytes: totalBytes,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
outputDir: opts.output,
|
||||
manifestPath,
|
||||
failures,
|
||||
});
|
||||
console.error(summary);
|
||||
|
||||
if (counters.failed > 0) process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function record(path: string | undefined, entry: object): Promise<void> {
|
||||
if (!path) return;
|
||||
await appendFile(path, JSON.stringify(entry) + "\n");
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { createInterface } from "node:readline";
|
||||
import { cleanUrl, matchesHost, normalizeHost } from "./utils/url.js";
|
||||
import { extractAttachmentGuids } from "./utils/sql.js";
|
||||
import { buildMediaUrlRegex, DEFAULT_MEDIA_EXTENSIONS } from "./utils/media.js";
|
||||
|
||||
export interface ParseOptions {
|
||||
/** Restrict URLs to this site (host or scheme://host). If omitted, accept all. */
|
||||
site?: string;
|
||||
/** Override the default media extension list. */
|
||||
extensions?: readonly string[];
|
||||
}
|
||||
|
||||
export interface ParseResult {
|
||||
urls: Set<string>;
|
||||
attachmentGuids: Set<string>;
|
||||
}
|
||||
|
||||
export async function parseSqlDump(
|
||||
filePath: string,
|
||||
options: ParseOptions = {},
|
||||
): Promise<ParseResult> {
|
||||
const urls = new Set<string>();
|
||||
const attachmentGuids = new Set<string>();
|
||||
|
||||
const exts = options.extensions ?? DEFAULT_MEDIA_EXTENSIONS;
|
||||
const urlRe = buildMediaUrlRegex(exts);
|
||||
const hostFilter = normalizeHost(options.site);
|
||||
|
||||
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
|
||||
for await (const line of rl) {
|
||||
const matches = line.match(urlRe);
|
||||
if (matches) {
|
||||
for (const m of matches) {
|
||||
const cleaned = cleanUrl(m);
|
||||
if (matchesHost(cleaned, hostFilter)) urls.add(cleaned);
|
||||
}
|
||||
}
|
||||
|
||||
if (line.includes("'attachment'") && line.includes("INSERT INTO")) {
|
||||
for (const guid of extractAttachmentGuids(line)) {
|
||||
if (matchesHost(guid, hostFilter)) {
|
||||
attachmentGuids.add(guid);
|
||||
urls.add(guid);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { urls, attachmentGuids };
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export const DEFAULT_MEDIA_EXTENSIONS = [
|
||||
"jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "ico",
|
||||
"mp3", "wav", "ogg", "m4a", "flac",
|
||||
"mp4", "webm", "mov", "avi", "mkv",
|
||||
"pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
|
||||
"zip", "rar", "7z", "tar", "gz",
|
||||
] as const;
|
||||
|
||||
export function buildMediaUrlRegex(extensions: readonly string[]): RegExp {
|
||||
if (extensions.length === 0) {
|
||||
throw new Error("at least one extension is required");
|
||||
}
|
||||
const escaped = extensions.map(escapeRegex).join("|");
|
||||
return new RegExp(
|
||||
`https?:\\/\\/[^\\s'"<>()\\\\]+\\.(?:${escaped})(?:\\?[^\\s'"<>()\\\\]*)?`,
|
||||
"gi",
|
||||
);
|
||||
}
|
||||
|
||||
function escapeRegex(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
export interface ProgressCounters {
|
||||
ok: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
export interface ProgressBarOptions {
|
||||
total: number;
|
||||
width?: number;
|
||||
stream?: NodeJS.WriteStream;
|
||||
/** Force on/off; defaults to stream.isTTY. */
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal terminal progress bar. Designed so the same instance can be used in
|
||||
* tests by passing a non-TTY stream or `enabled: false` (in which case render
|
||||
* is a no-op and `format()` returns the string for assertions).
|
||||
*/
|
||||
export class ProgressBar {
|
||||
private done = 0;
|
||||
private readonly counters: ProgressCounters = { ok: 0, skipped: 0, failed: 0 };
|
||||
private readonly stream: NodeJS.WriteStream;
|
||||
private readonly width: number;
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(private readonly opts: ProgressBarOptions) {
|
||||
this.stream = opts.stream ?? process.stderr;
|
||||
this.width = opts.width ?? 30;
|
||||
this.enabled = opts.enabled ?? Boolean(this.stream.isTTY);
|
||||
}
|
||||
|
||||
tick(kind: keyof ProgressCounters): void {
|
||||
this.counters[kind] += 1;
|
||||
this.done += 1;
|
||||
this.render();
|
||||
}
|
||||
|
||||
/** Build the displayable line without writing to the stream. */
|
||||
format(): string {
|
||||
const total = Math.max(this.opts.total, 1);
|
||||
const ratio = Math.min(this.done / total, 1);
|
||||
const filled = Math.round(this.width * ratio);
|
||||
const bar = "█".repeat(filled) + "░".repeat(this.width - filled);
|
||||
const pct = String(Math.round(ratio * 100)).padStart(3, " ");
|
||||
const { ok, skipped, failed } = this.counters;
|
||||
return `[${bar}] ${pct}% ${this.done}/${this.opts.total} ok:${ok} skip:${skipped} fail:${failed}`;
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (!this.enabled) return;
|
||||
this.stream.write("\n");
|
||||
}
|
||||
|
||||
snapshot(): ProgressCounters & { done: number } {
|
||||
return { ...this.counters, done: this.done };
|
||||
}
|
||||
|
||||
private render(): void {
|
||||
if (!this.enabled) return;
|
||||
this.stream.write(`\r${this.format()}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
export interface SliceOptions {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a comma-separated list of file extensions into a normalized lowercase
|
||||
* set without leading dots. Returns undefined for missing/blank input.
|
||||
*/
|
||||
export function parseExtensionsOption(
|
||||
value: string | undefined,
|
||||
): Set<string> | undefined {
|
||||
if (value === undefined || value === "") return undefined;
|
||||
const exts = value
|
||||
.split(",")
|
||||
.map((e) => e.trim().toLowerCase().replace(/^\.+/, ""))
|
||||
.filter((e) => e.length > 0);
|
||||
if (exts.length === 0) return undefined;
|
||||
return new Set(exts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only URLs whose pathname ends with one of the given extensions
|
||||
* (case-insensitive, query/fragment stripped). Malformed URLs fall back to
|
||||
* a string suffix check so we don't silently drop them.
|
||||
*/
|
||||
export function filterUrlsByExtensions(
|
||||
urls: Iterable<string>,
|
||||
extensions: Set<string> | undefined,
|
||||
): string[] {
|
||||
const arr = [...urls];
|
||||
if (!extensions || extensions.size === 0) return arr;
|
||||
return arr.filter((u) => {
|
||||
let path = u;
|
||||
try {
|
||||
path = new URL(u).pathname;
|
||||
} catch {
|
||||
const q = u.indexOf("?");
|
||||
const h = u.indexOf("#");
|
||||
const cut = [q, h].filter((i) => i >= 0).sort((a, b) => a - b)[0];
|
||||
if (cut !== undefined) path = u.slice(0, cut);
|
||||
}
|
||||
const dot = path.lastIndexOf(".");
|
||||
if (dot < 0) return false;
|
||||
return extensions.has(path.slice(dot + 1).toLowerCase());
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply offset/limit to an iterable of URLs. Sorts deterministically first
|
||||
* so re-runs with the same dump and the same `--offset`/`--limit` always pick
|
||||
* the same window — important when chunking a large recovery across runs.
|
||||
*
|
||||
* `offset` and `limit` are treated as non-negative; negatives are clamped to 0
|
||||
* (limit < 0 means "no limit").
|
||||
*/
|
||||
export function sliceUrls(
|
||||
urls: Iterable<string>,
|
||||
{ offset = 0, limit }: SliceOptions = {},
|
||||
): string[] {
|
||||
const sorted = [...urls].sort();
|
||||
const start = Math.max(0, Math.floor(offset));
|
||||
if (limit === undefined) return sorted.slice(start);
|
||||
const n = Math.floor(limit);
|
||||
if (n < 0) return sorted.slice(start);
|
||||
return sorted.slice(start, start + n);
|
||||
}
|
||||
|
||||
/** Parse a CLI integer option, returning undefined for missing/blank input. */
|
||||
export function parseIntOption(
|
||||
value: string | undefined,
|
||||
optionName: string,
|
||||
): number | undefined {
|
||||
if (value === undefined || value === "") return undefined;
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n) || !Number.isInteger(n)) {
|
||||
throw new Error(`${optionName} must be an integer, got "${value}"`);
|
||||
}
|
||||
if (n < 0) {
|
||||
throw new Error(`${optionName} must be >= 0, got ${n}`);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Build a Wayback "raw asset" URL that resolves to the closest capture of
|
||||
* `targetUrl` to `timestamp`, server-side. The `id_` flag asks for the original
|
||||
* bytes instead of the rewritten archive page.
|
||||
*
|
||||
* https://web.archive.org/web/<TIMESTAMP>id_/<TARGET>
|
||||
*
|
||||
* `timestamp` may be a full 14-digit string (YYYYMMDDhhmmss) or any prefix
|
||||
* (e.g. "2020", "202003"). If omitted we default to "2", which means "any
|
||||
* capture from the year 2000 onward" — broad enough to cover every WordPress
|
||||
* upload while still being a valid prefix (an empty timestamp redirects to
|
||||
* the archive homepage).
|
||||
*/
|
||||
export function buildSnapshotUrl(targetUrl: string, timestamp?: string): string {
|
||||
const ts = normalizeTimestamp(timestamp);
|
||||
return `https://web.archive.org/web/${ts}id_/${targetUrl}`;
|
||||
}
|
||||
|
||||
export function normalizeTimestamp(ts?: string): string {
|
||||
if (!ts) return "2";
|
||||
const cleaned = ts.replace(/[^0-9]/g, "");
|
||||
return cleaned.length > 0 ? cleaned : "2";
|
||||
}
|
||||
|
||||
const TS_RE = /\/web\/(\d{1,14})(?:id_)?\//;
|
||||
|
||||
/** Pull the resolved 14-digit timestamp out of an effective Wayback URL after redirects. */
|
||||
export function extractTimestamp(snapshotUrl: string): string | undefined {
|
||||
const m = snapshotUrl.match(TS_RE);
|
||||
return m ? m[1] : undefined;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { cleanUrl } from "./url.js";
|
||||
|
||||
/** Split an INSERT line into individual VALUES tuples, respecting SQL string escaping. */
|
||||
export function splitTuples(line: string): string[] {
|
||||
const tuples: string[] = [];
|
||||
let depth = 0;
|
||||
let start = -1;
|
||||
let inStr = false;
|
||||
let escaped = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const c = line[i];
|
||||
if (inStr) {
|
||||
if (escaped) { escaped = false; continue; }
|
||||
if (c === "\\") { escaped = true; continue; }
|
||||
if (c === "'") inStr = false;
|
||||
continue;
|
||||
}
|
||||
if (c === "'") { inStr = true; continue; }
|
||||
if (c === "(") {
|
||||
if (depth === 0) start = i + 1;
|
||||
depth++;
|
||||
} else if (c === ")") {
|
||||
depth--;
|
||||
if (depth === 0 && start >= 0) {
|
||||
tuples.push(line.slice(start, i));
|
||||
start = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return tuples;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an `INSERT INTO wp_posts ...` line and pull guid values for attachment rows.
|
||||
* Column order can vary across exports, so we look for the first URL-shaped string
|
||||
* inside any tuple that also contains the literal 'attachment'.
|
||||
*/
|
||||
export function extractAttachmentGuids(line: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const tuple of splitTuples(line)) {
|
||||
if (!/'attachment'/.test(tuple)) continue;
|
||||
const urlMatch = tuple.match(/'(https?:\/\/[^']+)'/);
|
||||
if (urlMatch) out.push(cleanUrl(urlMatch[1]));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
export interface FailureEntry {
|
||||
url: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface SummaryInput {
|
||||
totalFound: number;
|
||||
attachmentGuids: number;
|
||||
ok: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
bytes: number;
|
||||
elapsedMs: number;
|
||||
outputDir: string;
|
||||
manifestPath?: string;
|
||||
failures?: FailureEntry[];
|
||||
/** Cap how many failure URLs to print. Default: 10. */
|
||||
maxFailuresShown?: number;
|
||||
}
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
const units = ["KB", "MB", "GB", "TB"];
|
||||
let value = bytes / 1024;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${value.toFixed(2)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
export function formatDuration(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
const totalSec = Math.round(ms / 1000);
|
||||
const h = Math.floor(totalSec / 3600);
|
||||
const m = Math.floor((totalSec % 3600) / 60);
|
||||
const s = totalSec % 60;
|
||||
if (h > 0) return `${h}h${m}m${s}s`;
|
||||
if (m > 0) return `${m}m${s}s`;
|
||||
return `${s}s`;
|
||||
}
|
||||
|
||||
export function buildSummary(input: SummaryInput): string {
|
||||
const max = input.maxFailuresShown ?? 10;
|
||||
const total = input.ok + input.skipped + input.failed;
|
||||
const successRate =
|
||||
total > 0 ? `${Math.round((input.ok / total) * 100)}%` : "n/a";
|
||||
|
||||
const lines = [
|
||||
"",
|
||||
"─── Recovery summary ─────────────────────────",
|
||||
` URLs found in dump : ${input.totalFound} (${input.attachmentGuids} attachment GUIDs)`,
|
||||
` Processed : ${total}`,
|
||||
` ✓ Downloaded : ${input.ok}`,
|
||||
` ↷ Skipped (exists) : ${input.skipped}`,
|
||||
` ✗ Failed : ${input.failed}`,
|
||||
` Success rate : ${successRate}`,
|
||||
` Total downloaded : ${formatBytes(input.bytes)}`,
|
||||
` Elapsed : ${formatDuration(input.elapsedMs)}`,
|
||||
` Output directory : ${input.outputDir}`,
|
||||
];
|
||||
|
||||
if (input.manifestPath) {
|
||||
lines.push(` Manifest (JSONL) : ${input.manifestPath}`);
|
||||
}
|
||||
|
||||
const failures = input.failures ?? [];
|
||||
if (failures.length > 0) {
|
||||
lines.push("", " Failures:");
|
||||
for (const f of failures.slice(0, max)) {
|
||||
lines.push(` - ${f.url} (${f.reason})`);
|
||||
}
|
||||
if (failures.length > max) {
|
||||
lines.push(` … and ${failures.length - max} more`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("──────────────────────────────────────────────", "");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/** Strip trailing punctuation that often glues onto URLs in SQL strings. */
|
||||
export function cleanUrl(u: string): string {
|
||||
return u.replace(/[),;.'"\\]+$/g, "");
|
||||
}
|
||||
|
||||
/** Normalize a site argument ("example.com", "https://example.com/") to a bare host. */
|
||||
export function normalizeHost(site?: string): string | null {
|
||||
if (!site) return null;
|
||||
try {
|
||||
const url = new URL(site.includes("://") ? site : `https://${site}`);
|
||||
return url.host.toLowerCase();
|
||||
} catch {
|
||||
return site.toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
/** Does `url` belong to `host`? When host is null, every URL passes. */
|
||||
export function matchesHost(url: string, host: string | null): boolean {
|
||||
if (!host) return true;
|
||||
try {
|
||||
return new URL(url).host.toLowerCase() === host;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { join } from "node:path";
|
||||
|
||||
export interface UrlToPathOptions {
|
||||
/** Prefix the output path with the URL host. Default: true. */
|
||||
hostPrefix?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map an original URL to a path on disk under outputDir, preserving the
|
||||
* original site directory structure (e.g. wp-content/uploads/2020/01/foo.jpg).
|
||||
*/
|
||||
export function urlToOutputPath(
|
||||
targetUrl: string,
|
||||
outputDir: string,
|
||||
options: UrlToPathOptions = {},
|
||||
): string {
|
||||
const u = new URL(targetUrl);
|
||||
const hostPrefix = options.hostPrefix ?? true;
|
||||
const pathname = decodeURIComponent(u.pathname);
|
||||
const segments = pathname.split("/").map(sanitizeSegment).filter(Boolean);
|
||||
if (segments.length === 0) segments.push("index");
|
||||
const prefix = hostPrefix ? [sanitizeSegment(u.host)] : [];
|
||||
return join(outputDir, ...prefix, ...segments);
|
||||
}
|
||||
|
||||
/** Strip path traversal and characters not safe across filesystems. */
|
||||
function sanitizeSegment(s: string): string {
|
||||
return s
|
||||
.replace(/\\/g, "_")
|
||||
.replace(/[<>:"|?*\x00-\x1f]/g, "_")
|
||||
.replace(/^\.+$/, "_");
|
||||
}
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
import { mkdir, rename, stat, unlink } from "node:fs/promises";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { buildSnapshotUrl, extractTimestamp } from "./utils/snapshot.js";
|
||||
|
||||
const UA = "wp-media-rewind/0.1 (+https://archive.org)";
|
||||
|
||||
export interface RecoverOptions {
|
||||
timestamp?: string;
|
||||
signal?: AbortSignal;
|
||||
/** Number of retry attempts on 5xx / network errors. Default: 2. */
|
||||
retries?: number;
|
||||
}
|
||||
|
||||
export type RecoverResult =
|
||||
| { status: "downloaded"; bytes: number; snapshotUrl: string; timestamp?: string }
|
||||
| { status: "no_snapshot" }
|
||||
| { status: "error"; error: string };
|
||||
|
||||
/**
|
||||
* Pull `targetUrl` from the Wayback Machine into `destPath`. Uses the
|
||||
* `web.archive.org/web/<ts>id_/<url>` form so the server picks the closest
|
||||
* capture and we get raw bytes (not a rewritten archive page).
|
||||
*/
|
||||
export async function recover(
|
||||
targetUrl: string,
|
||||
destPath: string,
|
||||
opts: RecoverOptions = {},
|
||||
): Promise<RecoverResult> {
|
||||
const url = buildSnapshotUrl(targetUrl, opts.timestamp);
|
||||
const retries = opts.retries ?? 2;
|
||||
|
||||
let lastError = "";
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": UA },
|
||||
redirect: "follow",
|
||||
signal: opts.signal,
|
||||
});
|
||||
|
||||
if (res.status === 404) {
|
||||
await drain(res);
|
||||
return { status: "no_snapshot" };
|
||||
}
|
||||
|
||||
if (res.status >= 500 || res.status === 429) {
|
||||
await drain(res);
|
||||
lastError = `http ${res.status}`;
|
||||
await backoff(attempt, opts.signal);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!res.ok || !res.body) {
|
||||
await drain(res);
|
||||
return { status: "error", error: `http ${res.status}` };
|
||||
}
|
||||
|
||||
// Sanity check: the archive sometimes serves an HTML "calendar" page even on
|
||||
// 200 when it can't find a real capture. Detect that and treat as no_snapshot.
|
||||
const contentType = res.headers.get("content-type") ?? "";
|
||||
const effective = res.url || url;
|
||||
if (looksLikeCalendar(effective, contentType, targetUrl)) {
|
||||
await drain(res);
|
||||
return { status: "no_snapshot" };
|
||||
}
|
||||
|
||||
await mkdir(dirname(destPath), { recursive: true });
|
||||
const tempPath = `${destPath}.part`;
|
||||
try {
|
||||
await pipeline(Readable.fromWeb(res.body as any), createWriteStream(tempPath));
|
||||
await rename(tempPath, destPath);
|
||||
} catch (err) {
|
||||
await unlink(tempPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
const s = await stat(destPath);
|
||||
return {
|
||||
status: "downloaded",
|
||||
bytes: s.size,
|
||||
snapshotUrl: effective,
|
||||
timestamp: extractTimestamp(effective),
|
||||
};
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
if (attempt < retries) await backoff(attempt, opts.signal);
|
||||
}
|
||||
}
|
||||
|
||||
return { status: "error", error: lastError || "unknown error" };
|
||||
}
|
||||
|
||||
function looksLikeCalendar(
|
||||
effectiveUrl: string,
|
||||
contentType: string,
|
||||
targetUrl: string,
|
||||
): boolean {
|
||||
// Pattern: archive.org redirected to /web/<ts>/<target> without id_, or to a
|
||||
// wildcard calendar page like /web/*/<target>, and served HTML.
|
||||
if (!contentType.toLowerCase().includes("html")) return false;
|
||||
if (/\/web\/\*\//.test(effectiveUrl)) return true;
|
||||
// If we asked for image/pdf/etc. and got HTML back, it's the calendar.
|
||||
if (/\.(jpg|jpeg|png|gif|webp|bmp|svg|ico|mp3|mp4|webm|pdf|zip)(\?|$)/i.test(targetUrl)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function drain(res: Response): Promise<void> {
|
||||
try {
|
||||
if (res.body) await res.body.cancel();
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
async function backoff(attempt: number, signal?: AbortSignal): Promise<void> {
|
||||
const ms = Math.min(2_000 * 2 ** attempt, 10_000);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const t = setTimeout(resolve, ms);
|
||||
if (signal) {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(t);
|
||||
reject(new Error("aborted"));
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export async function fileExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { buildMediaUrlRegex, DEFAULT_MEDIA_EXTENSIONS } from "../src/utils/media.js";
|
||||
|
||||
test("default regex matches typical WordPress upload URLs", () => {
|
||||
const re = buildMediaUrlRegex(DEFAULT_MEDIA_EXTENSIONS);
|
||||
const text =
|
||||
"see <img src=\"https://site.test/wp-content/uploads/2020/01/photo.jpg\"/> and " +
|
||||
"https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2 and ignore https://site.test/page";
|
||||
const matches = text.match(re) ?? [];
|
||||
assert.deepEqual(matches, [
|
||||
"https://site.test/wp-content/uploads/2020/01/photo.jpg",
|
||||
"https://site.test/wp-content/uploads/2020/01/doc.pdf?v=2",
|
||||
]);
|
||||
});
|
||||
|
||||
test("regex is case-insensitive on extension", () => {
|
||||
const re = buildMediaUrlRegex(["jpg"]);
|
||||
assert.ok(re.test("https://x.test/a.JPG"));
|
||||
});
|
||||
|
||||
test("regex respects custom extensions only", () => {
|
||||
const re = buildMediaUrlRegex(["xyz"]);
|
||||
assert.equal("https://x.test/a.jpg".match(re), null);
|
||||
assert.ok(re.test("https://x.test/a.xyz"));
|
||||
});
|
||||
|
||||
test("buildMediaUrlRegex throws on empty list", () => {
|
||||
assert.throws(() => buildMediaUrlRegex([]), /at least one extension/);
|
||||
});
|
||||
|
||||
test("regex stops at quotes and whitespace", () => {
|
||||
const re = buildMediaUrlRegex(["png"]);
|
||||
const text = `'https://x.test/a.png' "https://x.test/b.png"`;
|
||||
const matches = text.match(re) ?? [];
|
||||
assert.deepEqual(matches, ["https://x.test/a.png", "https://x.test/b.png"]);
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { ProgressBar } from "../src/utils/progress.js";
|
||||
|
||||
test("format reflects ticks", () => {
|
||||
const bar = new ProgressBar({ total: 4, width: 10, enabled: false });
|
||||
bar.tick("ok");
|
||||
bar.tick("skipped");
|
||||
const line = bar.format();
|
||||
assert.match(line, /50%/);
|
||||
assert.match(line, /2\/4/);
|
||||
assert.match(line, /ok:1/);
|
||||
assert.match(line, /skip:1/);
|
||||
assert.match(line, /fail:0/);
|
||||
});
|
||||
|
||||
test("snapshot returns running totals", () => {
|
||||
const bar = new ProgressBar({ total: 3, enabled: false });
|
||||
bar.tick("ok");
|
||||
bar.tick("failed");
|
||||
assert.deepEqual(bar.snapshot(), { ok: 1, skipped: 0, failed: 1, done: 2 });
|
||||
});
|
||||
|
||||
test("format reaches 100% on completion", () => {
|
||||
const bar = new ProgressBar({ total: 2, width: 4, enabled: false });
|
||||
bar.tick("ok");
|
||||
bar.tick("ok");
|
||||
assert.match(bar.format(), /100%/);
|
||||
});
|
||||
|
||||
test("disabled bar does not write to stream", () => {
|
||||
let written = "";
|
||||
const fakeStream = {
|
||||
isTTY: false,
|
||||
write: (s: string) => {
|
||||
written += s;
|
||||
return true;
|
||||
},
|
||||
} as unknown as NodeJS.WriteStream;
|
||||
const bar = new ProgressBar({ total: 1, stream: fakeStream });
|
||||
bar.tick("ok");
|
||||
bar.finish();
|
||||
assert.equal(written, "");
|
||||
});
|
||||
|
||||
test("enabled bar writes carriage-return updates", () => {
|
||||
let written = "";
|
||||
const fakeStream = {
|
||||
isTTY: true,
|
||||
write: (s: string) => {
|
||||
written += s;
|
||||
return true;
|
||||
},
|
||||
} as unknown as NodeJS.WriteStream;
|
||||
const bar = new ProgressBar({ total: 2, width: 4, stream: fakeStream });
|
||||
bar.tick("ok");
|
||||
bar.tick("ok");
|
||||
bar.finish();
|
||||
assert.ok(written.startsWith("\r"));
|
||||
assert.ok(written.endsWith("\n"));
|
||||
assert.match(written, /100%/);
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
filterUrlsByExtensions,
|
||||
parseExtensionsOption,
|
||||
parseIntOption,
|
||||
sliceUrls,
|
||||
} from "../src/utils/slice.js";
|
||||
|
||||
const sample = [
|
||||
"https://x.test/c.jpg",
|
||||
"https://x.test/a.jpg",
|
||||
"https://x.test/b.jpg",
|
||||
"https://x.test/e.jpg",
|
||||
"https://x.test/d.jpg",
|
||||
];
|
||||
|
||||
test("sliceUrls sorts deterministically with no slice options", () => {
|
||||
assert.deepEqual(sliceUrls(sample), [
|
||||
"https://x.test/a.jpg",
|
||||
"https://x.test/b.jpg",
|
||||
"https://x.test/c.jpg",
|
||||
"https://x.test/d.jpg",
|
||||
"https://x.test/e.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sliceUrls applies offset only", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { offset: 2 }), [
|
||||
"https://x.test/c.jpg",
|
||||
"https://x.test/d.jpg",
|
||||
"https://x.test/e.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sliceUrls applies limit only", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { limit: 2 }), [
|
||||
"https://x.test/a.jpg",
|
||||
"https://x.test/b.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sliceUrls applies offset and limit together", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { offset: 1, limit: 2 }), [
|
||||
"https://x.test/b.jpg",
|
||||
"https://x.test/c.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sliceUrls returns [] when offset exceeds length", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { offset: 99 }), []);
|
||||
});
|
||||
|
||||
test("sliceUrls works with a Set", () => {
|
||||
const set = new Set(sample);
|
||||
assert.deepEqual(sliceUrls(set, { limit: 1 }), ["https://x.test/a.jpg"]);
|
||||
});
|
||||
|
||||
test("sliceUrls clamps negative offset to 0", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { offset: -5, limit: 1 }), [
|
||||
"https://x.test/a.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("sliceUrls treats limit=0 as empty", () => {
|
||||
assert.deepEqual(sliceUrls(sample, { limit: 0 }), []);
|
||||
});
|
||||
|
||||
test("parseIntOption returns undefined when missing", () => {
|
||||
assert.equal(parseIntOption(undefined, "--limit"), undefined);
|
||||
assert.equal(parseIntOption("", "--limit"), undefined);
|
||||
});
|
||||
|
||||
test("parseIntOption parses integers", () => {
|
||||
assert.equal(parseIntOption("0", "--offset"), 0);
|
||||
assert.equal(parseIntOption("42", "--limit"), 42);
|
||||
});
|
||||
|
||||
test("parseIntOption rejects non-integers", () => {
|
||||
assert.throws(() => parseIntOption("3.14", "--limit"), /must be an integer/);
|
||||
assert.throws(() => parseIntOption("abc", "--offset"), /must be an integer/);
|
||||
});
|
||||
|
||||
test("parseIntOption rejects negatives", () => {
|
||||
assert.throws(() => parseIntOption("-1", "--limit"), /must be >= 0/);
|
||||
});
|
||||
|
||||
test("parseExtensionsOption returns undefined for missing/blank", () => {
|
||||
assert.equal(parseExtensionsOption(undefined), undefined);
|
||||
assert.equal(parseExtensionsOption(""), undefined);
|
||||
assert.equal(parseExtensionsOption(" , , "), undefined);
|
||||
});
|
||||
|
||||
test("parseExtensionsOption normalizes case, dots, and whitespace", () => {
|
||||
const set = parseExtensionsOption(" .JPG, png ,.WebP");
|
||||
assert.deepEqual([...(set ?? [])].sort(), ["jpg", "png", "webp"]);
|
||||
});
|
||||
|
||||
test("filterUrlsByExtensions returns input unchanged when no filter", () => {
|
||||
const urls = ["https://x.test/a.jpg", "https://x.test/b.png"];
|
||||
assert.deepEqual(filterUrlsByExtensions(urls, undefined), urls);
|
||||
});
|
||||
|
||||
test("filterUrlsByExtensions keeps only matching extensions", () => {
|
||||
const urls = [
|
||||
"https://x.test/a.jpg",
|
||||
"https://x.test/b.png",
|
||||
"https://x.test/c.JPG",
|
||||
"https://x.test/d.gif",
|
||||
];
|
||||
const set = parseExtensionsOption("jpg");
|
||||
assert.deepEqual(filterUrlsByExtensions(urls, set), [
|
||||
"https://x.test/a.jpg",
|
||||
"https://x.test/c.JPG",
|
||||
]);
|
||||
});
|
||||
|
||||
test("filterUrlsByExtensions ignores query string and fragment", () => {
|
||||
const urls = [
|
||||
"https://x.test/a.jpg?v=2",
|
||||
"https://x.test/b.png#frag",
|
||||
"https://x.test/c.gif?x=1#y",
|
||||
];
|
||||
const set = parseExtensionsOption("jpg,png");
|
||||
assert.deepEqual(filterUrlsByExtensions(urls, set), [
|
||||
"https://x.test/a.jpg?v=2",
|
||||
"https://x.test/b.png#frag",
|
||||
]);
|
||||
});
|
||||
|
||||
test("filterUrlsByExtensions drops URLs without an extension", () => {
|
||||
const urls = ["https://x.test/no-ext", "https://x.test/a.jpg"];
|
||||
const set = parseExtensionsOption("jpg");
|
||||
assert.deepEqual(filterUrlsByExtensions(urls, set), ["https://x.test/a.jpg"]);
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
buildSnapshotUrl,
|
||||
extractTimestamp,
|
||||
normalizeTimestamp,
|
||||
} from "../src/utils/snapshot.js";
|
||||
|
||||
test("buildSnapshotUrl uses '2' as default timestamp prefix", () => {
|
||||
assert.equal(
|
||||
buildSnapshotUrl("https://example.com/a.jpg"),
|
||||
"https://web.archive.org/web/2id_/https://example.com/a.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("buildSnapshotUrl honors a full timestamp", () => {
|
||||
assert.equal(
|
||||
buildSnapshotUrl("https://example.com/a.jpg", "20200101000000"),
|
||||
"https://web.archive.org/web/20200101000000id_/https://example.com/a.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("buildSnapshotUrl honors a partial timestamp", () => {
|
||||
assert.equal(
|
||||
buildSnapshotUrl("https://example.com/a.jpg", "2020"),
|
||||
"https://web.archive.org/web/2020id_/https://example.com/a.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("normalizeTimestamp strips non-digits", () => {
|
||||
assert.equal(normalizeTimestamp("2020-01-01"), "20200101");
|
||||
assert.equal(normalizeTimestamp("2020/01"), "202001");
|
||||
});
|
||||
|
||||
test("normalizeTimestamp falls back to '2' on empty input", () => {
|
||||
assert.equal(normalizeTimestamp(undefined), "2");
|
||||
assert.equal(normalizeTimestamp(""), "2");
|
||||
assert.equal(normalizeTimestamp("---"), "2");
|
||||
});
|
||||
|
||||
test("extractTimestamp pulls timestamp from a resolved snapshot URL", () => {
|
||||
assert.equal(
|
||||
extractTimestamp(
|
||||
"https://web.archive.org/web/20221020185708id_/https://x.test/a.jpg",
|
||||
),
|
||||
"20221020185708",
|
||||
);
|
||||
});
|
||||
|
||||
test("extractTimestamp works without id_ marker", () => {
|
||||
assert.equal(
|
||||
extractTimestamp("https://web.archive.org/web/20221020185708/https://x.test/a.jpg"),
|
||||
"20221020185708",
|
||||
);
|
||||
});
|
||||
|
||||
test("extractTimestamp returns undefined for non-wayback URLs", () => {
|
||||
assert.equal(extractTimestamp("https://example.com/a.jpg"), undefined);
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { extractAttachmentGuids, splitTuples } from "../src/utils/sql.js";
|
||||
|
||||
test("splitTuples splits flat tuples", () => {
|
||||
const line = "INSERT INTO `t` VALUES (1,'a'),(2,'b'),(3,'c');";
|
||||
assert.deepEqual(splitTuples(line), ["1,'a'", "2,'b'", "3,'c'"]);
|
||||
});
|
||||
|
||||
test("splitTuples ignores parens inside quoted strings", () => {
|
||||
const line = "INSERT INTO `t` VALUES (1,'foo (bar) baz'),(2,'x');";
|
||||
assert.deepEqual(splitTuples(line), ["1,'foo (bar) baz'", "2,'x'"]);
|
||||
});
|
||||
|
||||
test("splitTuples handles escaped quotes", () => {
|
||||
const line = "INSERT INTO `t` VALUES (1,'it\\'s ok'),(2,'y');";
|
||||
const tuples = splitTuples(line);
|
||||
assert.equal(tuples.length, 2);
|
||||
assert.equal(tuples[0], "1,'it\\'s ok'");
|
||||
});
|
||||
|
||||
test("extractAttachmentGuids returns URLs from attachment rows", () => {
|
||||
const line =
|
||||
"INSERT INTO `wp_posts` VALUES " +
|
||||
"(1,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/wp-content/uploads/2020/01/foo.jpg',0,'attachment','image/jpeg',0)," +
|
||||
"(2,1,'2020-01-01','','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);";
|
||||
assert.deepEqual(extractAttachmentGuids(line), [
|
||||
"https://site.test/wp-content/uploads/2020/01/foo.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
test("extractAttachmentGuids returns [] when no attachment rows", () => {
|
||||
const line =
|
||||
"INSERT INTO `wp_posts` VALUES (2,1,'2020-01-01','c','t','','publish','open','open','','slug','','','2020-01-01','2020-01-01','',0,'https://site.test/?p=2',0,'post','',0);";
|
||||
assert.deepEqual(extractAttachmentGuids(line), []);
|
||||
});
|
||||
|
||||
test("extractAttachmentGuids handles multiple attachment rows", () => {
|
||||
const line =
|
||||
"INSERT INTO `wp_posts` VALUES " +
|
||||
"(1,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/a.jpg',0,'attachment','image/jpeg',0)," +
|
||||
"(2,1,'d','','c','t','','publish','open','open','','s','','','d','d','',0,'https://site.test/b.png',0,'attachment','image/png',0);";
|
||||
assert.deepEqual(extractAttachmentGuids(line), [
|
||||
"https://site.test/a.jpg",
|
||||
"https://site.test/b.png",
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { buildSummary, formatBytes, formatDuration } from "../src/utils/summary.js";
|
||||
|
||||
test("formatBytes scales units", () => {
|
||||
assert.equal(formatBytes(0), "0 B");
|
||||
assert.equal(formatBytes(512), "512 B");
|
||||
assert.equal(formatBytes(2048), "2.00 KB");
|
||||
assert.equal(formatBytes(5 * 1024 * 1024), "5.00 MB");
|
||||
assert.equal(formatBytes(3 * 1024 ** 3), "3.00 GB");
|
||||
});
|
||||
|
||||
test("formatDuration formats ranges", () => {
|
||||
assert.equal(formatDuration(250), "250ms");
|
||||
assert.equal(formatDuration(1500), "2s");
|
||||
assert.equal(formatDuration(65_000), "1m5s");
|
||||
assert.equal(formatDuration(3_725_000), "1h2m5s");
|
||||
});
|
||||
|
||||
test("buildSummary includes counters and rate", () => {
|
||||
const out = buildSummary({
|
||||
totalFound: 100,
|
||||
attachmentGuids: 40,
|
||||
ok: 80,
|
||||
skipped: 10,
|
||||
failed: 10,
|
||||
bytes: 1024,
|
||||
elapsedMs: 5000,
|
||||
outputDir: "/tmp/out",
|
||||
});
|
||||
assert.match(out, /URLs found in dump : 100 \(40 attachment GUIDs\)/);
|
||||
assert.match(out, /✓ Downloaded\s*: 80/);
|
||||
assert.match(out, /↷ Skipped\s*\(exists\)\s*: 10/);
|
||||
assert.match(out, /✗ Failed\s*: 10/);
|
||||
assert.match(out, /Success rate\s*: 80%/);
|
||||
assert.match(out, /1\.00 KB/);
|
||||
assert.match(out, /5s/);
|
||||
assert.match(out, /\/tmp\/out/);
|
||||
});
|
||||
|
||||
test("buildSummary lists failures with truncation", () => {
|
||||
const failures = Array.from({ length: 12 }, (_, i) => ({
|
||||
url: `https://x.test/f${i}.jpg`,
|
||||
reason: "no_snapshot",
|
||||
}));
|
||||
const out = buildSummary({
|
||||
totalFound: 12,
|
||||
attachmentGuids: 0,
|
||||
ok: 0,
|
||||
skipped: 0,
|
||||
failed: 12,
|
||||
bytes: 0,
|
||||
elapsedMs: 100,
|
||||
outputDir: "/o",
|
||||
failures,
|
||||
maxFailuresShown: 5,
|
||||
});
|
||||
assert.match(out, /f0\.jpg/);
|
||||
assert.match(out, /f4\.jpg/);
|
||||
assert.ok(!out.includes("f5.jpg"));
|
||||
assert.match(out, /and 7 more/);
|
||||
});
|
||||
|
||||
test("buildSummary omits manifest line when not provided", () => {
|
||||
const out = buildSummary({
|
||||
totalFound: 1,
|
||||
attachmentGuids: 0,
|
||||
ok: 1,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
bytes: 100,
|
||||
elapsedMs: 10,
|
||||
outputDir: "/o",
|
||||
});
|
||||
assert.ok(!out.includes("Manifest"));
|
||||
});
|
||||
|
||||
test("buildSummary handles 0 processed", () => {
|
||||
const out = buildSummary({
|
||||
totalFound: 0,
|
||||
attachmentGuids: 0,
|
||||
ok: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
bytes: 0,
|
||||
elapsedMs: 5,
|
||||
outputDir: "/o",
|
||||
});
|
||||
assert.match(out, /Success rate\s*: n\/a/);
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { cleanUrl, matchesHost, normalizeHost } from "../src/utils/url.js";
|
||||
|
||||
test("cleanUrl strips trailing punctuation", () => {
|
||||
assert.equal(cleanUrl("https://x.com/a.jpg)."), "https://x.com/a.jpg");
|
||||
assert.equal(cleanUrl("https://x.com/a.jpg',"), "https://x.com/a.jpg");
|
||||
assert.equal(cleanUrl("https://x.com/a.jpg"), "https://x.com/a.jpg");
|
||||
});
|
||||
|
||||
test("cleanUrl preserves query strings", () => {
|
||||
assert.equal(
|
||||
cleanUrl("https://x.com/a.jpg?v=1"),
|
||||
"https://x.com/a.jpg?v=1",
|
||||
);
|
||||
});
|
||||
|
||||
test("normalizeHost accepts bare hosts and full URLs", () => {
|
||||
assert.equal(normalizeHost("Example.COM"), "example.com");
|
||||
assert.equal(normalizeHost("https://Example.com/path"), "example.com");
|
||||
assert.equal(normalizeHost("http://example.com:8080/"), "example.com:8080");
|
||||
});
|
||||
|
||||
test("normalizeHost returns null for empty input", () => {
|
||||
assert.equal(normalizeHost(undefined), null);
|
||||
assert.equal(normalizeHost(""), null);
|
||||
});
|
||||
|
||||
test("matchesHost null filter accepts everything", () => {
|
||||
assert.equal(matchesHost("https://anywhere.test/x.jpg", null), true);
|
||||
});
|
||||
|
||||
test("matchesHost compares case-insensitively", () => {
|
||||
assert.equal(matchesHost("https://Example.com/x.jpg", "example.com"), true);
|
||||
assert.equal(matchesHost("https://other.com/x.jpg", "example.com"), false);
|
||||
});
|
||||
|
||||
test("matchesHost rejects malformed URLs", () => {
|
||||
assert.equal(matchesHost("not a url", "example.com"), false);
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { urlToOutputPath } from "../src/utils/wayback-url.js";
|
||||
|
||||
test("urlToOutputPath preserves host and path layout", () => {
|
||||
assert.equal(
|
||||
urlToOutputPath(
|
||||
"https://example.com/wp-content/uploads/2020/01/photo.jpg",
|
||||
"/out",
|
||||
),
|
||||
"/out/example.com/wp-content/uploads/2020/01/photo.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("urlToOutputPath decodes percent-encoded segments", () => {
|
||||
assert.equal(
|
||||
urlToOutputPath("https://example.com/uploads/foto%20test.jpg", "/out"),
|
||||
"/out/example.com/uploads/foto test.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("urlToOutputPath sanitizes path-traversal segments", () => {
|
||||
const out = urlToOutputPath("https://example.com/../etc/passwd", "/out");
|
||||
assert.ok(!out.includes(".."));
|
||||
});
|
||||
|
||||
test("urlToOutputPath omits host directory when hostPrefix=false", () => {
|
||||
assert.equal(
|
||||
urlToOutputPath(
|
||||
"https://example.com/wp-content/uploads/2020/01/photo.jpg",
|
||||
"/out",
|
||||
{ hostPrefix: false },
|
||||
),
|
||||
"/out/wp-content/uploads/2020/01/photo.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
test("urlToOutputPath falls back to 'index' for empty paths", () => {
|
||||
assert.equal(
|
||||
urlToOutputPath("https://example.com/", "/out"),
|
||||
"/out/example.com/index",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": false,
|
||||
"rootDir": "src"
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": false,
|
||||
"sourceMap": false
|
||||
},
|
||||
"include": ["src/**/*", "tests/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user