diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4262191..c5bc737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,11 +17,11 @@ jobs: - run: bun install --frozen-lockfile # Biome: format + lint + organize-imports - run: bun run check - # TypeScript: per-package typecheck (no root orchestration exists) + # TypeScript - run: bun --cwd packages/types typecheck - run: bun --cwd packages/browser typecheck - run: bun --cwd packages/tiers typecheck - run: bun --cwd apps/api typecheck - # Unit tests - - run: cd packages/tiers && bun test - - run: cd packages/browser && bun test tests/pool.test.ts \ No newline at end of file + - run: bun --cwd apps/web typecheck + # Unit and proxy integration tests + - run: bun test diff --git a/CHANGELOG.md b/CHANGELOG.md index 3183804..ec4286b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,24 +5,28 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.2.0] - 2026-07-23 +## [1.2.0] - 2026-07-26 ### Added - **Tier 0 direct forward in the MITM proxy** (`apps/api/src/proxy/directForward.ts`): the proxy at `:8192` now forwards requests directly to upstream via raw TCP/TLS instead of spinning up a browser for every request. Pool is reserved for the requests that actually need CF bypass; Netflix/YouTube/banks/etc. flow at near-direct speed. - **Smart adaptive streaming** (`apps/api/src/proxy/streaming.ts`): small JSON/HTML/text responses are buffered so the challenge detector can inspect them; video/audio/binary files (`.mp4`, `.mkv`, `.m3u8`, `.zip`, `.exe`, `.dmg`, etc.) are streamed straight through to the client. Default threshold: 8 MiB. - **Per-hostname challenge cache** (`apps/api/src/proxy/challengeCache.ts`): the proxy remembers which hostnames recently returned Cloudflare challenges and sends repeat visits directly to the tiered solver for 5 minutes. +- **Persistent browser context cache**: solved sessions retain their browser fingerprint, cookies, cache, and storage across proxy requests, with bounded per-proxy reuse and cleanup. +- **General HTTP/HTTPS proxy support**: CONNECT tunneling, plain HTTP absolute-form requests, WebSocket upgrades, request bodies, redirects, authentication headers, cookies, and HTTP Range/206 responses are forwarded with their required semantics. - **`proxySanitizeHeaders()`** (`packages/tiers/src/utils/sanitize.ts`): permissive header sanitizer for the transparent MITM proxy — passes `Authorization`, `Cookie`, `Range`, `User-Agent`, `Referer`, and custom API tokens through, strips only RFC 7230 hop-by-hop headers. - **Raw bytes in `@trawl/tiers`**: `ScrapeResult` and tier results now expose `body?: Uint8Array`, `responseHeaders?: Record`, and `contentType?: string` alongside the existing `html` field. The proxy uses these fields to preserve binary content without HTML normalization. - **Tier 1 method handling fix**: `runTier1()` now forwards the request body for `PUT`, `PATCH`, `DELETE`, `QUERY` (was only `POST`). - **Graceful proxy shutdown** (`shutdownMitmProxy()`): the API captures the proxy handle on startup and `lifecycle.ts` calls `shutdownMitmProxy()` on `SIGTERM`/`SIGINT` before the browser pool shutdown, so in-flight connections drain. - **MITM proxy port + CA volume in docker-compose**: `8192:8192` port mapping, `MITM_PROXY_*` env vars, and a persistent `trawl_proxy_ca` volume so the CA survives container restarts. - **Proxy tests**: header sanitization, response policy, adaptive streaming, challenge caching, direct HTTP forwarding, Range/206 handling, chunked responses, compressed challenge detection, and explicit media streaming. +- **Proxy documentation**: architecture, configuration, client setup, and CA installation guides for operating systems, browsers, Java/JDownloader, containers, and application-specific trust stores. ### Changed - `packages/types/src/index.ts` — `ScrapeResult` extended additively with optional `body`, `responseHeaders`, and `contentType`; native `/scrape` consumers should tolerate these additional fields. - `apps/api/src/proxy/server.ts` — `fetchRaw`/`reissue` replaced by `proxyRequest()` (Tier 0 with `scrape()` fallback) for both CONNECT-based HTTPS and plain HTTP traffic. - `lifecycle.ts` — `registerLifecycleHandlers()` accepts an optional `{ onShutdown }` callback. - `apps/api/src/index.ts` — captures the proxy handle on startup and wires it into `registerLifecycleHandlers`. +- All application and internal package versions bumped to `1.2.0`. ## [1.1.0] - 2026-07-22 diff --git a/apps/api/package.json b/apps/api/package.json index 77dac9f..05a2986 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -1,6 +1,6 @@ { "name": "@trawl/api", - "version": "1.1.0", + "version": "1.2.0", "private": true, "scripts": { "dev": "bun run --watch src/index.ts", @@ -17,7 +17,7 @@ "node-forge": "^1.4.0" }, "devDependencies": { - "typescript": "^7.0.2", + "typescript": "^5.9.3", "@types/bun": "^1.3.14", "@types/node-forge": "^1.3.14" } diff --git a/apps/docs/api-reference/health-stats.md b/apps/docs/api-reference/health-stats.md index 8bea8ca..8096f1a 100644 --- a/apps/docs/api-reference/health-stats.md +++ b/apps/docs/api-reference/health-stats.md @@ -18,7 +18,7 @@ FlareSolverr-style readiness message — confirms the API process is up (does no ```json { "msg": "TRAWL is ready!", - "version": "0.1.0", + "version": "1.2.0", "uptime": 42 } ``` diff --git a/apps/web/package.json b/apps/web/package.json index 7ff3ba9..0498c22 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@trawl/web", - "version": "1.1.0", + "version": "1.2.0", "private": true, "scripts": { "dev": "nuxt dev", @@ -16,7 +16,7 @@ "@takumi-rs/core": "^2.4.1", "@vueuse/motion": "^3.0.3", "nuxt": "^4.5.0", - "typescript": "^7.0.2", + "typescript": "^5.9.3", "vue-tsc": "^3.3.8" } } diff --git a/bun.lock b/bun.lock index c4da448..57cc12b 100644 --- a/bun.lock +++ b/bun.lock @@ -10,7 +10,7 @@ }, "apps/api": { "name": "@trawl/api", - "version": "1.1.0", + "version": "1.2.0", "dependencies": { "@trawl/browser": "workspace:*", "@trawl/tiers": "workspace:*", @@ -23,7 +23,7 @@ "devDependencies": { "@types/bun": "^1.3.14", "@types/node-forge": "^1.3.14", - "typescript": "^7.0.2", + "typescript": "^5.9.3", }, }, "apps/docs": { @@ -34,7 +34,7 @@ }, "apps/web": { "name": "@trawl/web", - "version": "1.1.0", + "version": "1.2.0", "devDependencies": { "@nuxt/fonts": "^0.14.0", "@nuxt/image": "^2.0.0", @@ -43,13 +43,13 @@ "@takumi-rs/core": "^2.4.1", "@vueuse/motion": "^3.0.3", "nuxt": "^4.5.0", - "typescript": "^7.0.2", + "typescript": "^5.9.3", "vue-tsc": "^3.3.8", }, }, "packages/browser": { "name": "@trawl/browser", - "version": "1.1.0", + "version": "1.2.0", "dependencies": { "@trawl/types": "workspace:*", "camoufox-js": "^0.11.2", @@ -58,12 +58,12 @@ "@types/bun": "^1.3.14", "patchright": "^1.61.1", "playwright-core": "^1.61.1", - "typescript": "^7.0.2", + "typescript": "^5.9.3", }, }, "packages/tiers": { "name": "@trawl/tiers", - "version": "1.1.0", + "version": "1.2.0", "dependencies": { "@trawl/browser": "workspace:*", "@trawl/types": "workspace:*", @@ -71,14 +71,14 @@ }, "devDependencies": { "@types/bun": "^1.3.14", - "typescript": "^7.0.2", + "typescript": "^5.9.3", }, }, "packages/types": { "name": "@trawl/types", - "version": "1.1.0", + "version": "1.2.0", "devDependencies": { - "typescript": "^7.0.2", + "typescript": "^5.9.3", }, }, }, @@ -651,46 +651,6 @@ "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], - "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], - - "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], - - "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], - - "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], - - "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], - - "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], - - "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], - - "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], - - "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], - - "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], - - "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], - - "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], - - "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], - - "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], - - "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], - - "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], - - "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], - - "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], - - "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], - - "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], - "@ungap/structured-clone": ["@ungap/structured-clone@1.3.3", "", {}, "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg=="], "@unhead/bundler": ["@unhead/bundler@3.2.1", "", { "dependencies": { "@vitejs/devtools-kit": "^0.3.4", "magic-string": "^1.0.0", "oxc-parser": "^0.140.0", "oxc-walker": "^1.0.0", "ufo": "^1.6.4", "unplugin": "^3.3.0" }, "peerDependencies": { "@unhead/cli": "^3.2.1", "esbuild": ">=0.17.0", "lightningcss": ">=1.20.0", "rolldown": ">=1.0.0-beta.0", "unhead": "^3.2.1", "vite": ">=6.4.2", "webpack": ">=5.0.0" }, "optionalPeers": ["@unhead/cli", "esbuild", "lightningcss", "rolldown", "vite", "webpack"] }, "sha512-lMJGiviHfmZT1dtL+VAi4AgP0NrT1dZXDVK96lajUF4tIYxha1FEo0yQun0lp1J+RL8BE2pm3u2LZY9564dA1A=="], @@ -1765,7 +1725,7 @@ "type-level-regexp": ["type-level-regexp@0.1.17", "", {}, "sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg=="], - "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "ua-is-frozen": ["ua-is-frozen@0.1.2", "", {}, "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw=="], @@ -2235,6 +2195,8 @@ "csso/css-tree/mdn-data": ["mdn-data@2.0.28", "", {}, "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g=="], + "devframe/h3/crossws": ["crossws@0.4.10", "", { "peerDependencies": { "srvx": ">=0.11.5" }, "optionalPeers": ["srvx"] }, "sha512-pz3oubH/dt12KjqsUB0IuXW4nwRDQ583iDsP4555Cpdqx0NoU7pGlWBcayyFI8f/l/idRpgjMEfwuOxSWJYlIA=="], + "devframe/h3/rou3": ["rou3@0.8.1", "", {}, "sha512-ePa+XGk00/3HuCqrEnK3LxJW7I0SdNg6EFzKUJG73hMAdDcOUC/i/aSz7LSDwLrGr33kal/rqOGydzwl6U7zBA=="], "devframe/nostics/magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], diff --git a/package.json b/package.json index 3d0f512..7f4015d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "trawl", - "version": "1.1.0", + "version": "1.2.0", "workspaces": [ "apps/*", "packages/*" diff --git a/packages/browser/package.json b/packages/browser/package.json index a454302..30bffa2 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@trawl/browser", - "version": "1.1.0", + "version": "1.2.0", "private": true, "main": "./src/index.ts", "types": "./src/index.ts", @@ -12,7 +12,7 @@ "camoufox-js": "^0.11.2" }, "devDependencies": { - "typescript": "^7.0.2", + "typescript": "^5.9.3", "@types/bun": "^1.3.14", "playwright-core": "^1.61.1", "patchright": "^1.61.1" diff --git a/packages/tiers/package.json b/packages/tiers/package.json index f62cc5f..9f55009 100644 --- a/packages/tiers/package.json +++ b/packages/tiers/package.json @@ -1,6 +1,6 @@ { "name": "@trawl/tiers", - "version": "1.1.0", + "version": "1.2.0", "private": true, "main": "./src/index.ts", "types": "./src/index.ts", @@ -14,6 +14,6 @@ }, "devDependencies": { "@types/bun": "^1.3.14", - "typescript": "^7.0.2" + "typescript": "^5.9.3" } } diff --git a/packages/types/package.json b/packages/types/package.json index 08bea7b..35da2fd 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@trawl/types", - "version": "1.1.0", + "version": "1.2.0", "private": true, "main": "./src/index.ts", "types": "./src/index.ts", @@ -8,6 +8,6 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "typescript": "^7.0.2" + "typescript": "^5.9.3" } }