mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
Merge branch 'dev' into feat/mitm-proxy-mode
This commit is contained in:
Germond
@@ -7,6 +7,8 @@ name: Publish Baseline
|
||||
# Triggers:
|
||||
# - workflow_dispatch: run manually from any branch; pick a tag via the input
|
||||
# (default: "baseline") e.g. "baseline", "1.0.0-baseline"
|
||||
# - push of `v*` tags (e.g., `v1.0.1`) -> docker tag derived as `1.0.1-baseline`
|
||||
# (so cutting a release builds both `:1.0.1` and `:1.0.1-baseline` in one push)
|
||||
# - push of `*-baseline` tags (e.g. `1.0.0-baseline`) → docker tag mirrors git tag
|
||||
|
||||
on:
|
||||
@@ -17,7 +19,9 @@ on:
|
||||
required: false
|
||||
default: 'baseline'
|
||||
push:
|
||||
tags: ["*-baseline"]
|
||||
tags:
|
||||
- "v*"
|
||||
- "*-baseline"
|
||||
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository_owner }}/trawl
|
||||
@@ -106,7 +110,19 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||
TAG="${GITHUB_REF_NAME}"
|
||||
REF="${GITHUB_REF_NAME}"
|
||||
if [[ "${REF}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
# Versioned release tag (e.g., v1.0.1) -> derive `1.0.1-baseline`
|
||||
TAG="${REF#v}-baseline"
|
||||
elif [[ "${REF}" =~ ^v.*-baseline$ ]]; then
|
||||
# Should never happen given trigger patterns, but guard so we
|
||||
# never produce a literal `v1.0.1-baseline-baseline` docker tag.
|
||||
echo "::error::Tag '${REF}' mixes both conventions. Push the bare vX.Y.Z tag - publish.yml handles :1.0.1, this workflow handles :1.0.1-baseline."
|
||||
exit 1
|
||||
else
|
||||
# Explicit baseline tag push (e.g., `1.0.1-baseline`) -> use as-is
|
||||
TAG="${REF}"
|
||||
fi
|
||||
else
|
||||
TAG="${{ inputs.tag }}"
|
||||
fi
|
||||
|
||||
@@ -98,9 +98,12 @@ jobs:
|
||||
with:
|
||||
images: ${{ env.IMAGE }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
type=semver,pattern={{version}}
|
||||
type=sha,prefix=sha-
|
||||
# Versioned release tag (e.g. v1.0.0) → keep the v prefix.
|
||||
type=ref,event=tag
|
||||
# main branch push → :nightly (disposable test build).
|
||||
type=raw,value=nightly,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
# main branch push → :nightly-<sha> for traceability.
|
||||
type=sha,prefix=nightly-
|
||||
|
||||
- name: Create and push multi-arch manifest
|
||||
working-directory: /tmp/digests
|
||||
@@ -110,4 +113,15 @@ jobs:
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect manifest
|
||||
run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ steps.meta.outputs.version }}
|
||||
env:
|
||||
IMAGE: ${{ env.IMAGE }}
|
||||
REF: ${{ github.ref }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
if [[ "$REF" == refs/tags/v* ]]; then
|
||||
tag="${REF#refs/tags/}"
|
||||
else
|
||||
tag="nightly-$SHA"
|
||||
fi
|
||||
docker buildx imagetools inspect "$IMAGE:$tag"
|
||||
|
||||
|
||||
+12
-1
@@ -5,7 +5,18 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
## [1.0.1] - 2026-07-18
|
||||
|
||||
### Changed
|
||||
- `packages/browser/src/pool.ts` — renamed Firefox prefs key from `prefs` (silently ignored by camoufox-js@0.11.1) to `firefox_user_prefs` (which camoufox-js maps to Playwright's `firefoxUserPrefs`). The prefs are now actually applied.
|
||||
- `packages/browser/src/pool.ts` — added the safe-only subset of Firefox prefs: telemetry off (`datareporting.*`, `toolkit.telemetry.*`, `app.crashreporter`, `breakpad.*`), dead UI features off (`extensions.screenshots.*`, `browser.sessionstore.max_tabs_undo`), dead network services off (`browser.safebrowsing.*`, `extensions.update.*`, `browser.fixup.alternate.*`, `app.normandy.*`, `app.shield.*`, `network.connectivity-service.*`, `network.captive-portal-service.*`, `network.prefetch-next`, `beacon.enabled`), `security.OCSP.enabled: 0`, and tightened network timeouts (`tls-handshake-timeout: 30`, `connection-timeout: 60`, `response.timeout: 120`). None of these touch the JS/CSS fingerprint surface.
|
||||
- `apps/api/Dockerfile` — stage-3 prune of apt cache + `/usr/share/{locale,doc,man}` (image-size win, runtime-neutral).
|
||||
- `apps/api/Dockerfile` — added 9 Bun runtime ENV flags (`BUN_DISABLE_CJS=1`, `BUN_DEBUG=0`, `BUN_DISABLE_SOURCEMAPS=1`, `BUN_HTTP_KEEPALIVE=0`, `BUN_AGENT_DISABLE=1`, `BUN_INSPECT=0`, `BUN_LOCKFILE_MIGRATION=false`, `MIMALLOC_PURGE_DELAY=0`, `NODE_NO_WARNINGS=1`). All verified runtime-neutral in smoke tests.
|
||||
- `packages/browser/package.json` — moved `patchright` + `playwright-core` from `dependencies` to `devDependencies` (build hygiene; camoufox-js bundles both transitively at runtime).
|
||||
- All packages bumped to `1.0.1`.
|
||||
|
||||
### Added
|
||||
- `scripts/bench-targets.sh`, `scripts/bench-success-rate.sh`, `scripts/bench-compare.sh` — observability harnesses for measuring CF challenge latency + bypass success rate.
|
||||
|
||||
### Added
|
||||
- **MITM forward-proxy mode** (`MITM_PROXY_ENABLED`) — a browser-backed HTTP(S) forward proxy.
|
||||
|
||||
+14
-2
@@ -108,7 +108,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
fonts-liberation \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& apt-get clean
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /usr/share/locale /usr/share/doc /usr/share/man
|
||||
|
||||
COPY --from=oven/bun:1.3.14 /usr/local/bin/bun /usr/local/bin/bun
|
||||
COPY --from=camoufox /opt/camoufox /opt/camoufox
|
||||
@@ -128,7 +129,18 @@ COPY packages/tiers/ /app/packages/tiers/
|
||||
COPY apps/api/ /app/apps/api/
|
||||
COPY package.json /app/
|
||||
|
||||
ENV CAMOUFOX_INSTALL_DIR=/opt/camoufox
|
||||
ENV CAMOUFOX_INSTALL_DIR=/opt/camoufox \
|
||||
# Safe Bun runtime knobs — all tested runtime-neutral (no behavior change).
|
||||
BUN_DISABLE_CJS=1 \
|
||||
BUN_DEBUG=0 \
|
||||
BUN_DISABLE_SOURCEMAPS=1 \
|
||||
BUN_HTTP_KEEPALIVE=0 \
|
||||
BUN_AGENT_DISABLE=1 \
|
||||
BUN_INSPECT=0 \
|
||||
BUN_LOCKFILE_MIGRATION=false \
|
||||
MIMALLOC_PURGE_DELAY=0 \
|
||||
NODE_NO_WARNINGS=1
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8191
|
||||
CMD ["bun", "run", "apps/api/src/index.ts"]
|
||||
|
||||
@@ -47,7 +47,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
|
||||
# Pin Camoufox to v150.0.2-beta.25 (verified working). The camoufox-js `fetch` always
|
||||
# downloads the latest release, which broke in 152.x (browser binary no longer in the
|
||||
# zip). Direct curl keeps builds reproducible.
|
||||
# zip — see Camoufox upstream release notes). Direct curl keeps builds reproducible.
|
||||
#
|
||||
# Asset naming: arm64 = alpha.25 build, x86_64 = alpha.26 rebuild (only x86_64 was
|
||||
# rebuilt in the v150.0.2-beta.25 release).
|
||||
ARG TARGETARCH
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
|
||||
case "$TARGETARCH" in \
|
||||
@@ -84,8 +87,6 @@ RUN curl -fsSL \
|
||||
# Camoufox/Firefox require glibc; Alpine's musl is incompatible.
|
||||
FROM debian:bookworm-slim
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ARG TARGETARCH
|
||||
ARG BUN_VERSION=1.3.14
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
|
||||
@@ -106,13 +107,16 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
libasound2 \
|
||||
fonts-liberation \
|
||||
ca-certificates unzip curl \
|
||||
&& apt-get clean
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /usr/share/locale /usr/share/doc /usr/share/man
|
||||
|
||||
# Pull the Bun runtime directly from GitHub releases.
|
||||
# amd64 uses the BASELINE variant (pre-2013 CPUs without AVX2 — covers older
|
||||
# Synology/Atom/Kabylake-era hardware). arm64 has no baseline variant because
|
||||
# arm64 doesn't use x86 SIMD — the standard `bun-linux-aarch64.zip` works on
|
||||
# all armv8 CPUs (including every ARM64 Synology).
|
||||
# Pull the Bun runtime directly from GitHub releases. The baseline-built binary
|
||||
# supports older x86_64 CPUs without AVX2 (pre-Haswell / 2013-era hardware,
|
||||
# common on Synology NAS, Atom tablets, and older mini-PCs).
|
||||
# arm64 has no baseline variant — every armv8 CPU is supported by the standard
|
||||
# `bun-linux-aarch64.zip` binary.
|
||||
ARG TARGETARCH
|
||||
ARG BUN_VERSION=1.3.14
|
||||
RUN set -eux; \
|
||||
case "$TARGETARCH" in \
|
||||
amd64) BUN_ZIP=bun-linux-x64-baseline.zip ;; \
|
||||
@@ -137,38 +141,24 @@ RUN strip --strip-all /usr/local/bin/bun 2>/dev/null || true \
|
||||
&& find /opt/camoufox -type f \( -name 'firefox' -o -name 'firefox-bin' -o -name '*.so*' \) \
|
||||
-exec strip --strip-unneeded {} + 2>/dev/null || true
|
||||
|
||||
# Prune better-sqlite3's C source/docs/binding.gyp — keeps only the .node binary +
|
||||
# JS wrapper that the runtime actually loads. Saves ~12 MB.
|
||||
RUN find /app/node_modules -path '*/better-sqlite3*/deps' -prune -exec rm -rf {} + \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*/src' -prune -exec rm -rf {} + \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*' -name '*.md' -delete \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*' -name '*.gyp' -delete \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*' -name '*.c' -delete \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*' -name '*.h' -delete \
|
||||
&& find /app/node_modules -path '*/better-sqlite3*' -name '*.map' -delete
|
||||
|
||||
# Drop TypeScript devDeps that leaked through `--production` (bun 1.3.x bug:
|
||||
# workspace devDeps still get installed when packages are in a workspace).
|
||||
# ts/bun-types/@types are not needed at runtime.
|
||||
RUN rm -rf /app/node_modules/typescript \
|
||||
/app/node_modules/bun-types \
|
||||
/app/node_modules/@types
|
||||
|
||||
# Drop impit musl variant — we run glibc (debian-slim), only need the gnu binary.
|
||||
RUN rm -rf /app/node_modules/impit-linux-arm64-musl \
|
||||
/app/node_modules/impit-linux-x64-musl \
|
||||
/app/node_modules/impit-darwin-* \
|
||||
/app/node_modules/impit-win32-*
|
||||
|
||||
|
||||
|
||||
COPY packages/types/ /app/packages/types/
|
||||
COPY packages/browser/ /app/packages/browser/
|
||||
COPY packages/tiers/ /app/packages/tiers/
|
||||
COPY apps/api/ /app/apps/api/
|
||||
COPY package.json /app/
|
||||
|
||||
ENV CAMOUFOX_INSTALL_DIR=/opt/camoufox
|
||||
ENV CAMOUFOX_INSTALL_DIR=/opt/camoufox \
|
||||
# Safe Bun runtime knobs — all tested runtime-neutral (no behavior change).
|
||||
BUN_DISABLE_CJS=1 \
|
||||
BUN_DEBUG=0 \
|
||||
BUN_DISABLE_SOURCEMAPS=1 \
|
||||
BUN_HTTP_KEEPALIVE=0 \
|
||||
BUN_AGENT_DISABLE=1 \
|
||||
BUN_INSPECT=0 \
|
||||
BUN_LOCKFILE_MIGRATION=false \
|
||||
MIMALLOC_PURGE_DELAY=0 \
|
||||
NODE_NO_WARNINGS=1
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8191
|
||||
CMD ["bun", "run", "apps/api/src/index.ts"]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@trawl/api",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "bun run --watch src/index.ts",
|
||||
@@ -11,9 +11,9 @@
|
||||
"@trawl/browser": "workspace:*",
|
||||
"@trawl/tiers": "workspace:*",
|
||||
"@trawl/types": "workspace:*",
|
||||
"camoufox-js": "0.11.1",
|
||||
"camoufox-js": "0.11.2",
|
||||
"elysia": "^1.4.29",
|
||||
"memoirist": "1.1.0",
|
||||
"memoirist": "1.2.0",
|
||||
"node-forge": "^1.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -59,8 +59,8 @@ const rows = [
|
||||
},
|
||||
{ feature: "Adaptive tier execution", trawl: "✓ 4 tiers", flaresolver: "✗ always browser", byparr: "~" },
|
||||
{ feature: "Browser engine", trawl: "✓ Camoufox Firefox", flaresolver: "✗ Chrome", byparr: "✓ Camoufox Firefox" },
|
||||
{ feature: "Docker image size", trawl: "✓ 1.18 GB", flaresolver: "✓ 1.03 GB", byparr: "✗ 4.10 GB" },
|
||||
{ feature: "Memory footprint (peak)", trawl: "✓ 770 MB", flaresolver: "✗ 500 MB", byparr: "✗ 1.35 GB" },
|
||||
{ feature: "Docker image size", trawl: "✓ 1.17 GB", flaresolver: "✓ 1.03 GB", byparr: "✗ 4.10 GB" },
|
||||
{ feature: "Memory footprint (peak)", trawl: "✓ 770 MB", flaresolver: "✓ 500 MB", byparr: "✗ 1.35 GB" },
|
||||
{ feature: "Cloudflare challenge speed", trawl: "✓ 4–15s", flaresolver: "✗ 11–18s", byparr: "✗ 13–18s" },
|
||||
{ feature: "CF Turnstile solving", trawl: "✓ shadow DOM click", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "reCAPTCHA v2 solving", trawl: "✓ audio STT (free)", flaresolver: "✗", byparr: "✗" },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@trawl/web",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "nuxt dev",
|
||||
@@ -13,7 +13,7 @@
|
||||
"@nuxt/image": "^2.0.0",
|
||||
"@nuxtjs/color-mode": "^4.0.1",
|
||||
"@nuxtjs/seo": "5.3.2",
|
||||
"@takumi-rs/core": "^2.0.2",
|
||||
"@takumi-rs/core": "^2.3.0",
|
||||
"@vueuse/motion": "^3.0.3",
|
||||
"nuxt": "^4.4.8",
|
||||
"typescript": "^7.0.2",
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/2.5.3/schema.json",
|
||||
"$schema": "https://biomejs.dev/schemas/2.5.4/schema.json",
|
||||
"vcs": {
|
||||
"enabled": true,
|
||||
"clientKind": "git"
|
||||
|
||||
@@ -5,20 +5,20 @@
|
||||
"": {
|
||||
"name": "trawl",
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.3",
|
||||
"@biomejs/biome": "2.5.4",
|
||||
},
|
||||
},
|
||||
"apps/api": {
|
||||
"name": "@trawl/api",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"@trawl/browser": "workspace:*",
|
||||
"@trawl/tiers": "workspace:*",
|
||||
"@trawl/types": "workspace:*",
|
||||
"camoufox-js": "0.11.1",
|
||||
"camoufox-js": "0.11.2",
|
||||
"elysia": "^1.4.29",
|
||||
"memoirist": "1.1.0",
|
||||
"node-forge": "^1.3.1",
|
||||
"memoirist": "1.2.0",
|
||||
"node-forge": "^1.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.3.14",
|
||||
@@ -34,13 +34,13 @@
|
||||
},
|
||||
"apps/web": {
|
||||
"name": "@trawl/web",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"devDependencies": {
|
||||
"@nuxt/fonts": "^0.14.0",
|
||||
"@nuxt/image": "^2.0.0",
|
||||
"@nuxtjs/color-mode": "^4.0.1",
|
||||
"@nuxtjs/seo": "5.3.2",
|
||||
"@takumi-rs/core": "^2.0.2",
|
||||
"@takumi-rs/core": "^2.3.0",
|
||||
"@vueuse/motion": "^3.0.3",
|
||||
"nuxt": "^4.4.8",
|
||||
"typescript": "^7.0.2",
|
||||
@@ -49,21 +49,21 @@
|
||||
},
|
||||
"packages/browser": {
|
||||
"name": "@trawl/browser",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"@trawl/types": "workspace:*",
|
||||
"camoufox-js": "^0.11.1",
|
||||
"patchright": "^1.61.1",
|
||||
"playwright-core": "^1.61.1",
|
||||
"camoufox-js": "^0.11.2",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.3.14",
|
||||
"patchright": "^1.61.1",
|
||||
"playwright-core": "^1.61.1",
|
||||
"typescript": "^7.0.2",
|
||||
},
|
||||
},
|
||||
"packages/tiers": {
|
||||
"name": "@trawl/tiers",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"@trawl/browser": "workspace:*",
|
||||
"@trawl/types": "workspace:*",
|
||||
@@ -76,7 +76,7 @@
|
||||
},
|
||||
"packages/types": {
|
||||
"name": "@trawl/types",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"devDependencies": {
|
||||
"typescript": "^7.0.2",
|
||||
},
|
||||
@@ -175,23 +175,23 @@
|
||||
|
||||
"@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="],
|
||||
|
||||
"@biomejs/biome": ["@biomejs/biome@2.5.3", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.5.3", "@biomejs/cli-darwin-x64": "2.5.3", "@biomejs/cli-linux-arm64": "2.5.3", "@biomejs/cli-linux-arm64-musl": "2.5.3", "@biomejs/cli-linux-x64": "2.5.3", "@biomejs/cli-linux-x64-musl": "2.5.3", "@biomejs/cli-win32-arm64": "2.5.3", "@biomejs/cli-win32-x64": "2.5.3" }, "bin": { "biome": "bin/biome" } }, "sha512-MrJswFdei9EfDwwUy2tQrPDpK0AO+RmMFvBoaaJ6ayBc3sUbHdCE+XG5N8vp+5So41ZupZJQm0roHFFhMGVD7A=="],
|
||||
"@biomejs/biome": ["@biomejs/biome@2.5.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.5.4", "@biomejs/cli-darwin-x64": "2.5.4", "@biomejs/cli-linux-arm64": "2.5.4", "@biomejs/cli-linux-arm64-musl": "2.5.4", "@biomejs/cli-linux-x64": "2.5.4", "@biomejs/cli-linux-x64-musl": "2.5.4", "@biomejs/cli-win32-arm64": "2.5.4", "@biomejs/cli-win32-x64": "2.5.4" }, "bin": { "biome": "bin/biome" } }, "sha512-xy5FNE5kQJKyK5MR1gJy6ztXYx4WBAbYGlK04lMEgmyPRWKybY9NFwiG9yo0XdzOU8Xvhj41u034J1ywfoWfMw=="],
|
||||
|
||||
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.5.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QhYP9muVQ0nUO5zztFuPbEwi4+94sJWVjaZds9aMi1l/KNZBiUjdiSUrGHsTaMGDXrYl+r4AS2sUKfgH3w+V3g=="],
|
||||
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.5.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-4o3NFRobXHynkgcFVrlZsoDAFtF2ldlEGN8sORSws5ZQqyY4PXnPUIylu4ksfyHuwkfvDREuWh3JK+niRwGq3w=="],
|
||||
|
||||
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.5.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-NC1Ss13UaW7QZX+y8j44bF7AP0jSJdBl6iRhe0MAkvaSqZy+mWg3GaXsrb+eSoHoGDBtaXWEbMVV0iVN2cZ7cQ=="],
|
||||
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.5.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-D32P5HkU2Y6PySuC/WsVDTOgsDwVFmujzhhhOQjajtATpVWFDXuVd3oRbsWNSEA+aaFzyzZm22szsyydBYlSyQ=="],
|
||||
|
||||
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.5.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-ksx1KWeyYW18ILL04msF/J4ZBtBDN33znYK8Z/aNv/vlBVxL9/g3mGP+omgHJKy4+KWbK87vcmmpmurfNjSgiA=="],
|
||||
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-pSEfW7B8kTsXUjUxC1xVVK+y85Ht3C5XxZ9gclmC7/3Ku9Vqz8jmI7k0p/BNIjQ6t4sFERI2sFeH73ybiZl6YQ=="],
|
||||
|
||||
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.5.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-fccix0w6xp6csCXgxeC0dU/3ecgRQal0y+cv2SP9ajNlhe7Yrk2Ug7UDe2j9AT9ZDYitkXpvUKgZjjuoYeP4Vg=="],
|
||||
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-Rpm5/AT1m+DlJmUoYvS4/vXc+0tXJPJ2NQz25TGPyHVF5JrWy75PE0GH6kVxsKtQDuCH4OgzquZq0R4kj/wCVg=="],
|
||||
|
||||
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.5.3", "", { "os": "linux", "cpu": "x64" }, "sha512-yMkJtilsgvILDcVkh187aVLTb64xYsrxYajx5kym+r1ULkO5HUOfu9AYKLGQbOVLwJtT2utNw7hhFNg+17mUYA=="],
|
||||
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-FNxojWJkL7EajAuzBgoLe0T2G0y112M4lBrDIFl/DomFTx8yqenYOIdsRLNXvOvBBofE8hJi85LjzLmBDpY7/Q=="],
|
||||
|
||||
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.5.3", "", { "os": "linux", "cpu": "x64" }, "sha512-O/yU9YKRUiHhmcjF2f38PSjseVk3G4VLWYc0G2HWpzdBVREV6G8IGWIVEFf7MFPfWIzNUIvPsEjeAZQIOgnLcQ=="],
|
||||
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-aby/PohmmgbShcHqFsZVzG8H6D98+P+A6xRWRrQcLW1pCjabcov5UUlke4UqNQBYTkDQav+jB4zyyDDeKB2GaA=="],
|
||||
|
||||
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.5.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-cX5z+GYwRcqEok0AH3KSfQGgqYd0Nomfp6Fbe1uiTtELE38hdH2k842wQ9wLNaF/JJ7r4rjJQ4VR+ce+fRmQbw=="],
|
||||
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.5.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-emoXexPZIPAZkz2RKmA95WJUqK3I5MJNYtwEbL5ESciRzhmFMMyekDhNG8hpeOaK+ZGRDxAU4wvGuA5IHQ0h0w=="],
|
||||
|
||||
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.3", "", { "os": "win32", "cpu": "x64" }, "sha512-ExSaJWi4/u6+GXCszlSKpWSjKNbDseAYqqkCznsCsZ/4uidZ/BEqsCc5/3ctlq6dfIubdIIRSVLC/PG9xPl70Q=="],
|
||||
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.5.4", "", { "os": "win32", "cpu": "x64" }, "sha512-U1jaluLw1qQc2Tx7/CeSoL9N5XcqIH+GWjpUAy1ouB5nVjSCMNO+NNHdY3RAs8zxNurLWAdj6pehQdCA2zyU+Q=="],
|
||||
|
||||
"@bomb.sh/tab": ["@bomb.sh/tab@0.0.17", "", { "peerDependencies": { "cac": "^6.7.14", "citty": "^0.1.6 || ^0.2.0", "commander": "^13.1.0" }, "optionalPeers": ["cac", "citty", "commander"], "bin": { "tab": "dist/bin/cli.mjs" } }, "sha512-rGhqfWwaSF6qN5Gm5P9EH9eybrwLEowHTkV+wsRgFewT6aQCQWVWXLclVk0McgVIlWCGX+W9mYYC1Egsg+Znsw=="],
|
||||
|
||||
@@ -727,25 +727,25 @@
|
||||
|
||||
"@tailwindcss/vite": ["@tailwindcss/vite@4.3.1", "", { "dependencies": { "@tailwindcss/node": "4.3.1", "@tailwindcss/oxide": "4.3.1", "tailwindcss": "4.3.1" }, "peerDependencies": { "vite": "^5.2.0 || ^6 || ^7 || ^8" } }, "sha512-hItDHuIIlEV61R+faXu66s1K36aTurO/Qw0e45Vskz57gXl9pWOT6eg3zmcEui6CZXddbN7zd41bwmvag4JGwQ=="],
|
||||
|
||||
"@takumi-rs/core": ["@takumi-rs/core@2.0.2", "", { "dependencies": { "@takumi-rs/helpers": "2.0.2" }, "optionalDependencies": { "@takumi-rs/core-darwin-arm64": "2.0.2", "@takumi-rs/core-darwin-x64": "2.0.2", "@takumi-rs/core-linux-arm64-gnu": "2.0.2", "@takumi-rs/core-linux-arm64-musl": "2.0.2", "@takumi-rs/core-linux-x64-gnu": "2.0.2", "@takumi-rs/core-linux-x64-musl": "2.0.2", "@takumi-rs/core-win32-arm64-msvc": "2.0.2", "@takumi-rs/core-win32-x64-msvc": "2.0.2" }, "peerDependencies": { "csstype": "*" }, "optionalPeers": ["csstype"] }, "sha512-AjH3yYYqzOPP3UiOuzqwgNpBmUlRG3KQWOTLPd6dPtn84/wFvGz2IUQKyvbeVDDFBI4jH57C1I4d/4hA/D70JQ=="],
|
||||
"@takumi-rs/core": ["@takumi-rs/core@2.3.0", "", { "dependencies": { "@takumi-rs/helpers": "2.3.0" }, "optionalDependencies": { "@takumi-rs/core-darwin-arm64": "2.3.0", "@takumi-rs/core-darwin-x64": "2.3.0", "@takumi-rs/core-linux-arm64-gnu": "2.3.0", "@takumi-rs/core-linux-arm64-musl": "2.3.0", "@takumi-rs/core-linux-x64-gnu": "2.3.0", "@takumi-rs/core-linux-x64-musl": "2.3.0", "@takumi-rs/core-win32-arm64-msvc": "2.3.0", "@takumi-rs/core-win32-x64-msvc": "2.3.0" }, "peerDependencies": { "csstype": "*" }, "optionalPeers": ["csstype"] }, "sha512-thOAMYzS2v7kbXHdBH5NItKM7qwusG2lR8ed8g26TpnkZBr5kLMLZnZnarTjF6A3YQ58Vzxw0d8aUrgrk5oK3Q=="],
|
||||
|
||||
"@takumi-rs/core-darwin-arm64": ["@takumi-rs/core-darwin-arm64@2.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Xd4CWJoE/8sbc8YBUEznGLKms8UrV8PpYmZFpN5PKYq/DF6Xx2nbzoZygihSf2dJxg1WXlsDdoDbmFQ8WjEoMw=="],
|
||||
"@takumi-rs/core-darwin-arm64": ["@takumi-rs/core-darwin-arm64@2.3.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6V+TDWmsiox4gGEYoazHQYkGU+vadvFdhBMBJ0tXlFx2GuW2tWgc6ShStZYksUTvVWK3MmoQL5/u/HAZE8RdVA=="],
|
||||
|
||||
"@takumi-rs/core-darwin-x64": ["@takumi-rs/core-darwin-x64@2.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-RBcazK7MAVaZlrmn5F90x5OYxo4SeRMQRQmnAXm3WAT7cqLYWkdHkHd3uHwoIO/2HXP4bQUvOWAoeWJ6ESM67g=="],
|
||||
"@takumi-rs/core-darwin-x64": ["@takumi-rs/core-darwin-x64@2.3.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-BkuI4GTOXIC1nRhm9N3t4QshOLSFqs/l64e61Wnu9aJp1VbifnaFcKyLVe7Dw2iIywbqazGkdKAhbqo8ol909A=="],
|
||||
|
||||
"@takumi-rs/core-linux-arm64-gnu": ["@takumi-rs/core-linux-arm64-gnu@2.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-9dmFXYPC/USu5eOa9jLFwkxGEVQrkZvRJxzsJrg3dyYBkBfgd7ejHTZxJu8DdvXHPWoI+vdDulABc944EU5T+Q=="],
|
||||
"@takumi-rs/core-linux-arm64-gnu": ["@takumi-rs/core-linux-arm64-gnu@2.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-VXyXcjExj8h4TJNXJOWY57hO6TCaJteAJe3Q1JxmP4HqjoHGpkt5A8PmL5zj4F0dlIpaPsO+AfhuWVQw9WrmwA=="],
|
||||
|
||||
"@takumi-rs/core-linux-arm64-musl": ["@takumi-rs/core-linux-arm64-musl@2.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-/xH8s05ee97rTtX8Cdhj6Bc9RkKU10sbiwt7iJWueqxLy6S2Y+Z1jSo/sCipStEGrdFGxEx/hEJ1n/jYTvwBqA=="],
|
||||
"@takumi-rs/core-linux-arm64-musl": ["@takumi-rs/core-linux-arm64-musl@2.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-aH/Puouk5ZVFud2vl4qDQMKdvGbnK75SxtrciWZLcYreUVX0A6Du+UC6qlehgFa0gBPCwEbQC0Bf5Rfo+cXWpg=="],
|
||||
|
||||
"@takumi-rs/core-linux-x64-gnu": ["@takumi-rs/core-linux-x64-gnu@2.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-4wdbiFrtD7+O43EuyvWngQimB2Cp/Egebtgc9eoBSNpZOYS+pTxTlXoB2+ldbQenW+2VW41oxpXml7nE9Xn7Fg=="],
|
||||
"@takumi-rs/core-linux-x64-gnu": ["@takumi-rs/core-linux-x64-gnu@2.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-nZ0M+1KFvBLdNcdBM1jazwLhKp0kj8W6BwHJs3RKW+CbON8RXaBLQqvjIAOs6Gyw+/v1S2gkCs7aJGI3TZ/uxA=="],
|
||||
|
||||
"@takumi-rs/core-linux-x64-musl": ["@takumi-rs/core-linux-x64-musl@2.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-itHuOGs1FHAbGZWJEwA9oOvPE9fCd/67sw7aLFcROm0ZSu0I6gh8iIZ+OeaRpI4wq7xTEVqM0QTDUkO2YzDTmg=="],
|
||||
"@takumi-rs/core-linux-x64-musl": ["@takumi-rs/core-linux-x64-musl@2.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-Afu9h+v51zFixvG3It4OgzDjjiLB4kPv0r+iJDwoRKSwP5tMju8HS10Qig+idWLSkWIe2jOIAV+wud2o9A8kPg=="],
|
||||
|
||||
"@takumi-rs/core-win32-arm64-msvc": ["@takumi-rs/core-win32-arm64-msvc@2.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-BByPhDUbwcvJT9KymuhmwgQivbw0mC7r04SnoSltDj9BqvVP9yl++fB0rOzkW7bZX7WGncXnemSrp+bRaYXp3g=="],
|
||||
"@takumi-rs/core-win32-arm64-msvc": ["@takumi-rs/core-win32-arm64-msvc@2.3.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-2b1GwWYIwM/i23eTmSZnzEj1tz3g3URZjk+a6tO5qif8dEhx17ufn6MnXa2doGjXcrx3EIsgAYIRvtmKCrGUKg=="],
|
||||
|
||||
"@takumi-rs/core-win32-x64-msvc": ["@takumi-rs/core-win32-x64-msvc@2.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-pEcwQPMyKn1s/DwQhneL6NQbVqmVjyyrOqSEio0lk6xgHsjMlMdefmEeLs2C6ZZHWktPfQyn7ctklZ4VJO2E0Q=="],
|
||||
"@takumi-rs/core-win32-x64-msvc": ["@takumi-rs/core-win32-x64-msvc@2.3.0", "", { "os": "win32", "cpu": "x64" }, "sha512-kh0y7fLzqNV3rlPUp3XKThykySrohS98neiPWm+D71Yot3USXkYqPCFKHpDdZoan6ZyQyoldcf2jbN3+KLmp+w=="],
|
||||
|
||||
"@takumi-rs/helpers": ["@takumi-rs/helpers@2.0.2", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-449j/koF+qamizJC8v/L5RVQ0oVufPzZ4h07bJZ93RWJq+ZxdlGCLLAXV3sOUGTcf9FPUk3l4kDsG5fUMq4Z8g=="],
|
||||
"@takumi-rs/helpers": ["@takumi-rs/helpers@2.3.0", "", { "peerDependencies": { "preact": "^10.0.0", "react": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["preact", "react"] }, "sha512-2y/fFESu2c3JVZyk/uL+imEgPInqArWPDGbZrbyqcEKwnRmFKYMHFNkFWZhyjIfHc8us/4+NgPkydPxzz6Fwiw=="],
|
||||
|
||||
"@tanstack/table-core": ["@tanstack/table-core@8.21.3", "", {}, "sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg=="],
|
||||
|
||||
@@ -1083,7 +1083,7 @@
|
||||
|
||||
"callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="],
|
||||
|
||||
"camoufox-js": ["camoufox-js@0.11.1", "", { "dependencies": { "adm-zip": "^0.5.16", "better-sqlite3": "^12.10.0", "cli-progress": "^3.12.0", "commander": "^14.0.0", "fingerprint-generator": "^2.1.66", "glob": "^13.0.0", "impit": "^0.14.1", "language-tags": "^2.0.1", "maxmind": "^5.0.0", "pretty-bytes": "^7.1.0", "ua-parser-js": "^2.0.2", "xml2js": "^0.6.2" }, "peerDependencies": { "playwright-core": "*" }, "bin": { "camoufox-js": "dist/__main__.js" } }, "sha512-/tSgFAgjcbxudXFLd+u5VARZHOK9fm5bg1QRAP/2NySdaq6Vf7UTk2aMEUqaXAfnV7ncf4y87KjIFVpWVbZ+RQ=="],
|
||||
"camoufox-js": ["camoufox-js@0.11.2", "", { "dependencies": { "adm-zip": "^0.5.16", "better-sqlite3": "^12.10.0", "cli-progress": "^3.12.0", "commander": "^14.0.0", "fingerprint-generator": "^2.1.66", "glob": "^13.0.0", "impit": "^0.14.1", "language-tags": "^2.0.1", "maxmind": "^5.0.0", "pretty-bytes": "^7.1.0", "ua-parser-js": "^2.0.2", "xml2js": "^0.6.2" }, "peerDependencies": { "playwright-core": "*" }, "bin": { "camoufox-js": "dist/__main__.js" } }, "sha512-T9csy3MIlQRX+5V0+hPLpYaim8Qd2j3ivqtl+5v97T/J9OWqnG42PmketAX1pDA3T3BDCVGaMTfcz9eO2xjkkg=="],
|
||||
|
||||
"caniuse-api": ["caniuse-api@4.0.0", "", { "dependencies": { "browserslist": "^4.0.0", "caniuse-lite": "^1.0.0" } }, "sha512-B0hQ1OLyJuHTQSOWXvwibWqM6DCoqJdvBA6X1S/53bd4XU7LJ1yurIPlrsouol3mw1jh9pGI4ivubSpmJeIqCA=="],
|
||||
|
||||
@@ -1577,7 +1577,7 @@
|
||||
|
||||
"mdn-data": ["mdn-data@2.27.1", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
|
||||
|
||||
"memoirist": ["memoirist@1.1.0", "", {}, "sha512-9KhJFSx4d/zxM++Rsb5yC3gqg+9boFHpDLcV7NkTWIYfUb3tiRPV4/EIMgw2lX7RdZz04xHMe6JTyJBpUUc/Og=="],
|
||||
"memoirist": ["memoirist@1.2.0", "", {}, "sha512-ER/xZGlbn4tExSGnZ03jawdq9M7FN3J6VfK8BdYzJkLetZBchOTUG+RQZqaoD9tZegaHUmdsACAzHkXBEew4uA=="],
|
||||
|
||||
"merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="],
|
||||
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "trawl",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
"packages/*"
|
||||
@@ -15,6 +15,6 @@
|
||||
"check": "biome check . --write"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.3"
|
||||
"@biomejs/biome": "2.5.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@trawl/browser",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
@@ -9,12 +9,12 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@trawl/types": "workspace:*",
|
||||
"camoufox-js": "^0.11.1",
|
||||
"playwright-core": "^1.61.1",
|
||||
"patchright": "^1.61.1"
|
||||
"camoufox-js": "^0.11.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^7.0.2",
|
||||
"@types/bun": "^1.3.14"
|
||||
"@types/bun": "^1.3.14",
|
||||
"playwright-core": "^1.61.1",
|
||||
"patchright": "^1.61.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,12 +137,47 @@ export class BrowserPool {
|
||||
// + browser-side Intl locale all align.
|
||||
locale: fingerprint.locale,
|
||||
timezone: fingerprint.timezone,
|
||||
// Cap content processes per browser. Firefox's default (8) lets thread count climb
|
||||
// when Tier 3/Tier 4 churn contexts (see #13). Lower cap → bounded OS footprint.
|
||||
// `processPrelaunch: false` stops Firefox from pre-warming extra processes eagerly.
|
||||
prefs: {
|
||||
// Camoufox 150.x managed-mode relies on `firefox_user_prefs` (Playwright
|
||||
// maps this to firefoxUserPrefs). The earlier `prefs` key was silently
|
||||
// ignored, so these settings were dead code in 1.0.0.
|
||||
firefox_user_prefs: {
|
||||
"dom.ipc.processCount": this.contentProcesses,
|
||||
"dom.ipc.processPrelaunch": false,
|
||||
"dom.ipc.contentProcessCount": this.contentProcesses,
|
||||
|
||||
// Telemetry
|
||||
"datareporting.healthreport.uploadEnabled": false,
|
||||
"datareporting.policy.dataSubmissionEnabled": false,
|
||||
"datareporting.policy.dataSubmissionURL": "",
|
||||
"toolkit.telemetry.enabled": false,
|
||||
"toolkit.telemetry.unified": false,
|
||||
"toolkit.telemetry.archive": false,
|
||||
"toolkit.telemetry.updatePing.enabled": false,
|
||||
"app.crashreporter": false,
|
||||
"breakpad.reportURL": "",
|
||||
"breakpad.submitReportURL": "",
|
||||
|
||||
// Disabled Firefox services — not used in headless scraping
|
||||
"browser.safebrowsing.downloads.enabled": false,
|
||||
"browser.safebrowsing.malware.enabled": false,
|
||||
"extensions.update.enabled": false,
|
||||
"extensions.systemAddon.update.url": "",
|
||||
"browser.fixup.alternate.enabled": false,
|
||||
"app.normandy.enabled": false,
|
||||
"app.shield.optoutstudies.enabled": false,
|
||||
"network.connectivity-service.enabled": false,
|
||||
"network.captive-portal-service.enabled": false,
|
||||
"network.prefetch-next": false,
|
||||
"beacon.enabled": false,
|
||||
"security.OCSP.enabled": 0,
|
||||
"network.http.tls-handshake-timeout": 30,
|
||||
"network.http.connection-timeout": 60,
|
||||
"network.http.response.timeout": 120,
|
||||
|
||||
// UI chrome features a scraper never sees
|
||||
"extensions.screenshots.system.enabled": false,
|
||||
"extensions.screenshots.background.enabled": false,
|
||||
"browser.sessionstore.max_tabs_undo": 0,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@trawl/tiers",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
|
||||
@@ -4,7 +4,8 @@ import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } fro
|
||||
import { runTier1 } from "./tiers/1"
|
||||
import { runTier2 } from "./tiers/2"
|
||||
import { runTier3 } from "./tiers/3"
|
||||
import { runTier4 } from "./tiers/4"
|
||||
// Tier 4 (residential proxy) is dynamically imported only when needed.
|
||||
import type { runTier4 } from "./tiers/4"
|
||||
import { normalizeHtml } from "./utils/html"
|
||||
import type { ProxyPool } from "./utils/proxyRotator"
|
||||
import { requireContentTypeForBody, sanitizeHeaders } from "./utils/sanitize"
|
||||
@@ -204,10 +205,11 @@ export async function scrape(req: ScrapeRequest, deps: OrchestratorDeps): Promis
|
||||
}
|
||||
|
||||
let t4: Awaited<ReturnType<typeof runTier4>>
|
||||
const { runTier4: runTier4Lazy } = await import("./tiers/4")
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
console.log(`[orchestrator] Tier 4 via residential proxy: ${proxy4.replace(/\/\/[^@]*@/, "//**@")}`)
|
||||
const remaining4 = maxTimeout - (Date.now() - totalStart)
|
||||
t4 = await runTier4(req.url, handle, remaining4, proxy4, sanitizedHeaders, req.method, req.body)
|
||||
t4 = await runTier4Lazy(req.url, handle, remaining4, proxy4, sanitizedHeaders, req.method, req.body)
|
||||
|
||||
// Mirror Tier 3's recycle-on-suspect policy — only flag when the upstream
|
||||
// explicitly rejected the browser's profile.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@trawl/types",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"private": true,
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# E2E probe suite for TRAWL — covers every supported verb, sanitisation,
|
||||
# validation, and the legacy FlareSolverr shape. Run while the container is up.
|
||||
set -uo pipefail
|
||||
BASE="${BASE:-http://localhost:8191}"
|
||||
ok=0
|
||||
fail=0
|
||||
check() {
|
||||
local name="$1" expected="$2" actual="$3"
|
||||
if [[ "$actual" == "$expected" ]]; then
|
||||
printf " \033[32mOK\033[0m %-60s -> %s\n" "$name" "$actual"
|
||||
ok=$((ok+1))
|
||||
else
|
||||
printf " \033[31mFAIL\033[0m %-60s -> got %s, want %s\n" "$name" "$actual" "$expected"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
|
||||
probe() {
|
||||
local name="$1" expected="$2"; shift 2
|
||||
local code
|
||||
code=$(curl -s -o /tmp/probe-body -w "%{http_code}" "$@")
|
||||
check "$name" "$expected" "$code"
|
||||
if [[ "$code" != "$expected" ]]; then
|
||||
echo " body: $(head -c 300 /tmp/probe-body)"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== 1. health & stats ==="
|
||||
probe "/health -> 200" 200 -X GET "$BASE/health"
|
||||
probe "/stats -> 200" 200 -X GET "$BASE/stats"
|
||||
|
||||
echo
|
||||
echo "=== 2. native /scrape — full verb set (upstream method in JSON body) ==="
|
||||
# /scrape is registered as POST at the HTTP layer. The upstream HTTP verb the
|
||||
# scraper uses is passed inside the JSON body (`method` field). All upstream
|
||||
# verbs except GET must declare Content-Type upstream-side via `headers`.
|
||||
for verb in GET POST PUT PATCH DELETE HEAD OPTIONS TRACE QUERY; do
|
||||
case "$verb" in
|
||||
GET|HEAD|OPTIONS|TRACE)
|
||||
payload=$(printf '{"url":"https://example.com/","method":"%s"}' "$verb")
|
||||
expected=200
|
||||
;;
|
||||
*)
|
||||
payload=$(printf '{"url":"https://example.com/","method":"%s","body":"k=v","headers":{"Content-Type":"application/x-www-form-urlencoded"}}' "$verb")
|
||||
expected=200
|
||||
;;
|
||||
esac
|
||||
probe "/scrape upstream-method=$verb -> $expected" "$expected" \
|
||||
-X POST "$BASE/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data "$payload"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== 3. native /scrape — negative ==="
|
||||
probe "/scrape CONNECT -> 400" 400 \
|
||||
-X POST "$BASE/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"url":"https://example.com/","method":"CONNECT"}'
|
||||
probe "/scrape POST no Content-Type -> 400" 400 \
|
||||
-X POST "$BASE/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"url":"https://example.com/","method":"POST","body":"k=v","headers":{}}'
|
||||
probe "/scrape Host header (sanitised) -> 200" 200 \
|
||||
-X POST "$BASE/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"url":"https://example.com/","headers":{"Host":"evil.example","X-Custom":"keep-me"}}'
|
||||
|
||||
echo
|
||||
echo "=== 4. legacy /v1 (FlareSolverr compat) ==="
|
||||
probe "/v1 request.get -> 200" 200 \
|
||||
-X POST "$BASE/v1" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"cmd":"request.get","url":"https://example.com/"}'
|
||||
probe "/v1 request.post + Content-Type -> 200" 200 \
|
||||
-X POST "$BASE/v1" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{"Content-Type":"application/x-www-form-urlencoded"}}'
|
||||
probe "/v1 request.post no Content-Type -> 400" 400 \
|
||||
-X POST "$BASE/v1" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"cmd":"request.post","url":"https://example.com/post","postData":"hello=world","headers":{}}'
|
||||
probe "/v1 unknown cmd -> 400" 400 \
|
||||
-X POST "$BASE/v1" \
|
||||
-H 'content-type: application/json' \
|
||||
--data '{"cmd":"request.delete","url":"https://example.com/"}'
|
||||
|
||||
echo
|
||||
echo "=== 5. large body — no cap (~500 KiB) ==="
|
||||
big=$(python3 -c "import string,random; random.seed(0); print(''.join(random.choices(string.ascii_letters+string.digits,k=512000)),end='')")
|
||||
probe "/scrape POST large body (uncapped) -> 200" 200 \
|
||||
-X POST "$BASE/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data "$(python3 -c "import json,sys; print(json.dumps({'url':'https://example.com/','method':'POST','body':sys.argv[1],'headers':{'Content-Type':'text/plain'}}))" "$big")"
|
||||
|
||||
echo
|
||||
echo "-----------------------------------------------------"
|
||||
echo "Total: $ok passed, $fail failed"
|
||||
[[ $fail -eq 0 ]] || exit 1
|
||||
@@ -1,82 +0,0 @@
|
||||
// Direct end-to-end test of the new hCaptcha audio STT fallback.
|
||||
// Uses the production BrowserPool from @trawl/browser (same code path the API uses)
|
||||
// so bun's module resolution can't trip over camoufox-js's internal playwright-core dep.
|
||||
//
|
||||
// Run inside the running container:
|
||||
// docker exec -w /app trawl-hcaptcha-test bun run /tmp/hcaptcha-audio-test.ts
|
||||
|
||||
import { BrowserPool } from "@trawl/browser"
|
||||
import { solveHcaptcha } from "../packages/tiers/src/solvers/hcaptcha.ts"
|
||||
|
||||
process.on("uncaughtException", (err) => {
|
||||
console.error("[uncaught]", err.message)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
async function main() {
|
||||
const pool = new BrowserPool({
|
||||
size: 1,
|
||||
headless: true,
|
||||
geoip: true,
|
||||
humanize: true,
|
||||
block_webrtc: true,
|
||||
disable_coop: true,
|
||||
})
|
||||
|
||||
console.log("[test] initializing BrowserPool (size=1)...")
|
||||
await pool.init()
|
||||
console.log("[test] pool ready")
|
||||
|
||||
const handle = await pool.acquire("nopecha.com")
|
||||
console.log(`[test] acquired browser handle id=${handle.id}`)
|
||||
|
||||
// BrowserHandle exposes context + browser, not page — create one from the context
|
||||
// (same pattern apps/api/src/index.ts uses).
|
||||
const page = await handle.context.newPage()
|
||||
|
||||
try {
|
||||
const target = "https://nopecha.com/demo/hcaptcha"
|
||||
console.log(`[test] visiting ${target}...`)
|
||||
await page
|
||||
.goto(target, { waitUntil: "domcontentloaded", timeout: 30000 })
|
||||
.catch((e: Error) => console.log("[test] goto error:", e.message.slice(0, 120)))
|
||||
|
||||
// Give hCaptcha's api.js time to bootstrap the widget iframe.
|
||||
console.log("[test] waiting 8s for hCaptcha widget to render...")
|
||||
await new Promise((r) => setTimeout(r, 8000))
|
||||
await page.waitForLoadState("networkidle", { timeout: 15000 }).catch(() => {})
|
||||
|
||||
const html = await page.content().catch(() => "")
|
||||
console.log("[test] page.content() length:", html.length)
|
||||
console.log("[test] URL:", page.url())
|
||||
console.log("[test] Title:", await page.title().catch(() => "?"))
|
||||
|
||||
const frames = page
|
||||
.frames()
|
||||
.map((f: { url: () => string }) => f.url())
|
||||
.filter((u: string) => u && u !== "about:blank")
|
||||
console.log("[test] frames:", frames.slice(0, 10))
|
||||
|
||||
// ─── Run the new solver ────────────────────────────────────────────────
|
||||
console.log("[test] calling solveHcaptcha(page, 45000)...")
|
||||
const t0 = Date.now()
|
||||
const solved = await solveHcaptcha(page, 45_000)
|
||||
const elapsed = Date.now() - t0
|
||||
console.log(`[test] solveHcaptcha returned: ${solved} (took ${elapsed}ms)`)
|
||||
|
||||
await pool.release(handle.id)
|
||||
await pool.shutdown()
|
||||
console.log("[test] DONE")
|
||||
process.exit(solved ? 0 : 2)
|
||||
} catch (e) {
|
||||
console.error("[test] FAIL:", (e as Error).message)
|
||||
await pool.release(handle.id).catch(() => {})
|
||||
await pool.shutdown().catch(() => {})
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e: Error) => {
|
||||
console.error("[test] FATAL:", e.message)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -1,210 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Soak test for TRAWL — repeatable perf measurement for browser-pool changes.
|
||||
#
|
||||
# Builds the image from the current HEAD, starts a container on an isolated port
|
||||
# (default 8192), runs N forced Tier 3 scrapes to a configurable target URL while
|
||||
# sampling thread/process/CPU/MEM every 2 seconds, then prints a summary table.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/soak-test.sh # defaults: 25 scrapes, eztvx.to, port 8192
|
||||
# SCRAPES=50 TARGET=https://nopecha.com/demo/cloudflare scripts/soak-test.sh
|
||||
# POOL_SIZE=1 RECYCLE_AFTER=8 CONTENT_PROCS=2 scripts/soak-test.sh
|
||||
#
|
||||
# Outputs:
|
||||
# - Console table with p50/p95/p99 latencies, 429 count, thread/process peaks
|
||||
# - /tmp/soak/<scenario>/latencies.txt — per-request timing
|
||||
# - /tmp/soak/<scenario>/metrics.csv — sampled thread/proc/CPU/MEM every 2s
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 = soak ran to completion (regardless of 429 count, see below)
|
||||
# 1 = container failed to start healthy
|
||||
# 2 = build failed
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRAPES=${SCRAPES:-25}
|
||||
TARGET=${TARGET:-https://eztvx.to/home}
|
||||
PORT=${PORT:-8192}
|
||||
POOL_SIZE=${POOL_SIZE:-1}
|
||||
RECYCLE_AFTER=${RECYCLE_AFTER:-8}
|
||||
CONTENT_PROCS=${CONTENT_PROCS:-2}
|
||||
GITHUB_TOKEN=${GITHUB_TOKEN:-$(gh auth token 2>/dev/null || echo "")}
|
||||
|
||||
SCENARIO="${POOL_SIZE}p-r${RECYCLE_AFTER}-c${CONTENT_PROCS}"
|
||||
OUT_DIR="/tmp/soak/${SCENARIO}"
|
||||
CONTAINER_NAME="trawl-soak-${SCENARIO}"
|
||||
IMAGE_TAG="trawl:soak-${SCENARIO}"
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
rm -f "$OUT_DIR/metrics.csv" "$OUT_DIR/latencies.txt"
|
||||
|
||||
echo "=== Soak test scenario ==="
|
||||
echo " scrapes: $SCRAPES"
|
||||
echo " target: $TARGET"
|
||||
echo " port: $PORT"
|
||||
echo " pool_size: $POOL_SIZE"
|
||||
echo " recycle_after: $RECYCLE_AFTER"
|
||||
echo " content_procs: $CONTENT_PROCS"
|
||||
echo " container: $CONTAINER_NAME"
|
||||
echo " output: $OUT_DIR"
|
||||
echo
|
||||
|
||||
# 1. Build image
|
||||
echo "=== Building image ==="
|
||||
if ! DOCKER_BUILDKIT=1 docker build \
|
||||
-f apps/api/Dockerfile \
|
||||
-t "$IMAGE_TAG" \
|
||||
--secret id=GITHUB_TOKEN,env=GITHUB_TOKEN \
|
||||
. > "$OUT_DIR/build.log" 2>&1; then
|
||||
echo "BUILD FAILED — see $OUT_DIR/build.log"
|
||||
exit 2
|
||||
fi
|
||||
echo " built: $IMAGE_TAG"
|
||||
|
||||
# 2. Clean previous run
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null
|
||||
|
||||
# 3. Start container
|
||||
echo "=== Starting container ==="
|
||||
docker run -d \
|
||||
--name "$CONTAINER_NAME" \
|
||||
--rm \
|
||||
-p "${PORT}:8191" \
|
||||
--shm-size=1gb \
|
||||
-e "BROWSER_POOL_SIZE=${POOL_SIZE}" \
|
||||
-e "BROWSER_RECYCLE_AFTER_CONTEXTS=${RECYCLE_AFTER}" \
|
||||
-e "BROWSER_CONTENT_PROCESSES=${CONTENT_PROCS}" \
|
||||
"$IMAGE_TAG" > /dev/null
|
||||
|
||||
# 4. Wait for healthy
|
||||
for i in $(seq 1 60); do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/health" 2>/dev/null || echo "000")
|
||||
if [[ "$code" == "200" ]]; then
|
||||
echo " healthy after $((i*2))s"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ "$code" != "200" ]]; then
|
||||
echo "CONTAINER FAILED TO BECOME HEALTHY"
|
||||
docker logs --tail 50 "$CONTAINER_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Capture baseline metrics
|
||||
echo "=== Baseline ==="
|
||||
baseline_threads=$(docker exec "$CONTAINER_NAME" ps -eLf 2>/dev/null | wc -l)
|
||||
baseline_procs=$(docker exec "$CONTAINER_NAME" ps -ef 2>/dev/null | wc -l)
|
||||
echo " threads: $baseline_threads"
|
||||
echo " procs: $baseline_procs"
|
||||
echo
|
||||
|
||||
# 6. Start metrics sampler in background
|
||||
(
|
||||
while true; do
|
||||
ts=$(date +%s.%N)
|
||||
threads=$(docker exec "$CONTAINER_NAME" ps -eLf 2>/dev/null | wc -l)
|
||||
procs=$(docker exec "$CONTAINER_NAME" ps -ef 2>/dev/null | wc -l)
|
||||
cpu_mem=$(docker stats "$CONTAINER_NAME" --no-stream --format '{{.CPUPerc}} {{.MemUsage}}' 2>/dev/null || echo "N/A")
|
||||
restarts=$(curl -s "http://localhost:${PORT}/health" 2>/dev/null \
|
||||
| python3 -c 'import json,sys; print(json.loads(sys.stdin.read())["pool"]["restarts"])' 2>/dev/null || echo "err")
|
||||
echo "$ts,$threads,$procs,$cpu_mem,$restarts" >> "$OUT_DIR/metrics.csv"
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
SAMPLER_PID=$!
|
||||
|
||||
sleep 2
|
||||
|
||||
# 7. Run scrapes
|
||||
echo "=== Running $SCRAPES scrapes ==="
|
||||
for i in $(seq 1 "$SCRAPES"); do
|
||||
start=$(date +%s.%N)
|
||||
resp=$(curl -s --max-time 90 -X POST "http://localhost:${PORT}/scrape" \
|
||||
-H 'content-type: application/json' \
|
||||
--data "{\"url\":\"$TARGET\",\"skipHttp\":true,\"maxTier\":3,\"maxTimeout\":60000}" 2>/dev/null)
|
||||
end=$(date +%s.%N)
|
||||
elapsed=$(echo "$end - $start" | bc)
|
||||
|
||||
http_code=$(echo "$resp" | python3 -c '
|
||||
import json,sys
|
||||
try:
|
||||
d=json.loads(sys.stdin.read(), strict=False)
|
||||
# /scrape returns tier+status on success, status:error envelope on 429
|
||||
if "status" in d and d["status"] == "error":
|
||||
print("429")
|
||||
else:
|
||||
print("200")
|
||||
except Exception:
|
||||
print("err")
|
||||
' 2>/dev/null || echo "err")
|
||||
|
||||
echo "$i elapsed=${elapsed}s code=$http_code" | tee -a "$OUT_DIR/latencies.txt"
|
||||
sleep 1
|
||||
done
|
||||
echo
|
||||
|
||||
sleep 3
|
||||
kill $SAMPLER_PID 2>/dev/null
|
||||
|
||||
# 8. Capture final state
|
||||
echo "=== Final state ==="
|
||||
final_health=$(curl -s "http://localhost:${PORT}/health")
|
||||
echo " $final_health"
|
||||
final_threads=$(docker exec "$CONTAINER_NAME" ps -eLf 2>/dev/null | wc -l)
|
||||
echo " threads: $final_threads"
|
||||
echo
|
||||
|
||||
# 9. Summary
|
||||
echo "=== Summary ==="
|
||||
python3 << EOF
|
||||
import re, csv
|
||||
|
||||
# Latencies
|
||||
lats = []
|
||||
with open("$OUT_DIR/latencies.txt") as f:
|
||||
for line in f:
|
||||
m = re.search(r"elapsed=([\d.]+)s\s+code=(\S+)", line)
|
||||
if m:
|
||||
lats.append((float(m.group(1)), m.group(2)))
|
||||
|
||||
if not lats:
|
||||
print(" no requests completed")
|
||||
else:
|
||||
sorted_lats = sorted(l for l, _ in lats)
|
||||
n = len(sorted_lats)
|
||||
code_counts = {}
|
||||
for _, c in lats:
|
||||
code_counts[c] = code_counts.get(c, 0) + 1
|
||||
def pct(p):
|
||||
return sorted_lats[min(int(n * p), n-1)]
|
||||
fail = sum(1 for _, c in lats if c != "200")
|
||||
print(f" requests: {n} ({fail} non-200, {100*fail/n:.0f}%)")
|
||||
print(f" code counts: {code_counts}")
|
||||
print(f" latency min: {sorted_lats[0]:.2f}s")
|
||||
print(f" latency p50: {pct(0.50):.2f}s")
|
||||
print(f" latency p95: {pct(0.95):.2f}s")
|
||||
print(f" latency p99: {pct(0.99):.2f}s")
|
||||
print(f" latency max: {sorted_lats[-1]:.2f}s")
|
||||
print(f" latency avg: {sum(sorted_lats)/n:.2f}s")
|
||||
|
||||
# Metrics
|
||||
with open("$OUT_DIR/metrics.csv") as f:
|
||||
rows = list(csv.reader(f))
|
||||
threads = [int(r[1]) for r in rows if len(r) > 1]
|
||||
procs = [int(r[2]) for r in rows if len(r) > 2]
|
||||
restarts = [int(r[4]) for r in rows if len(r) > 4 and r[4].isdigit()]
|
||||
if threads:
|
||||
print(f" threads: first={threads[0]}, min={min(threads)}, max={max(threads)}, last={threads[-1]}")
|
||||
if procs:
|
||||
print(f" procs: first={procs[0]}, min={min(procs)}, max={max(procs)}, last={procs[-1]}")
|
||||
if restarts:
|
||||
print(f" restarts: max={max(restarts)}")
|
||||
EOF
|
||||
|
||||
# 10. Cleanup container (image stays)
|
||||
echo
|
||||
echo "=== Cleanup ==="
|
||||
docker stop "$CONTAINER_NAME" 2>&1 | tail -2
|
||||
echo " image kept: $IMAGE_TAG (delete with: docker rmi $IMAGE_TAG)"
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Burst-test /scrape under concurrent load against a Cloudflare-protected target.
|
||||
# Pool size is typically 3 (per docker-compose.yml BROWSER_POOL_SIZE=3), so
|
||||
# with CONCURRENCY=10 against a slow CF target:
|
||||
# - 3 requests → HTTP 200 with tier=3 (browser used), totalMs ≈ 30–60s
|
||||
# - 7 requests → HTTP 429 with FlareSolverr envelope after ~5s queue timeout
|
||||
# Expected histogram: HTTP 200: 3, HTTP 429: 7.
|
||||
|
||||
# On a target that resolves at Tier 1 (HTTP fetch, no browser), every request
|
||||
# should succeed quickly with tier=1 and no 429s.
|
||||
|
||||
# Configuration
|
||||
TRAWL_URL="http://localhost:8191/scrape"
|
||||
TARGET_URL="https://nopecha.com/demo/cloudflare"
|
||||
MAX_TIMEOUT=60000 # 60 seconds
|
||||
CONCURRENCY=10
|
||||
|
||||
echo "🚀 Starting $CONCURRENCY parallel requests to $TRAWL_URL → $TARGET_URL"
|
||||
|
||||
# Per-request scratch dir to avoid races on shared variables across &-spawned subshells.
|
||||
TMPDIR=$(mktemp -d)
|
||||
trap "rm -rf $TMPDIR" EXIT
|
||||
|
||||
for i in $(seq 1 $CONCURRENCY); do
|
||||
(
|
||||
BODY_FILE="$TMPDIR/$i.body"
|
||||
META_FILE="$TMPDIR/$i.meta"
|
||||
|
||||
# -w emits two whitespace-separated values: HTTP status code + total time (s).
|
||||
META=$(curl -s -o "$BODY_FILE" -w "%{http_code} %{time_total}" \
|
||||
-X POST "$TRAWL_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
--max-time 120 \
|
||||
--data-raw '{
|
||||
"url": "'"$TARGET_URL"'",
|
||||
"maxTimeout": '"$MAX_TIMEOUT"'
|
||||
}')
|
||||
|
||||
HTTP_CODE=$(echo "$META" | awk '{print $1}')
|
||||
TIME=$(echo "$META" | awk '{print $2}')
|
||||
echo "$HTTP_CODE" > "$META_FILE"
|
||||
RESPONSE=$(cat "$BODY_FILE")
|
||||
|
||||
case "$HTTP_CODE" in
|
||||
200)
|
||||
# Native ScrapeResult — extract tier, statusCode, totalMs.
|
||||
# head -1 because tier/statusCode/totalMs appear in nested timings[]
|
||||
# and in redirect HTML bodies, which would leak newlines into the echo.
|
||||
TIER=$(echo "$RESPONSE" | grep -o '"tier":[0-9]*' | head -1 | cut -d: -f2)
|
||||
STATUS_CODE=$(echo "$RESPONSE" | grep -o '"statusCode":[0-9]*' | head -1 | cut -d: -f2)
|
||||
TOTAL_MS=$(echo "$RESPONSE" | grep -o '"totalMs":[0-9]*' | head -1 | cut -d: -f2)
|
||||
echo "✅ [$i/$CONCURRENCY] HTTP 200 time=${TIME}s | tier=$TIER statusCode=$STATUS_CODE totalMs=${TOTAL_MS}ms"
|
||||
;;
|
||||
429)
|
||||
# FlareSolverr envelope — uniform with /v1
|
||||
STATUS=$(echo "$RESPONSE" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
MESSAGE=$(echo "$RESPONSE" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
|
||||
VERSION=$(echo "$RESPONSE" | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
|
||||
echo "⚠️ [$i/$CONCURRENCY] HTTP 429 time=${TIME}s (pool exhausted) | status=$STATUS version=$VERSION message=\"$MESSAGE\""
|
||||
;;
|
||||
503)
|
||||
# Pool initializing — native { error }
|
||||
ERROR=$(echo "$RESPONSE" | grep -o '"error":"[^"]*"' | cut -d'"' -f4)
|
||||
echo "⏳ [$i/$CONCURRENCY] HTTP 503 time=${TIME}s (pool initializing) | error=\"$ERROR\""
|
||||
;;
|
||||
500)
|
||||
# Generic scrape exception — native { error }
|
||||
ERROR=$(echo "$RESPONSE" | grep -o '"error":"[^"]*"' | cut -d'"' -f4)
|
||||
echo "❌ [$i/$CONCURRENCY] HTTP 500 time=${TIME}s | error=\"$ERROR\""
|
||||
;;
|
||||
*)
|
||||
echo "❓ [$i/$CONCURRENCY] HTTP $HTTP_CODE time=${TIME}s (unexpected)"
|
||||
echo " Response preview: ${RESPONSE:0:300}..."
|
||||
;;
|
||||
esac
|
||||
) &
|
||||
done
|
||||
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo "📊 Status histogram:"
|
||||
# Extract HTTP status from each .meta file, then count occurrences.
|
||||
# Avoids bash 4+ associative arrays (macOS /bin/bash is 3.2).
|
||||
{
|
||||
for f in "$TMPDIR"/*.meta; do
|
||||
awk '{print $1}' "$f"
|
||||
done
|
||||
} | sort | uniq -c | awk '{printf " HTTP %s: %d\n", $2, $1}'
|
||||
|
||||
echo ""
|
||||
echo "✅ All $CONCURRENCY requests completed!"
|
||||
Reference in New Issue
Block a user