feat(web): refresh landing page content, comparisons, proxy showcase, and SEO

This commit is contained in:
germondai
2026-08-02 02:34:34 +02:00
parent d899a507e1
commit 357767ce3c
15 changed files with 719 additions and 182 deletions
+86 -22
View File
@@ -1,37 +1,44 @@
<script lang="ts" setup>
useHead({
script: [
{
type: "application/ld+json",
innerHTML: JSON.stringify({
"@context": "https://schema.org",
"@type": "SoftwareApplication",
name: "TRAWL",
url: "https://trawl.dev",
description:
"Self-hosted web scraping engine with adaptive tier execution. Solves Cloudflare challenges natively. Drop-in FlareSolverr replacement.",
applicationCategory: "DeveloperApplication",
operatingSystem: "Linux, Docker",
license: "https://www.gnu.org/licenses/agpl-3.0.html",
author: { "@type": "Person", name: "germondai", url: "https://github.com/germondai" },
codeRepository: "https://github.com/germondai/trawl",
offers: { "@type": "Offer", price: "0", priceCurrency: "USD" },
}),
const description =
"Adaptive, self-hosted web scraping engine for Cloudflare, Akamai, Imperva, and CAPTCHA-protected sites, with a challenge-aware HTTP/HTTPS proxy."
useSchemaOrg([
defineWebPage({
name: "TRAWL — Self-Hosted Web Scraping & Challenge Bypass",
description,
}),
defineSoftwareApp({
name: "TRAWL",
description,
applicationCategory: "DeveloperApplication",
operatingSystem: "Linux, Docker",
license: "https://www.gnu.org/licenses/agpl-3.0.html",
author: {
"@type": "Person",
name: "germondai",
url: "https://github.com/germondai",
},
],
})
offers: {
"@type": "Offer",
price: 0,
priceCurrency: "USD",
},
}),
])
</script>
<template>
<div class="layout">
<a class="skip-link" href="#main-content">Skip to content</a>
<NavBar />
<main>
<main id="main-content" tabindex="-1">
<HeroSection />
<EcosystemBanner />
<StatsBar />
<FeaturesGrid />
<ChallengeGrid />
<TierFlow />
<ProxySection />
<CompareTable />
<CtaSection />
<CodeExample />
@@ -44,6 +51,7 @@ useHead({
<style>
/* ── CSS custom properties ── */
:root {
color-scheme: light;
--bg: #fafafa;
--bg-subtle: #f4f4f5;
--surface: #ffffff;
@@ -58,6 +66,7 @@ useHead({
}
.dark {
color-scheme: dark;
--bg: #0d0d10;
--bg-subtle: #17171b;
--surface: #1c1c21;
@@ -103,6 +112,11 @@ button {
font-family: inherit;
cursor: pointer;
}
:where(a, button, [tabindex]):focus-visible {
outline: 2px solid var(--accent);
outline-offset: 3px;
}
h1,
h2,
h3,
@@ -135,12 +149,43 @@ main {
padding-top: 60px;
}
.skip-link {
position: fixed;
top: 8px;
left: 8px;
z-index: 1000;
padding: 8px 12px;
color: #080808;
background: var(--accent);
transform: translateY(-150%);
}
.skip-link:focus {
transform: translateY(0);
}
/* ── Shared section wrapper ── */
.section {
padding: 80px 0;
border-top: 1px solid var(--border);
}
[id] {
scroll-margin-top: 76px;
}
.sr-only {
position: absolute;
width: 1px;
height: 1px;
padding: 0;
margin: -1px;
overflow: hidden;
clip: rect(0, 0, 0, 0);
white-space: nowrap;
border: 0;
}
.container {
max-width: 1100px;
margin: 0 auto;
@@ -171,7 +216,11 @@ main {
font-size: 12px;
letter-spacing: 0.06em;
cursor: pointer;
transition: all 0.15s;
transition:
color 0.15s,
background-color 0.15s,
border-color 0.15s,
opacity 0.15s;
border: 1px solid transparent;
white-space: nowrap;
}
@@ -205,6 +254,21 @@ main {
opacity: 0;
}
@media (prefers-reduced-motion: reduce) {
html {
scroll-behavior: auto;
}
*,
*::before,
*::after {
scroll-behavior: auto;
animation-duration: 0.01ms;
animation-iteration-count: 1;
transition-duration: 0.01ms;
}
}
/* ── Mobile globals ── */
@media (max-width: 640px) {
.section {
+33 -10
View File
@@ -20,6 +20,26 @@ const challenges = [
byparr: "not handled",
byparrSupport: "no",
},
{
name: "Akamai Bot Manager",
desc: "Behavioral interstitial + press-and-hold",
trawl: "sensor flow · human-like input",
trawlSupport: "yes",
flaresolver: "not handled",
fsSupport: "no",
byparr: "not handled",
byparrSupport: "no",
},
{
name: "Imperva / Incapsula",
desc: "Sensor-cookie WAF challenge",
trawl: "browser sensor flow · best effort",
trawlSupport: "partial",
flaresolver: "not handled",
fsSupport: "no",
byparr: "not handled",
byparrSupport: "no",
},
{
name: "reCAPTCHA v2",
desc: "Google checkbox + audio challenge",
@@ -74,14 +94,17 @@ function iconClass(s: string) {
<section id="captcha" class="section">
<div class="container">
<p class="eyebrow">challenge coverage</p>
<h2 class="section-title">every wall. handled.</h2>
<h2 class="section-title">solve more than Cloudflare.</h2>
<p class="section-sub">
TRAWL is the only self-hosted scraper with native solvers for all major protection layers — not just Cloudflare
interstitials, but the captchas embedded inside the pages themselves.
Native flows cover Cloudflare, Akamai, Imperva, and the captchas embedded inside protected pages — all within
the same self-hosted tier engine.
</p>
<div class="table-wrap">
<table>
<caption class="sr-only">
Challenge handling support across TRAWL, FlareSolverr, and Byparr
</caption>
<thead>
<tr>
<th class="col-name">challenge type</th>
@@ -97,7 +120,7 @@ function iconClass(s: string) {
<span class="challenge-desc">{{ c.desc }}</span>
</td>
<td class="col-trawl">
<span class="cell-yes">✓</span>
<span :class="iconClass(c.trawlSupport)">{{ icon(c.trawlSupport) }}</span>
{{ c.trawl }}
</td>
<td>
@@ -114,16 +137,16 @@ function iconClass(s: string) {
<tr>
<td class="col-name score-spacer"></td>
<td class="col-trawl score-cell">
<span class="score-num accent">6 / 6</span>
<span class="score-label">challenge types</span>
<span class="score-num accent">8</span>
<span class="score-label">challenge paths</span>
</td>
<td class="score-cell">
<span class="score-num">1 / 6</span>
<span class="score-label">challenge types</span>
<span class="score-num">1</span>
<span class="score-label">dedicated path</span>
</td>
<td class="score-cell">
<span class="score-num">1 / 6</span>
<span class="score-label">challenge types</span>
<span class="score-num">1</span>
<span class="score-label">dedicated path</span>
</td>
</tr>
</tfoot>
+14 -5
View File
@@ -39,7 +39,7 @@ curl -s -X POST http://localhost:8191/scrape \\
<span class="kw">const</span> { solution } = <span class="kw">await</span> res.json()
console.log(solution.response) <span class="c">// full HTML</span>
console.log(solution.cookies) <span class="c">// Cloudflare cookies</span>
console.log(solution.cookies) <span class="c">// solved session cookies</span>
console.log(solution.userAgent) <span class="c">// browser UA used</span>`,
python: `<span class="kw">import</span> requests
@@ -105,21 +105,23 @@ curl -s -X POST http://localhost:8191/scrape \\
:visible-once="{ opacity: 1, y: 0, transition: { duration: 400 } }"
class="code-block"
>
<div class="tabs">
<fieldset class="tabs">
<legend class="sr-only">API example language</legend>
<button
type="button"
v-for="tab in tabs"
:key="tab.id"
class="tab"
:class="{ active: active === tab.id }"
:aria-pressed="active === tab.id"
@click="active = tab.id"
>
{{ tab.label }}
</button>
<div class="tab-spacer" />
<span class="tab-hint">POST /v1 · FlareSolverr v2 compat</span>
</div>
<pre class="code-body"><code v-html="code[active]" /></pre>
</fieldset>
<pre class="code-body" aria-live="polite"><code v-html="code[active]" /></pre>
</div>
</div>
</section>
@@ -140,6 +142,10 @@ curl -s -X POST http://localhost:8191/scrape \\
}
.tabs {
min-width: 0;
padding: 0;
margin: 0;
border: 0;
display: flex;
align-items: center;
border-bottom: 1px solid var(--border);
@@ -158,7 +164,10 @@ curl -s -X POST http://localhost:8191/scrape \\
border-right: 1px solid var(--border);
cursor: pointer;
white-space: nowrap;
transition: all 0.12s;
transition:
color 0.12s,
background-color 0.12s,
border-color 0.12s;
}
.tab:hover {
+190 -53
View File
@@ -1,21 +1,24 @@
<script lang="ts" setup>
const barsVisible = shallowRef(false)
const benchGrid = useTemplateRef<HTMLElement>("benchGrid")
let benchmarkObserver: IntersectionObserver
onMounted(() => {
const el = document.querySelector(".bench-grid")
if (!el) return
const observer = new IntersectionObserver(
if (!benchGrid.value) return
benchmarkObserver = new IntersectionObserver(
(entries) => {
if (entries[0]?.isIntersecting) {
barsVisible.value = true
observer.disconnect()
benchmarkObserver.disconnect()
}
},
{ threshold: 0.2 },
)
observer.observe(el)
benchmarkObserver.observe(benchGrid.value)
})
onBeforeUnmount(() => benchmarkObserver?.disconnect())
const benchmarks = [
{
url: "nowsecure.nl",
@@ -49,52 +52,107 @@ const benchmarks = [
},
]
const rows = [
{ feature: "Persistent browser pool", trawl: "✓ N instances", flaresolver: "✗ 1 instance", byparr: "✗ 1 instance" },
{
feature: "Domain session cache",
trawl: "✓ Redis, <500ms repeat",
flaresolver: "✗ always full solve",
byparr: "✗ always full solve",
},
{ feature: "Adaptive tier execution", trawl: "✓ 4 tiers", flaresolver: "✗ always browser", byparr: "~" },
{ feature: "Browser engine", trawl: "✓ Camoufox Firefox", flaresolver: "✗ Chrome", byparr: "✓ Camoufox Firefox" },
{ feature: "Docker image size", trawl: "✓ 1.17 GB", flaresolver: "✓ 1.03 GB", byparr: "✗ 4.10 GB" },
{ feature: "Memory footprint (peak)", trawl: "✓ 770 MB", flaresolver: "✓ 500 MB", byparr: "✗ 1.35 GB" },
{ feature: "Cloudflare challenge speed", trawl: "✓ 4–15s", flaresolver: "✗ 11–18s", byparr: "✗ 13–18s" },
{ feature: "CF Turnstile solving", trawl: "✓ shadow DOM click", flaresolver: "✗", byparr: "✗" },
{ feature: "reCAPTCHA v2 solving", trawl: "✓ audio STT (free)", flaresolver: "✗", byparr: "✗" },
{ feature: "hCaptcha solving", trawl: "✓ auto-pass", flaresolver: "✗", byparr: "✗" },
{ feature: "GeeTest v4 solving", trawl: "✓ canvas gap detection", flaresolver: "✗", byparr: "✗" },
{ feature: "Custom headers (all tiers)", trawl: "✓ all 4 tiers", flaresolver: "✗", byparr: "✗" },
{ feature: "Proxy escalation", trawl: "✓ DC + residential", flaresolver: "✗", byparr: "~" },
{ feature: "Self-healing pool", trawl: "✓", flaresolver: "✗", byparr: "✗" },
{ feature: "FlareSolverr v2 compat", trawl: "✓", flaresolver: "✓ native", byparr: "✓" },
{
feature: "External solver APIs",
trawl: "✓ none required",
flaresolver: "✓ none required",
byparr: "✓ none required",
},
{ feature: "Self-hosted", trawl: "✓", flaresolver: "✓", byparr: "✓" },
]
function renderCell(val: string) {
if (val.startsWith("✓")) return `<span class="check">${val}</span>`
if (val.startsWith("✗")) return `<span class="cross">${val}</span>`
if (val.startsWith("~")) return `<span class="partial">${val}</span>`
return val
type CellTone = "check" | "cross" | "partial"
interface ComparisonCell {
text: string
tone: CellTone
}
const statusMarks: Record<CellTone, string> = { check: "✓", partial: "~", cross: "✗" }
const cell = (tone: CellTone, text: string): ComparisonCell => ({ text, tone })
const rows = [
{
icon: "⬡",
feature: "Adaptive routing",
trawl: cell("check", "HTTP → cache → browser → residential"),
flaresolver: cell("cross", "Browser only"),
byparr: cell("cross", "Browser only"),
},
{
icon: "⇄",
feature: "Challenge proxy",
trawl: cell("check", "HTTP/S, WebSockets, Range"),
flaresolver: cell("cross", "API only"),
byparr: cell("cross", "API only"),
},
{
icon: "↝",
feature: "Proxy escalation",
trawl: cell("check", "DC → residential"),
flaresolver: cell("partial", "Manual proxy"),
byparr: cell("partial", "Manual proxy"),
},
{
icon: "◎",
feature: "Session reuse",
trawl: cell("check", "Redis by domain"),
flaresolver: cell("partial", "Manual sessions"),
byparr: cell("cross", "No domain cache"),
},
{
icon: "▣",
feature: "Browser pool",
trawl: cell("check", "Configurable and warm"),
flaresolver: cell("partial", "Temporary or session"),
byparr: cell("cross", "New browser per request"),
},
{
icon: "⟳",
feature: "Pool lifecycle",
trawl: cell("check", "Health checks + recycling"),
flaresolver: cell("cross", "No warm pool"),
byparr: cell("cross", "No warm pool"),
},
{
icon: "⊕",
feature: "Custom headers",
trawl: cell("check", "Safe across all tiers"),
flaresolver: cell("cross", "Not supported"),
byparr: cell("cross", "Not supported"),
},
{
icon: "◈",
feature: "Browser engine",
trawl: cell("check", "Camoufox"),
flaresolver: cell("partial", "Chrome + UDC"),
byparr: cell("check", "Camoufox"),
},
{
icon: "◇",
feature: "/v1 API",
trawl: cell("check", "GET + POST"),
flaresolver: cell("check", "Native"),
byparr: cell("partial", "GET only"),
},
{
icon: "○",
feature: "Paid solver APIs",
trawl: cell("check", "Not required"),
flaresolver: cell("check", "Not required"),
byparr: cell("check", "Not required"),
},
{
icon: "⌂",
feature: "Self-hosted",
trawl: cell("check", "Yes"),
flaresolver: cell("check", "Yes"),
byparr: cell("check", "Yes"),
},
]
</script>
<template>
<section id="compare" class="section">
<div class="container">
<p class="eyebrow">benchmarks</p>
<h2 class="section-title">TRAWL vs the alternatives.</h2>
<p class="section-sub">Measured on the same machine, same network, same target URLs.</p>
<h2 class="section-title">Less browser overhead. More ways through.</h2>
<p class="section-lead">
Compare real request timings and the paths each engine can take—from direct HTTP to full browser solving and
proxy escalation.
</p>
<div class="bench-grid">
<div ref="benchGrid" class="bench-grid">
<div
v-for="b in benchmarks"
:key="b.url"
@@ -121,8 +179,23 @@ function renderCell(val: string) {
</div>
</div>
<p class="benchmark-note">
Measurements recorded on the same machine and network. They are illustrative, not guarantees; live results vary
with target protection, IP reputation, software versions, and session state. Capability rows were reviewed
against the upstream projects.
</p>
<div class="table-wrap" style="margin-top: 56px;">
<table>
<caption class="sr-only">
Feature comparison across TRAWL, FlareSolverr, and Byparr
</caption>
<colgroup>
<col class="feature-width" />
<col class="trawl-width" />
<col class="competitor-width" />
<col class="competitor-width" />
</colgroup>
<thead>
<tr>
<th>capability</th>
@@ -133,10 +206,25 @@ function renderCell(val: string) {
</thead>
<tbody>
<tr v-for="row in rows" :key="row.feature">
<td class="feature-col">{{ row.feature }}</td>
<td class="col-trawl" v-html="renderCell(row.trawl)" />
<td v-html="renderCell(row.flaresolver)" />
<td v-html="renderCell(row.byparr)" />
<td class="feature-col">
<span class="feature-icon" aria-hidden="true">{{ row.icon }}</span>{{ row.feature }}
</td>
<td class="col-trawl">
<span :class="row.trawl.tone"
><span class="status-mark">{{ statusMarks[row.trawl.tone] }}</span>{{ row.trawl.text }}</span
>
</td>
<td>
<span :class="row.flaresolver.tone"
><span class="status-mark">{{ statusMarks[row.flaresolver.tone] }}</span>
{{ row.flaresolver.text }}</span
>
</td>
<td>
<span :class="row.byparr.tone"
><span class="status-mark">{{ statusMarks[row.byparr.tone] }}</span>{{ row.byparr.text }}</span
>
</td>
</tr>
</tbody>
</table>
@@ -154,11 +242,22 @@ function renderCell(val: string) {
margin-bottom: 12px;
}
.section-sub {
.section-lead {
font-size: 13px;
line-height: 1.75;
color: var(--text-muted);
max-width: 540px;
max-width: 650px;
margin-bottom: 24px;
}
.benchmark-note {
color: var(--text-muted);
font-size: 10px;
line-height: 1.6;
}
.benchmark-note {
max-width: 760px;
margin-bottom: 48px;
}
@@ -251,13 +350,26 @@ function renderCell(val: string) {
table {
width: 100%;
table-layout: fixed;
border-collapse: collapse;
font-size: 12px;
}
.feature-width {
width: 19%;
}
.trawl-width {
width: 31%;
}
.competitor-width {
width: 25%;
}
th,
td {
padding: 12px 18px;
padding: 11px 14px;
text-align: left;
border-bottom: 1px solid var(--border);
white-space: nowrap;
@@ -282,6 +394,20 @@ td {
white-space: normal;
}
.feature-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 18px;
height: 18px;
margin-right: 8px;
color: var(--accent);
font-size: 11px;
font-weight: 600;
line-height: 1;
vertical-align: middle;
}
.col-trawl {
background: var(--accent-tint);
}
@@ -296,21 +422,32 @@ tr:hover .col-trawl {
background: color-mix(in srgb, var(--accent-tint) 150%, transparent);
}
:deep(.check) {
.check {
color: var(--accent);
font-weight: 600;
}
:deep(.cross) {
.cross {
color: var(--text-muted);
opacity: 0.4;
}
:deep(.partial) {
.partial {
color: #f59e0b;
}
.status-mark {
display: inline-block;
width: 16px;
font-weight: 700;
}
@media (max-width: 800px) {
.bench-grid {
grid-template-columns: 1fr;
}
table {
min-width: 860px;
table-layout: auto;
}
}
</style>
+54 -22
View File
@@ -22,8 +22,8 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
class="cta-inner"
>
<div class="cta-label">open source · AGPL-3.0 · self-hosted</div>
<h2 class="cta-heading">up and running<br />in 60 seconds.</h2>
<p class="cta-sub">No clone. No build. Pull the image and start.</p>
<h2 class="cta-heading">scrape protected sites<br />in 60 seconds.</h2>
<p class="cta-sub">No clone. No build. Pull the Docker image and send your first request.</p>
<div class="cta-actions">
<a href="https://github.com/germondai/trawl" target="_blank" rel="noopener noreferrer" class="btn-primary">
<span class="btn-icon">↗</span>
@@ -35,18 +35,20 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
<!-- Step 1: Setup -->
<div class="step-label">1 — choose your setup</div>
<div class="cta-snippet">
<div class="setup-tabs">
<fieldset class="setup-tabs">
<legend class="sr-only">Installation method</legend>
<button
type="button"
v-for="tab in tabs"
:key="tab.id"
class="setup-tab"
:class="{ active: active === tab.id }"
:aria-pressed="active === tab.id"
@click="active = tab.id"
>
{{ tab.label }}
</button>
</div>
</fieldset>
<div v-if="active === 'docker'" class="snippet-body-wrap">
<div class="snippet-bar inner-bar">
@@ -61,64 +63,85 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
<div v-else-if="active === 'minimal'" class="snippet-body-wrap">
<div class="snippet-bar inner-bar">
<span class="snippet-hint">single service · pool size 1 · 15s acquire wait · no redis</span>
<span class="snippet-hint">docker-compose.minimal.yml · single service · no redis</span>
</div>
<pre class="snippet-body"><code><span class="k">services:</span>
trawl:
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
<span class="k">ports:</span>
- <span class="s">"${PORT:-8191}:8191"</span>
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
<span class="k">shm_size:</span> <span class="s">1gb</span>
<span class="k">environment:</span>
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">1</span>
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span></code></pre>
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
<span class="k">volumes:</span>
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
<span class="k">healthcheck:</span>
<span class="k">test:</span> [<span class="s">"CMD"</span>, <span class="s">"curl"</span>, <span class="s">"-sf"</span>, <span class="s">"http://localhost:8191/health"</span>]
<span class="k">volumes:</span>
trawl_proxy_ca:</code></pre>
</div>
<div v-else-if="active === 'cached'" class="snippet-body-wrap">
<div class="snippet-bar inner-bar">
<span class="snippet-hint">redis session cache · repeat requests ~500ms</span>
<span class="snippet-hint">docker-compose.yml · redis cache · pool size 1</span>
</div>
<pre class="snippet-body"><code><span class="k">services:</span>
redis:
<span class="k">image:</span> <span class="s">redis:7-alpine</span>
<span class="k">image:</span> <span class="s">redis:8.8-alpine</span>
<span class="k">volumes:</span> [<span class="s">redis_data:/data</span>]
trawl:
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
<span class="k">ports:</span>
- <span class="s">"${PORT:-8191}:8191"</span>
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
<span class="k">shm_size:</span> <span class="s">1gb</span>
<span class="k">environment:</span>
<span class="k">REDIS_URL:</span> <span class="s">redis://redis:6379</span>
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">3</span>
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span>
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">${BROWSER_POOL_SIZE:-1}</span>
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
<span class="k">volumes:</span>
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
<span class="k">depends_on:</span> [<span class="s">redis</span>]
<span class="k">volumes:</span>
redis_data:</code></pre>
redis_data:
trawl_proxy_ca:</code></pre>
</div>
<div v-else-if="active === 'production'" class="snippet-body-wrap">
<div class="snippet-bar inner-bar">
<span class="snippet-hint">auto-restart · memory limit · healthcheck</span>
<span class="snippet-hint">docker-compose.prod.yml · pool size 3 · restart + healthcheck</span>
</div>
<pre class="snippet-body"><code><span class="k">services:</span>
redis:
<span class="k">image:</span> <span class="s">redis:7-alpine</span>
<span class="k">image:</span> <span class="s">redis:8.8-alpine</span>
<span class="k">restart:</span> <span class="s">always</span>
<span class="k">volumes:</span> [<span class="s">redis_data:/data</span>]
trawl:
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
<span class="k">restart:</span> <span class="s">always</span>
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
<span class="k">ports:</span>
- <span class="s">"${PORT:-8191}:8191"</span>
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
<span class="k">shm_size:</span> <span class="s">1gb</span>
<span class="k">mem_limit:</span> <span class="s">3g</span>
<span class="k">environment:</span>
<span class="k">REDIS_URL:</span> <span class="s">redis://redis:6379</span>
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">5</span>
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span>
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">3</span>
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
<span class="k">volumes:</span>
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
<span class="k">depends_on:</span> [<span class="s">redis</span>]
<span class="k">healthcheck:</span>
<span class="k">test:</span> <span class="s">wget -qO- http://localhost:8191/health</span>
<span class="k">test:</span> [<span class="s">"CMD"</span>, <span class="s">"curl"</span>, <span class="s">"-sf"</span>, <span class="s">"http://localhost:8191/health"</span>]
<span class="k">interval:</span> <span class="s">30s</span>
<span class="k">volumes:</span>
redis_data:</code></pre>
redis_data:
trawl_proxy_ca:</code></pre>
</div>
</div>
@@ -218,7 +241,9 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
font-size: 12px;
letter-spacing: 0.05em;
text-decoration: none;
transition: all 0.12s;
transition:
color 0.12s,
border-color 0.12s;
}
.btn-ghost:hover {
border-color: var(--text-muted);
@@ -240,6 +265,10 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
}
.setup-tabs {
min-width: 0;
padding: 0;
margin: 0;
border: 0;
display: flex;
border-bottom: 1px solid var(--border);
background: var(--bg-subtle);
@@ -257,7 +286,10 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
border-right: 1px solid var(--border);
cursor: pointer;
white-space: nowrap;
transition: all 0.12s;
transition:
color 0.12s,
background-color 0.12s,
border-color 0.12s;
}
.setup-tab:hover {
color: var(--text);
+1 -1
View File
@@ -19,7 +19,7 @@ const apps = [
>{{ app.name }}</span
>
</div>
<span class="eco-label">— plug in, zero config changes</span>
<span class="eco-label">— connect with a single URL change</span>
</div>
</div>
</template>
+26 -7
View File
@@ -7,6 +7,13 @@ const features = [
featured: true,
desc: "Every request takes the cheapest path first — plain HTTP, then cached session, then live browser solve, then residential proxy. You pay the full browser cost only when you have to.",
},
{
icon: "⇄",
title: "General HTTP/HTTPS Proxy",
badge: "v1.2+",
featured: true,
desc: "Proxy ordinary HTTP, HTTPS, WebSockets, binary files, and Range requests through port 8192. Direct traffic stays lightweight; detected challenge pages escalate through TRAWL's solver tiers.",
},
{
icon: "◈",
title: "Camoufox Firefox",
@@ -16,26 +23,26 @@ const features = [
},
{
icon: "⟁",
title: "Cloudflare Bypass",
title: "Cloudflare + WAF Bypass",
badge: "fastest",
desc: "Fresh browser context triggers CF managed-mode evaluation — solves in 4–15s vs 11–18s with alternatives. Cookies are extracted and cached immediately after the first solve.",
desc: "Fresh browser contexts handle Cloudflare managed mode, Akamai behavioral interstitials, and Imperva sensor flows. Solved cookies are extracted and cached immediately.",
},
{
icon: "⬢",
title: "Captcha Suite",
badge: "auto",
desc: "Turnstile via shadow DOM click, reCAPTCHA v2 via Google's free audio STT, hCaptcha auto-pass, and GeeTest v4 slide via canvas gap detection. No external solver APIs. No cost per solve.",
desc: "Turnstile via shadow DOM click, reCAPTCHA v2 via free audio STT, hCaptcha auto-pass, and GeeTest v4 slide via canvas gap detection. No external solver APIs. No cost per solve.",
},
{
icon: "⊕",
title: "Custom Headers",
badge: "exclusive",
desc: "Pass Authorization, Referer, Origin, or any header to the target. Forwarded through all four tiers — including CF challenge solving — via scoped route interception that never leaks to subresources or challenge endpoints.",
desc: "Pass Authorization, Referer, Origin, or any custom header through the native API. The general proxy additionally preserves browser and authenticated traffic headers while filtering unsafe hop-by-hop values.",
},
{
icon: "◎",
title: "Session Cache",
desc: "Cloudflare cookies are stored in Redis per domain after every successful solve. The next request returns in under 500ms — zero re-challenge. 1-hour TTL, automatically refreshed.",
desc: "Solved cookies and browser identity are stored in Redis per domain. Accepted sessions avoid unnecessary re-challenges and keep repeat requests fast. TTL is configurable.",
},
{
icon: "▣",
@@ -59,8 +66,10 @@ const features = [
<section id="features" class="section">
<div class="container">
<p class="eyebrow">capabilities</p>
<h2 class="section-title">built for the modern web.</h2>
<p class="section-sub">Every protection layer. Every captcha type. One API. Nothing hosted externally.</p>
<h2 class="section-title">one engine for the protected web.</h2>
<p class="section-sub">
Adaptive scraping tiers, browser-backed challenge handling, embedded CAPTCHA flows, and a general-purpose proxy.
</p>
<div class="features-grid">
<div
@@ -151,6 +160,16 @@ const features = [
gap: 10px;
}
@media (min-width: 901px) {
.feature-card:last-child:nth-child(3n + 1) {
grid-column: 1 / -1;
}
.feature-card:last-child:nth-child(3n + 1) .feature-body {
max-width: 720px;
}
}
.feature-title {
font-size: 12px;
font-weight: 600;
+6 -3
View File
@@ -7,7 +7,8 @@ function toggleTheme() {
}
function scrollToTop() {
window.scrollTo({ top: 0, behavior: "smooth" })
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches
window.scrollTo({ top: 0, behavior: reducedMotion ? "auto" : "smooth" })
}
function onScroll() {
@@ -24,10 +25,11 @@ onUnmounted(() => window.removeEventListener("scroll", onScroll))
class="float-btn float-theme"
:class="{ raised: showScrollTop }"
:title="colorMode.value === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'"
:aria-label="colorMode.value === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'"
@click="toggleTheme"
>
<span v-if="colorMode.value === 'dark'">☀</span>
<span v-else>◐</span>
<span v-if="colorMode.value === 'dark'" aria-hidden="true">☀</span>
<span v-else aria-hidden="true">◐</span>
</button>
<Transition name="slide-up">
@@ -36,6 +38,7 @@ onUnmounted(() => window.removeEventListener("scroll", onScroll))
v-if="showScrollTop"
class="float-btn float-scroll"
title="Scroll to top"
aria-label="Scroll to top"
@click="scrollToTop"
>
↑
+11 -4
View File
@@ -16,10 +16,11 @@ function closeMenu() {
<span class="brand-dot" />
</a>
<nav class="nav-links">
<nav class="nav-links" aria-label="Primary navigation">
<a href="#features" class="nav-link">features</a>
<a href="#captcha" class="nav-link">captcha</a>
<a href="#how-it-works" class="nav-link">tiers</a>
<a href="#proxy" class="nav-link">proxy</a>
<a href="#compare" class="nav-link">benchmarks</a>
<a href="#code" class="nav-link">quickstart</a>
<a :href="docsUrl" class="nav-link">docs</a>
@@ -29,6 +30,7 @@ function closeMenu() {
rel="noopener"
class="nav-link nav-github"
title="GitHub"
aria-label="TRAWL on GitHub"
>
<svg viewBox="0 0 24 24" width="16" height="16" fill="currentColor" aria-hidden="true">
<path
@@ -47,6 +49,8 @@ function closeMenu() {
:class="{ open: menuOpen }"
@click="menuOpen = !menuOpen"
aria-label="Toggle menu"
:aria-expanded="menuOpen"
aria-controls="mobile-navigation"
>
<span /><span /><span />
</button>
@@ -55,10 +59,11 @@ function closeMenu() {
<!-- Mobile menu -->
<Transition name="menu">
<div v-if="menuOpen" class="mobile-menu">
<nav v-show="menuOpen" id="mobile-navigation" class="mobile-menu" aria-label="Mobile navigation">
<a href="#features" class="mobile-link" @click="closeMenu">features</a>
<a href="#captcha" class="mobile-link" @click="closeMenu">captcha</a>
<a href="#how-it-works" class="mobile-link" @click="closeMenu">tiers</a>
<a href="#proxy" class="mobile-link" @click="closeMenu">proxy</a>
<a href="#compare" class="mobile-link" @click="closeMenu">benchmarks</a>
<a href="#code" class="mobile-link" @click="closeMenu">quickstart</a>
<a :href="docsUrl" class="mobile-link" @click="closeMenu" @keydown.enter="closeMenu">docs</a>
@@ -77,7 +82,7 @@ function closeMenu() {
GitHub
<span v-if="stars" class="mobile-github-stars">★ {{ stars }}</span>
</a>
</div>
</nav>
</Transition>
</header>
@@ -196,7 +201,9 @@ function closeMenu() {
display: block;
height: 1px;
background: var(--text-muted);
transition: all 0.2s;
transition:
opacity 0.2s,
transform 0.2s;
transform-origin: center;
}
+231
View File
@@ -0,0 +1,231 @@
<script lang="ts" setup>
const docsUrl = useDocsUrl()
</script>
<template>
<section id="proxy" class="section proxy-section">
<div class="container">
<p class="eyebrow">challenge-aware proxy</p>
<h2 class="section-title">one proxy. challenge-aware by default.</h2>
<p class="section-sub">
Point any HTTP-compatible client at port 8192. Normal traffic takes the direct path; detected challenge pages
automatically escalate through TRAWL's browser solver and return the resolved response.
</p>
<div class="proxy-panel">
<div
v-motion
:initial="{ opacity: 0, x: -20 }"
:visible-once="{ opacity: 1, x: 0, transition: { duration: 450 } }"
class="proxy-flow"
>
<div class="flow-row">
<span class="flow-node">your client</span>
<span class="flow-arrow">→</span>
<span class="flow-node flow-node-accent">:8192</span>
<span class="flow-arrow">→</span>
<span class="flow-node">direct web</span>
</div>
<div class="flow-branch">
<span class="branch-line">└─ challenge detected</span>
<span class="flow-arrow">→</span>
<span class="flow-node flow-node-accent">tier solver</span>
<span class="flow-arrow">→</span>
<span class="flow-node">resolved response</span>
</div>
<div class="proxy-code">
<span class="code-prompt">$</span>
<code>curl --proxy http://localhost:8192 https://example.com</code>
</div>
</div>
<div
v-motion
:initial="{ opacity: 0, x: 20 }"
:visible-once="{ opacity: 1, x: 0, transition: { duration: 450, delay: 100 } }"
class="proxy-capabilities"
>
<div class="capability">
<span class="capability-mark">✓</span>
<span><strong>General traffic</strong> HTTP, HTTPS, WebSockets, binary files, and downloads.</span>
</div>
<div class="capability">
<span class="capability-mark">✓</span>
<span><strong>Complete requests</strong> Authentication, cookies, custom headers, and request bodies.</span>
</div>
<div class="capability">
<span class="capability-mark">✓</span>
<span><strong>Download semantics</strong> Range requests, 206 responses, and large-file streaming.</span>
</div>
<div class="capability">
<span class="capability-mark">✓</span>
<span><strong>Automatic escalation</strong> Browser work starts only after a challenge is detected.</span>
</div>
</div>
</div>
<p class="proxy-note">
HTTPS interception requires the generated TRAWL CA certificate to be trusted by the client.
<a :href="`${docsUrl}/proxy/ca-installation`" target="_blank" rel="noopener noreferrer">
installation guide ↗
</a>
</p>
</div>
</section>
</template>
<style scoped>
.section-title {
margin-bottom: 12px;
color: var(--text);
font-size: clamp(22px, 3vw, 34px);
font-weight: 700;
letter-spacing: -0.02em;
}
.section-sub {
max-width: 720px;
margin-bottom: 42px;
color: var(--text-muted);
font-size: 13px;
line-height: 1.75;
}
.proxy-panel {
display: grid;
grid-template-columns: 1.15fr 0.85fr;
border: 1px solid var(--border-strong);
background: var(--bg);
}
.proxy-flow,
.proxy-capabilities {
padding: 34px;
}
.proxy-flow {
display: flex;
min-width: 0;
flex-direction: column;
justify-content: center;
border-right: 1px solid var(--border);
}
.flow-row,
.flow-branch {
display: flex;
align-items: center;
gap: 10px;
white-space: nowrap;
}
.flow-branch {
margin: 16px 0 0 72px;
}
.flow-node {
padding: 7px 10px;
border: 1px solid var(--border-strong);
color: var(--text);
font-size: 10px;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.flow-node-accent {
border-color: var(--accent);
background: var(--accent-tint);
color: var(--accent);
}
.flow-arrow,
.branch-line {
color: var(--text-muted);
font-size: 11px;
}
.proxy-code {
display: flex;
gap: 10px;
overflow-x: auto;
margin-top: 32px;
padding: 14px 16px;
border: 1px solid var(--border);
background: var(--bg-subtle);
color: var(--text-muted);
font-size: 11px;
white-space: nowrap;
}
.code-prompt,
.capability-mark {
color: var(--accent);
}
.proxy-capabilities {
display: flex;
flex-direction: column;
justify-content: center;
gap: 22px;
}
.capability {
display: grid;
grid-template-columns: 16px 1fr;
gap: 10px;
color: var(--text-muted);
font-size: 11px;
line-height: 1.7;
}
.capability strong {
display: block;
color: var(--text);
font-size: 11px;
font-weight: 600;
}
.proxy-note {
margin-top: 16px;
color: var(--text-muted);
font-size: 10px;
line-height: 1.6;
}
.proxy-note a {
color: var(--accent);
}
@media (max-width: 800px) {
.proxy-panel {
grid-template-columns: 1fr;
}
.proxy-flow {
border-right: 0;
border-bottom: 1px solid var(--border);
}
}
@media (max-width: 600px) {
.proxy-flow,
.proxy-capabilities {
padding: 24px 20px;
}
.flow-row,
.flow-branch {
align-items: flex-start;
flex-direction: column;
}
.flow-branch {
margin-left: 0;
}
.flow-arrow {
transform: rotate(90deg);
}
}
</style>
+4 -4
View File
@@ -12,21 +12,21 @@ const tiers = [
name: "Cached Browser Session",
time: "~500ms",
tag: "redis hit",
desc: "Restores Cloudflare cookies from Redis into a warm pooled browser and navigates. No challenge re-solve. Returns the page in under a second. Stays cached for 1 hour.",
desc: "Restores solved cookies from Redis into a warm pooled browser. No challenge re-solve while the target accepts the session. Returns the page in under a second. Cache TTL is configurable.",
},
{
num: "03",
name: "Live Cloudflare Solve",
name: "Live Challenge Solve",
time: "4–15s",
tag: "fresh context",
desc: "A fresh browser context triggers CF managed-mode — the fastest possible challenge path. Turnstile, reCAPTCHA, hCaptcha, and GeeTest are solved automatically. Cookies are cached on success.",
desc: "A fresh Camoufox context runs the matching Cloudflare, Akamai, or Imperva flow. Turnstile, reCAPTCHA, hCaptcha, and GeeTest are solved automatically when encountered. Cookies are cached on success.",
},
{
num: "04",
name: "Residential Proxy",
time: "8–25s",
tag: "optional",
desc: "Same as Tier 3 but routes through a residential proxy. Only triggered when the datacenter IP is reputation-flagged by Cloudflare. Bandwidth cost is incurred only when truly necessary.",
desc: "Same as Tier 3 but routed through a residential proxy. Only triggered after the direct or datacenter path fails, so paid residential bandwidth is used only when necessary.",
},
]
</script>
+63 -40
View File
@@ -1,8 +1,18 @@
const title = "Self-Hosted Web Scraping & Challenge Bypass"
const description =
"Adaptive, self-hosted web scraping engine for Cloudflare, Akamai, Imperva, and CAPTCHA-protected sites, with a challenge-aware HTTP/HTTPS proxy."
const socialDescription =
"Scrape protected websites with adaptive HTTP and browser tiers, CAPTCHA handling, session caching, and a challenge-aware proxy."
export default defineNuxtConfig({
compatibilityDate: "2026-06-23",
modules: ["@nuxt/fonts", "@nuxtjs/color-mode", "@nuxtjs/seo", "@vueuse/motion/nuxt"],
alias: {
tslib: "tslib/tslib.es6.js",
},
colorMode: {
classSuffix: "",
fallback: "dark",
@@ -14,7 +24,8 @@ export default defineNuxtConfig({
{
name: "Geist Mono",
provider: "google",
weights: ["300", "400", "500", "600", "700"],
weights: ["400", "500", "600", "700"],
subsets: ["latin"],
},
],
},
@@ -22,54 +33,66 @@ export default defineNuxtConfig({
site: {
url: "https://trawl.dev",
name: "TRAWL",
description:
"Self-hosted web scraping engine with adaptive tier execution. Solves Cloudflare challenges natively in 4–15s. Returns cached results in under 500ms. Drop-in FlareSolverr replacement for Prowlarr, Jackett, Sonarr, and Radarr.",
description,
defaultLocale: "en",
},
seo: {
meta: {
description,
themeColor: "#00e87a",
ogType: "website",
ogTitle: `TRAWL — ${title}`,
ogDescription: socialDescription,
twitterCard: "summary_large_image",
twitterTitle: `TRAWL — ${title}`,
twitterDescription:
"An adaptive web scraping engine for protected sites, with CAPTCHA handling and a general HTTP/HTTPS proxy.",
},
},
ogImage: {
enabled: false,
},
robots: {
credits: false,
},
sitemap: {
zeroRuntime: true,
},
nitro: {
compressPublicAssets: true,
routeRules: {
"/**": {
headers: {
"Content-Security-Policy":
"default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self' https://api.github.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'",
"Cache-Control": "public, max-age=0, must-revalidate",
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
"Referrer-Policy": "strict-origin-when-cross-origin",
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
},
},
"/_nuxt/**": {
headers: {
"Cache-Control": "public, max-age=31536000, immutable",
},
},
},
},
app: {
head: {
title: "TRAWL — Adaptive Web Scraping Engine",
title,
templateParams: {
siteName: "TRAWL",
separator: "—",
},
meta: [
{ charset: "utf-8" },
{ name: "viewport", content: "width=device-width, initial-scale=1" },
{ name: "theme-color", content: "#00e87a" },
{ name: "robots", content: "index, follow" },
{
name: "keywords",
content:
"cloudflare bypass, web scraping, flaresolver replacement, self-hosted scraper, browser automation, captcha solver, prowlarr, jackett, *arr",
},
{ property: "og:type", content: "website" },
{
property: "og:title",
content: "TRAWL — Adaptive Web Scraping Engine",
},
{
property: "og:description",
content:
"Solve Cloudflare challenges natively. Return cached results in <500ms. Self-hosted, zero external APIs, FlareSolverr-compatible.",
},
{ property: "og:url", content: "https://trawl.dev" },
{ name: "twitter:card", content: "summary" },
{
name: "twitter:title",
content: "TRAWL — Adaptive Web Scraping Engine",
},
{
name: "twitter:description",
content:
"Self-hosted scraping engine. Cloudflare bypass, captcha solving, session caching. Drop-in FlareSolverr replacement.",
},
],
link: [
{ rel: "canonical", href: "https://trawl.dev" },
{ rel: "icon", type: "image/svg+xml", href: "/favicon.svg" },
],
link: [{ rel: "icon", type: "image/svg+xml", href: "/favicon.svg" }],
},
},
})
-3
View File
@@ -1,3 +0,0 @@
User-agent: *
Allow: /
Sitemap: https://trawl.dev/sitemap.xml
Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

-8
View File
@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://trawl.dev/</loc>
<changefreq>weekly</changefreq>
<priority>1.0</priority>
</url>
</urlset>