mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
feat(web): refresh landing page content, comparisons, proxy showcase, and SEO
This commit is contained in:
+86
-22
@@ -1,37 +1,44 @@
|
||||
<script lang="ts" setup>
|
||||
useHead({
|
||||
script: [
|
||||
{
|
||||
type: "application/ld+json",
|
||||
innerHTML: JSON.stringify({
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
name: "TRAWL",
|
||||
url: "https://trawl.dev",
|
||||
description:
|
||||
"Self-hosted web scraping engine with adaptive tier execution. Solves Cloudflare challenges natively. Drop-in FlareSolverr replacement.",
|
||||
applicationCategory: "DeveloperApplication",
|
||||
operatingSystem: "Linux, Docker",
|
||||
license: "https://www.gnu.org/licenses/agpl-3.0.html",
|
||||
author: { "@type": "Person", name: "germondai", url: "https://github.com/germondai" },
|
||||
codeRepository: "https://github.com/germondai/trawl",
|
||||
offers: { "@type": "Offer", price: "0", priceCurrency: "USD" },
|
||||
}),
|
||||
const description =
|
||||
"Adaptive, self-hosted web scraping engine for Cloudflare, Akamai, Imperva, and CAPTCHA-protected sites, with a challenge-aware HTTP/HTTPS proxy."
|
||||
|
||||
useSchemaOrg([
|
||||
defineWebPage({
|
||||
name: "TRAWL — Self-Hosted Web Scraping & Challenge Bypass",
|
||||
description,
|
||||
}),
|
||||
defineSoftwareApp({
|
||||
name: "TRAWL",
|
||||
description,
|
||||
applicationCategory: "DeveloperApplication",
|
||||
operatingSystem: "Linux, Docker",
|
||||
license: "https://www.gnu.org/licenses/agpl-3.0.html",
|
||||
author: {
|
||||
"@type": "Person",
|
||||
name: "germondai",
|
||||
url: "https://github.com/germondai",
|
||||
},
|
||||
],
|
||||
})
|
||||
offers: {
|
||||
"@type": "Offer",
|
||||
price: 0,
|
||||
priceCurrency: "USD",
|
||||
},
|
||||
}),
|
||||
])
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="layout">
|
||||
<a class="skip-link" href="#main-content">Skip to content</a>
|
||||
<NavBar />
|
||||
<main>
|
||||
<main id="main-content" tabindex="-1">
|
||||
<HeroSection />
|
||||
<EcosystemBanner />
|
||||
<StatsBar />
|
||||
<FeaturesGrid />
|
||||
<ChallengeGrid />
|
||||
<TierFlow />
|
||||
<ProxySection />
|
||||
<CompareTable />
|
||||
<CtaSection />
|
||||
<CodeExample />
|
||||
@@ -44,6 +51,7 @@ useHead({
|
||||
<style>
|
||||
/* ── CSS custom properties ── */
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #fafafa;
|
||||
--bg-subtle: #f4f4f5;
|
||||
--surface: #ffffff;
|
||||
@@ -58,6 +66,7 @@ useHead({
|
||||
}
|
||||
|
||||
.dark {
|
||||
color-scheme: dark;
|
||||
--bg: #0d0d10;
|
||||
--bg-subtle: #17171b;
|
||||
--surface: #1c1c21;
|
||||
@@ -103,6 +112,11 @@ button {
|
||||
font-family: inherit;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
:where(a, button, [tabindex]):focus-visible {
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 3px;
|
||||
}
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
@@ -135,12 +149,43 @@ main {
|
||||
padding-top: 60px;
|
||||
}
|
||||
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
top: 8px;
|
||||
left: 8px;
|
||||
z-index: 1000;
|
||||
padding: 8px 12px;
|
||||
color: #080808;
|
||||
background: var(--accent);
|
||||
transform: translateY(-150%);
|
||||
}
|
||||
|
||||
.skip-link:focus {
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
/* ── Shared section wrapper ── */
|
||||
.section {
|
||||
padding: 80px 0;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
[id] {
|
||||
scroll-margin-top: 76px;
|
||||
}
|
||||
|
||||
.sr-only {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
padding: 0;
|
||||
margin: -1px;
|
||||
overflow: hidden;
|
||||
clip: rect(0, 0, 0, 0);
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 1100px;
|
||||
margin: 0 auto;
|
||||
@@ -171,7 +216,11 @@ main {
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.06em;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
transition:
|
||||
color 0.15s,
|
||||
background-color 0.15s,
|
||||
border-color 0.15s,
|
||||
opacity 0.15s;
|
||||
border: 1px solid transparent;
|
||||
white-space: nowrap;
|
||||
}
|
||||
@@ -205,6 +254,21 @@ main {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
html {
|
||||
scroll-behavior: auto;
|
||||
}
|
||||
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
scroll-behavior: auto;
|
||||
animation-duration: 0.01ms;
|
||||
animation-iteration-count: 1;
|
||||
transition-duration: 0.01ms;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Mobile globals ── */
|
||||
@media (max-width: 640px) {
|
||||
.section {
|
||||
|
||||
@@ -20,6 +20,26 @@ const challenges = [
|
||||
byparr: "not handled",
|
||||
byparrSupport: "no",
|
||||
},
|
||||
{
|
||||
name: "Akamai Bot Manager",
|
||||
desc: "Behavioral interstitial + press-and-hold",
|
||||
trawl: "sensor flow · human-like input",
|
||||
trawlSupport: "yes",
|
||||
flaresolver: "not handled",
|
||||
fsSupport: "no",
|
||||
byparr: "not handled",
|
||||
byparrSupport: "no",
|
||||
},
|
||||
{
|
||||
name: "Imperva / Incapsula",
|
||||
desc: "Sensor-cookie WAF challenge",
|
||||
trawl: "browser sensor flow · best effort",
|
||||
trawlSupport: "partial",
|
||||
flaresolver: "not handled",
|
||||
fsSupport: "no",
|
||||
byparr: "not handled",
|
||||
byparrSupport: "no",
|
||||
},
|
||||
{
|
||||
name: "reCAPTCHA v2",
|
||||
desc: "Google checkbox + audio challenge",
|
||||
@@ -74,14 +94,17 @@ function iconClass(s: string) {
|
||||
<section id="captcha" class="section">
|
||||
<div class="container">
|
||||
<p class="eyebrow">challenge coverage</p>
|
||||
<h2 class="section-title">every wall. handled.</h2>
|
||||
<h2 class="section-title">solve more than Cloudflare.</h2>
|
||||
<p class="section-sub">
|
||||
TRAWL is the only self-hosted scraper with native solvers for all major protection layers — not just Cloudflare
|
||||
interstitials, but the captchas embedded inside the pages themselves.
|
||||
Native flows cover Cloudflare, Akamai, Imperva, and the captchas embedded inside protected pages — all within
|
||||
the same self-hosted tier engine.
|
||||
</p>
|
||||
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<caption class="sr-only">
|
||||
Challenge handling support across TRAWL, FlareSolverr, and Byparr
|
||||
</caption>
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="col-name">challenge type</th>
|
||||
@@ -97,7 +120,7 @@ function iconClass(s: string) {
|
||||
<span class="challenge-desc">{{ c.desc }}</span>
|
||||
</td>
|
||||
<td class="col-trawl">
|
||||
<span class="cell-yes">✓</span>
|
||||
<span :class="iconClass(c.trawlSupport)">{{ icon(c.trawlSupport) }}</span>
|
||||
{{ c.trawl }}
|
||||
</td>
|
||||
<td>
|
||||
@@ -114,16 +137,16 @@ function iconClass(s: string) {
|
||||
<tr>
|
||||
<td class="col-name score-spacer"></td>
|
||||
<td class="col-trawl score-cell">
|
||||
<span class="score-num accent">6 / 6</span>
|
||||
<span class="score-label">challenge types</span>
|
||||
<span class="score-num accent">8</span>
|
||||
<span class="score-label">challenge paths</span>
|
||||
</td>
|
||||
<td class="score-cell">
|
||||
<span class="score-num">1 / 6</span>
|
||||
<span class="score-label">challenge types</span>
|
||||
<span class="score-num">1</span>
|
||||
<span class="score-label">dedicated path</span>
|
||||
</td>
|
||||
<td class="score-cell">
|
||||
<span class="score-num">1 / 6</span>
|
||||
<span class="score-label">challenge types</span>
|
||||
<span class="score-num">1</span>
|
||||
<span class="score-label">dedicated path</span>
|
||||
</td>
|
||||
</tr>
|
||||
</tfoot>
|
||||
|
||||
@@ -39,7 +39,7 @@ curl -s -X POST http://localhost:8191/scrape \\
|
||||
<span class="kw">const</span> { solution } = <span class="kw">await</span> res.json()
|
||||
|
||||
console.log(solution.response) <span class="c">// full HTML</span>
|
||||
console.log(solution.cookies) <span class="c">// Cloudflare cookies</span>
|
||||
console.log(solution.cookies) <span class="c">// solved session cookies</span>
|
||||
console.log(solution.userAgent) <span class="c">// browser UA used</span>`,
|
||||
|
||||
python: `<span class="kw">import</span> requests
|
||||
@@ -105,21 +105,23 @@ curl -s -X POST http://localhost:8191/scrape \\
|
||||
:visible-once="{ opacity: 1, y: 0, transition: { duration: 400 } }"
|
||||
class="code-block"
|
||||
>
|
||||
<div class="tabs">
|
||||
<fieldset class="tabs">
|
||||
<legend class="sr-only">API example language</legend>
|
||||
<button
|
||||
type="button"
|
||||
v-for="tab in tabs"
|
||||
:key="tab.id"
|
||||
class="tab"
|
||||
:class="{ active: active === tab.id }"
|
||||
:aria-pressed="active === tab.id"
|
||||
@click="active = tab.id"
|
||||
>
|
||||
{{ tab.label }}
|
||||
</button>
|
||||
<div class="tab-spacer" />
|
||||
<span class="tab-hint">POST /v1 · FlareSolverr v2 compat</span>
|
||||
</div>
|
||||
<pre class="code-body"><code v-html="code[active]" /></pre>
|
||||
</fieldset>
|
||||
<pre class="code-body" aria-live="polite"><code v-html="code[active]" /></pre>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -140,6 +142,10 @@ curl -s -X POST http://localhost:8191/scrape \\
|
||||
}
|
||||
|
||||
.tabs {
|
||||
min-width: 0;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
border: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
border-bottom: 1px solid var(--border);
|
||||
@@ -158,7 +164,10 @@ curl -s -X POST http://localhost:8191/scrape \\
|
||||
border-right: 1px solid var(--border);
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
transition: all 0.12s;
|
||||
transition:
|
||||
color 0.12s,
|
||||
background-color 0.12s,
|
||||
border-color 0.12s;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
<script lang="ts" setup>
|
||||
const barsVisible = shallowRef(false)
|
||||
const benchGrid = useTemplateRef<HTMLElement>("benchGrid")
|
||||
let benchmarkObserver: IntersectionObserver
|
||||
|
||||
onMounted(() => {
|
||||
const el = document.querySelector(".bench-grid")
|
||||
if (!el) return
|
||||
const observer = new IntersectionObserver(
|
||||
if (!benchGrid.value) return
|
||||
benchmarkObserver = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (entries[0]?.isIntersecting) {
|
||||
barsVisible.value = true
|
||||
observer.disconnect()
|
||||
benchmarkObserver.disconnect()
|
||||
}
|
||||
},
|
||||
{ threshold: 0.2 },
|
||||
)
|
||||
observer.observe(el)
|
||||
benchmarkObserver.observe(benchGrid.value)
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => benchmarkObserver?.disconnect())
|
||||
|
||||
const benchmarks = [
|
||||
{
|
||||
url: "nowsecure.nl",
|
||||
@@ -49,52 +52,107 @@ const benchmarks = [
|
||||
},
|
||||
]
|
||||
|
||||
const rows = [
|
||||
{ feature: "Persistent browser pool", trawl: "✓ N instances", flaresolver: "✗ 1 instance", byparr: "✗ 1 instance" },
|
||||
{
|
||||
feature: "Domain session cache",
|
||||
trawl: "✓ Redis, <500ms repeat",
|
||||
flaresolver: "✗ always full solve",
|
||||
byparr: "✗ always full solve",
|
||||
},
|
||||
{ feature: "Adaptive tier execution", trawl: "✓ 4 tiers", flaresolver: "✗ always browser", byparr: "~" },
|
||||
{ feature: "Browser engine", trawl: "✓ Camoufox Firefox", flaresolver: "✗ Chrome", byparr: "✓ Camoufox Firefox" },
|
||||
{ feature: "Docker image size", trawl: "✓ 1.17 GB", flaresolver: "✓ 1.03 GB", byparr: "✗ 4.10 GB" },
|
||||
{ feature: "Memory footprint (peak)", trawl: "✓ 770 MB", flaresolver: "✓ 500 MB", byparr: "✗ 1.35 GB" },
|
||||
{ feature: "Cloudflare challenge speed", trawl: "✓ 4–15s", flaresolver: "✗ 11–18s", byparr: "✗ 13–18s" },
|
||||
{ feature: "CF Turnstile solving", trawl: "✓ shadow DOM click", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "reCAPTCHA v2 solving", trawl: "✓ audio STT (free)", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "hCaptcha solving", trawl: "✓ auto-pass", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "GeeTest v4 solving", trawl: "✓ canvas gap detection", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "Custom headers (all tiers)", trawl: "✓ all 4 tiers", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "Proxy escalation", trawl: "✓ DC + residential", flaresolver: "✗", byparr: "~" },
|
||||
{ feature: "Self-healing pool", trawl: "✓", flaresolver: "✗", byparr: "✗" },
|
||||
{ feature: "FlareSolverr v2 compat", trawl: "✓", flaresolver: "✓ native", byparr: "✓" },
|
||||
{
|
||||
feature: "External solver APIs",
|
||||
trawl: "✓ none required",
|
||||
flaresolver: "✓ none required",
|
||||
byparr: "✓ none required",
|
||||
},
|
||||
{ feature: "Self-hosted", trawl: "✓", flaresolver: "✓", byparr: "✓" },
|
||||
]
|
||||
|
||||
function renderCell(val: string) {
|
||||
if (val.startsWith("✓")) return `<span class="check">${val}</span>`
|
||||
if (val.startsWith("✗")) return `<span class="cross">${val}</span>`
|
||||
if (val.startsWith("~")) return `<span class="partial">${val}</span>`
|
||||
return val
|
||||
type CellTone = "check" | "cross" | "partial"
|
||||
interface ComparisonCell {
|
||||
text: string
|
||||
tone: CellTone
|
||||
}
|
||||
|
||||
const statusMarks: Record<CellTone, string> = { check: "✓", partial: "~", cross: "✗" }
|
||||
const cell = (tone: CellTone, text: string): ComparisonCell => ({ text, tone })
|
||||
|
||||
const rows = [
|
||||
{
|
||||
icon: "⬡",
|
||||
feature: "Adaptive routing",
|
||||
trawl: cell("check", "HTTP → cache → browser → residential"),
|
||||
flaresolver: cell("cross", "Browser only"),
|
||||
byparr: cell("cross", "Browser only"),
|
||||
},
|
||||
{
|
||||
icon: "⇄",
|
||||
feature: "Challenge proxy",
|
||||
trawl: cell("check", "HTTP/S, WebSockets, Range"),
|
||||
flaresolver: cell("cross", "API only"),
|
||||
byparr: cell("cross", "API only"),
|
||||
},
|
||||
{
|
||||
icon: "↝",
|
||||
feature: "Proxy escalation",
|
||||
trawl: cell("check", "DC → residential"),
|
||||
flaresolver: cell("partial", "Manual proxy"),
|
||||
byparr: cell("partial", "Manual proxy"),
|
||||
},
|
||||
{
|
||||
icon: "◎",
|
||||
feature: "Session reuse",
|
||||
trawl: cell("check", "Redis by domain"),
|
||||
flaresolver: cell("partial", "Manual sessions"),
|
||||
byparr: cell("cross", "No domain cache"),
|
||||
},
|
||||
{
|
||||
icon: "▣",
|
||||
feature: "Browser pool",
|
||||
trawl: cell("check", "Configurable and warm"),
|
||||
flaresolver: cell("partial", "Temporary or session"),
|
||||
byparr: cell("cross", "New browser per request"),
|
||||
},
|
||||
{
|
||||
icon: "⟳",
|
||||
feature: "Pool lifecycle",
|
||||
trawl: cell("check", "Health checks + recycling"),
|
||||
flaresolver: cell("cross", "No warm pool"),
|
||||
byparr: cell("cross", "No warm pool"),
|
||||
},
|
||||
{
|
||||
icon: "⊕",
|
||||
feature: "Custom headers",
|
||||
trawl: cell("check", "Safe across all tiers"),
|
||||
flaresolver: cell("cross", "Not supported"),
|
||||
byparr: cell("cross", "Not supported"),
|
||||
},
|
||||
{
|
||||
icon: "◈",
|
||||
feature: "Browser engine",
|
||||
trawl: cell("check", "Camoufox"),
|
||||
flaresolver: cell("partial", "Chrome + UDC"),
|
||||
byparr: cell("check", "Camoufox"),
|
||||
},
|
||||
{
|
||||
icon: "◇",
|
||||
feature: "/v1 API",
|
||||
trawl: cell("check", "GET + POST"),
|
||||
flaresolver: cell("check", "Native"),
|
||||
byparr: cell("partial", "GET only"),
|
||||
},
|
||||
{
|
||||
icon: "○",
|
||||
feature: "Paid solver APIs",
|
||||
trawl: cell("check", "Not required"),
|
||||
flaresolver: cell("check", "Not required"),
|
||||
byparr: cell("check", "Not required"),
|
||||
},
|
||||
{
|
||||
icon: "⌂",
|
||||
feature: "Self-hosted",
|
||||
trawl: cell("check", "Yes"),
|
||||
flaresolver: cell("check", "Yes"),
|
||||
byparr: cell("check", "Yes"),
|
||||
},
|
||||
]
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section id="compare" class="section">
|
||||
<div class="container">
|
||||
<p class="eyebrow">benchmarks</p>
|
||||
<h2 class="section-title">TRAWL vs the alternatives.</h2>
|
||||
<p class="section-sub">Measured on the same machine, same network, same target URLs.</p>
|
||||
<h2 class="section-title">Less browser overhead. More ways through.</h2>
|
||||
<p class="section-lead">
|
||||
Compare real request timings and the paths each engine can take—from direct HTTP to full browser solving and
|
||||
proxy escalation.
|
||||
</p>
|
||||
|
||||
<div class="bench-grid">
|
||||
<div ref="benchGrid" class="bench-grid">
|
||||
<div
|
||||
v-for="b in benchmarks"
|
||||
:key="b.url"
|
||||
@@ -121,8 +179,23 @@ function renderCell(val: string) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="benchmark-note">
|
||||
Measurements recorded on the same machine and network. They are illustrative, not guarantees; live results vary
|
||||
with target protection, IP reputation, software versions, and session state. Capability rows were reviewed
|
||||
against the upstream projects.
|
||||
</p>
|
||||
|
||||
<div class="table-wrap" style="margin-top: 56px;">
|
||||
<table>
|
||||
<caption class="sr-only">
|
||||
Feature comparison across TRAWL, FlareSolverr, and Byparr
|
||||
</caption>
|
||||
<colgroup>
|
||||
<col class="feature-width" />
|
||||
<col class="trawl-width" />
|
||||
<col class="competitor-width" />
|
||||
<col class="competitor-width" />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>capability</th>
|
||||
@@ -133,10 +206,25 @@ function renderCell(val: string) {
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="row in rows" :key="row.feature">
|
||||
<td class="feature-col">{{ row.feature }}</td>
|
||||
<td class="col-trawl" v-html="renderCell(row.trawl)" />
|
||||
<td v-html="renderCell(row.flaresolver)" />
|
||||
<td v-html="renderCell(row.byparr)" />
|
||||
<td class="feature-col">
|
||||
<span class="feature-icon" aria-hidden="true">{{ row.icon }}</span>{{ row.feature }}
|
||||
</td>
|
||||
<td class="col-trawl">
|
||||
<span :class="row.trawl.tone"
|
||||
><span class="status-mark">{{ statusMarks[row.trawl.tone] }}</span>{{ row.trawl.text }}</span
|
||||
>
|
||||
</td>
|
||||
<td>
|
||||
<span :class="row.flaresolver.tone"
|
||||
><span class="status-mark">{{ statusMarks[row.flaresolver.tone] }}</span>
|
||||
{{ row.flaresolver.text }}</span
|
||||
>
|
||||
</td>
|
||||
<td>
|
||||
<span :class="row.byparr.tone"
|
||||
><span class="status-mark">{{ statusMarks[row.byparr.tone] }}</span>{{ row.byparr.text }}</span
|
||||
>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -154,11 +242,22 @@ function renderCell(val: string) {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.section-sub {
|
||||
.section-lead {
|
||||
font-size: 13px;
|
||||
line-height: 1.75;
|
||||
color: var(--text-muted);
|
||||
max-width: 540px;
|
||||
max-width: 650px;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.benchmark-note {
|
||||
color: var(--text-muted);
|
||||
font-size: 10px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.benchmark-note {
|
||||
max-width: 760px;
|
||||
margin-bottom: 48px;
|
||||
}
|
||||
|
||||
@@ -251,13 +350,26 @@ function renderCell(val: string) {
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
table-layout: fixed;
|
||||
border-collapse: collapse;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.feature-width {
|
||||
width: 19%;
|
||||
}
|
||||
|
||||
.trawl-width {
|
||||
width: 31%;
|
||||
}
|
||||
|
||||
.competitor-width {
|
||||
width: 25%;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 12px 18px;
|
||||
padding: 11px 14px;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--border);
|
||||
white-space: nowrap;
|
||||
@@ -282,6 +394,20 @@ td {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.feature-icon {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
margin-right: 8px;
|
||||
color: var(--accent);
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
line-height: 1;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.col-trawl {
|
||||
background: var(--accent-tint);
|
||||
}
|
||||
@@ -296,21 +422,32 @@ tr:hover .col-trawl {
|
||||
background: color-mix(in srgb, var(--accent-tint) 150%, transparent);
|
||||
}
|
||||
|
||||
:deep(.check) {
|
||||
.check {
|
||||
color: var(--accent);
|
||||
font-weight: 600;
|
||||
}
|
||||
:deep(.cross) {
|
||||
.cross {
|
||||
color: var(--text-muted);
|
||||
opacity: 0.4;
|
||||
}
|
||||
:deep(.partial) {
|
||||
.partial {
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
.status-mark {
|
||||
display: inline-block;
|
||||
width: 16px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
@media (max-width: 800px) {
|
||||
.bench-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
table {
|
||||
min-width: 860px;
|
||||
table-layout: auto;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -22,8 +22,8 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
class="cta-inner"
|
||||
>
|
||||
<div class="cta-label">open source · AGPL-3.0 · self-hosted</div>
|
||||
<h2 class="cta-heading">up and running<br />in 60 seconds.</h2>
|
||||
<p class="cta-sub">No clone. No build. Pull the image and start.</p>
|
||||
<h2 class="cta-heading">scrape protected sites<br />in 60 seconds.</h2>
|
||||
<p class="cta-sub">No clone. No build. Pull the Docker image and send your first request.</p>
|
||||
<div class="cta-actions">
|
||||
<a href="https://github.com/germondai/trawl" target="_blank" rel="noopener noreferrer" class="btn-primary">
|
||||
<span class="btn-icon">↗</span>
|
||||
@@ -35,18 +35,20 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
<!-- Step 1: Setup -->
|
||||
<div class="step-label">1 — choose your setup</div>
|
||||
<div class="cta-snippet">
|
||||
<div class="setup-tabs">
|
||||
<fieldset class="setup-tabs">
|
||||
<legend class="sr-only">Installation method</legend>
|
||||
<button
|
||||
type="button"
|
||||
v-for="tab in tabs"
|
||||
:key="tab.id"
|
||||
class="setup-tab"
|
||||
:class="{ active: active === tab.id }"
|
||||
:aria-pressed="active === tab.id"
|
||||
@click="active = tab.id"
|
||||
>
|
||||
{{ tab.label }}
|
||||
</button>
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
<div v-if="active === 'docker'" class="snippet-body-wrap">
|
||||
<div class="snippet-bar inner-bar">
|
||||
@@ -61,64 +63,85 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
|
||||
<div v-else-if="active === 'minimal'" class="snippet-body-wrap">
|
||||
<div class="snippet-bar inner-bar">
|
||||
<span class="snippet-hint">single service · pool size 1 · 15s acquire wait · no redis</span>
|
||||
<span class="snippet-hint">docker-compose.minimal.yml · single service · no redis</span>
|
||||
</div>
|
||||
<pre class="snippet-body"><code><span class="k">services:</span>
|
||||
trawl:
|
||||
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
|
||||
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
|
||||
<span class="k">ports:</span>
|
||||
- <span class="s">"${PORT:-8191}:8191"</span>
|
||||
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
|
||||
<span class="k">shm_size:</span> <span class="s">1gb</span>
|
||||
<span class="k">environment:</span>
|
||||
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">1</span>
|
||||
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span></code></pre>
|
||||
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
|
||||
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
|
||||
<span class="k">volumes:</span>
|
||||
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
|
||||
<span class="k">healthcheck:</span>
|
||||
<span class="k">test:</span> [<span class="s">"CMD"</span>, <span class="s">"curl"</span>, <span class="s">"-sf"</span>, <span class="s">"http://localhost:8191/health"</span>]
|
||||
<span class="k">volumes:</span>
|
||||
trawl_proxy_ca:</code></pre>
|
||||
</div>
|
||||
|
||||
<div v-else-if="active === 'cached'" class="snippet-body-wrap">
|
||||
<div class="snippet-bar inner-bar">
|
||||
<span class="snippet-hint">redis session cache · repeat requests ~500ms</span>
|
||||
<span class="snippet-hint">docker-compose.yml · redis cache · pool size 1</span>
|
||||
</div>
|
||||
<pre class="snippet-body"><code><span class="k">services:</span>
|
||||
redis:
|
||||
<span class="k">image:</span> <span class="s">redis:7-alpine</span>
|
||||
<span class="k">image:</span> <span class="s">redis:8.8-alpine</span>
|
||||
<span class="k">volumes:</span> [<span class="s">redis_data:/data</span>]
|
||||
trawl:
|
||||
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
|
||||
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
|
||||
<span class="k">ports:</span>
|
||||
- <span class="s">"${PORT:-8191}:8191"</span>
|
||||
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
|
||||
<span class="k">shm_size:</span> <span class="s">1gb</span>
|
||||
<span class="k">environment:</span>
|
||||
<span class="k">REDIS_URL:</span> <span class="s">redis://redis:6379</span>
|
||||
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">3</span>
|
||||
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span>
|
||||
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">${BROWSER_POOL_SIZE:-1}</span>
|
||||
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
|
||||
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
|
||||
<span class="k">volumes:</span>
|
||||
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
|
||||
<span class="k">depends_on:</span> [<span class="s">redis</span>]
|
||||
<span class="k">volumes:</span>
|
||||
redis_data:</code></pre>
|
||||
redis_data:
|
||||
trawl_proxy_ca:</code></pre>
|
||||
</div>
|
||||
|
||||
<div v-else-if="active === 'production'" class="snippet-body-wrap">
|
||||
<div class="snippet-bar inner-bar">
|
||||
<span class="snippet-hint">auto-restart · memory limit · healthcheck</span>
|
||||
<span class="snippet-hint">docker-compose.prod.yml · pool size 3 · restart + healthcheck</span>
|
||||
</div>
|
||||
<pre class="snippet-body"><code><span class="k">services:</span>
|
||||
redis:
|
||||
<span class="k">image:</span> <span class="s">redis:7-alpine</span>
|
||||
<span class="k">image:</span> <span class="s">redis:8.8-alpine</span>
|
||||
<span class="k">restart:</span> <span class="s">always</span>
|
||||
<span class="k">volumes:</span> [<span class="s">redis_data:/data</span>]
|
||||
trawl:
|
||||
<span class="k">image:</span> <span class="s">ghcr.io/germondai/trawl:latest</span>
|
||||
<span class="k">restart:</span> <span class="s">always</span>
|
||||
<span class="k">ports:</span> [<span class="s">"8191:8191"</span>]
|
||||
<span class="k">ports:</span>
|
||||
- <span class="s">"${PORT:-8191}:8191"</span>
|
||||
- <span class="s">"${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"</span>
|
||||
<span class="k">shm_size:</span> <span class="s">1gb</span>
|
||||
<span class="k">mem_limit:</span> <span class="s">3g</span>
|
||||
<span class="k">environment:</span>
|
||||
<span class="k">REDIS_URL:</span> <span class="s">redis://redis:6379</span>
|
||||
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">5</span>
|
||||
<span class="k">BROWSER_ACQUIRE_TIMEOUT_MS:</span> <span class="s">15000</span>
|
||||
<span class="k">BROWSER_POOL_SIZE:</span> <span class="s">3</span>
|
||||
<span class="k">MITM_PROXY_ENABLED:</span> <span class="s">${MITM_PROXY_ENABLED:-false}</span>
|
||||
<span class="k">MITM_PROXY_CA_DIR:</span> <span class="s">/data/proxy-ca</span>
|
||||
<span class="k">volumes:</span>
|
||||
- <span class="s">trawl_proxy_ca:/data/proxy-ca</span>
|
||||
<span class="k">depends_on:</span> [<span class="s">redis</span>]
|
||||
<span class="k">healthcheck:</span>
|
||||
<span class="k">test:</span> <span class="s">wget -qO- http://localhost:8191/health</span>
|
||||
<span class="k">test:</span> [<span class="s">"CMD"</span>, <span class="s">"curl"</span>, <span class="s">"-sf"</span>, <span class="s">"http://localhost:8191/health"</span>]
|
||||
<span class="k">interval:</span> <span class="s">30s</span>
|
||||
<span class="k">volumes:</span>
|
||||
redis_data:</code></pre>
|
||||
redis_data:
|
||||
trawl_proxy_ca:</code></pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -218,7 +241,9 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.05em;
|
||||
text-decoration: none;
|
||||
transition: all 0.12s;
|
||||
transition:
|
||||
color 0.12s,
|
||||
border-color 0.12s;
|
||||
}
|
||||
.btn-ghost:hover {
|
||||
border-color: var(--text-muted);
|
||||
@@ -240,6 +265,10 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
}
|
||||
|
||||
.setup-tabs {
|
||||
min-width: 0;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
border: 0;
|
||||
display: flex;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--bg-subtle);
|
||||
@@ -257,7 +286,10 @@ const tabs: { id: Tab; label: string; hint: string }[] = [
|
||||
border-right: 1px solid var(--border);
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
transition: all 0.12s;
|
||||
transition:
|
||||
color 0.12s,
|
||||
background-color 0.12s,
|
||||
border-color 0.12s;
|
||||
}
|
||||
.setup-tab:hover {
|
||||
color: var(--text);
|
||||
|
||||
@@ -19,7 +19,7 @@ const apps = [
|
||||
>{{ app.name }}</span
|
||||
>
|
||||
</div>
|
||||
<span class="eco-label">— plug in, zero config changes</span>
|
||||
<span class="eco-label">— connect with a single URL change</span>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -7,6 +7,13 @@ const features = [
|
||||
featured: true,
|
||||
desc: "Every request takes the cheapest path first — plain HTTP, then cached session, then live browser solve, then residential proxy. You pay the full browser cost only when you have to.",
|
||||
},
|
||||
{
|
||||
icon: "⇄",
|
||||
title: "General HTTP/HTTPS Proxy",
|
||||
badge: "v1.2+",
|
||||
featured: true,
|
||||
desc: "Proxy ordinary HTTP, HTTPS, WebSockets, binary files, and Range requests through port 8192. Direct traffic stays lightweight; detected challenge pages escalate through TRAWL's solver tiers.",
|
||||
},
|
||||
{
|
||||
icon: "◈",
|
||||
title: "Camoufox Firefox",
|
||||
@@ -16,26 +23,26 @@ const features = [
|
||||
},
|
||||
{
|
||||
icon: "⟁",
|
||||
title: "Cloudflare Bypass",
|
||||
title: "Cloudflare + WAF Bypass",
|
||||
badge: "fastest",
|
||||
desc: "Fresh browser context triggers CF managed-mode evaluation — solves in 4–15s vs 11–18s with alternatives. Cookies are extracted and cached immediately after the first solve.",
|
||||
desc: "Fresh browser contexts handle Cloudflare managed mode, Akamai behavioral interstitials, and Imperva sensor flows. Solved cookies are extracted and cached immediately.",
|
||||
},
|
||||
{
|
||||
icon: "⬢",
|
||||
title: "Captcha Suite",
|
||||
badge: "auto",
|
||||
desc: "Turnstile via shadow DOM click, reCAPTCHA v2 via Google's free audio STT, hCaptcha auto-pass, and GeeTest v4 slide via canvas gap detection. No external solver APIs. No cost per solve.",
|
||||
desc: "Turnstile via shadow DOM click, reCAPTCHA v2 via free audio STT, hCaptcha auto-pass, and GeeTest v4 slide via canvas gap detection. No external solver APIs. No cost per solve.",
|
||||
},
|
||||
{
|
||||
icon: "⊕",
|
||||
title: "Custom Headers",
|
||||
badge: "exclusive",
|
||||
desc: "Pass Authorization, Referer, Origin, or any header to the target. Forwarded through all four tiers — including CF challenge solving — via scoped route interception that never leaks to subresources or challenge endpoints.",
|
||||
desc: "Pass Authorization, Referer, Origin, or any custom header through the native API. The general proxy additionally preserves browser and authenticated traffic headers while filtering unsafe hop-by-hop values.",
|
||||
},
|
||||
{
|
||||
icon: "◎",
|
||||
title: "Session Cache",
|
||||
desc: "Cloudflare cookies are stored in Redis per domain after every successful solve. The next request returns in under 500ms — zero re-challenge. 1-hour TTL, automatically refreshed.",
|
||||
desc: "Solved cookies and browser identity are stored in Redis per domain. Accepted sessions avoid unnecessary re-challenges and keep repeat requests fast. TTL is configurable.",
|
||||
},
|
||||
{
|
||||
icon: "▣",
|
||||
@@ -59,8 +66,10 @@ const features = [
|
||||
<section id="features" class="section">
|
||||
<div class="container">
|
||||
<p class="eyebrow">capabilities</p>
|
||||
<h2 class="section-title">built for the modern web.</h2>
|
||||
<p class="section-sub">Every protection layer. Every captcha type. One API. Nothing hosted externally.</p>
|
||||
<h2 class="section-title">one engine for the protected web.</h2>
|
||||
<p class="section-sub">
|
||||
Adaptive scraping tiers, browser-backed challenge handling, embedded CAPTCHA flows, and a general-purpose proxy.
|
||||
</p>
|
||||
|
||||
<div class="features-grid">
|
||||
<div
|
||||
@@ -151,6 +160,16 @@ const features = [
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
@media (min-width: 901px) {
|
||||
.feature-card:last-child:nth-child(3n + 1) {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.feature-card:last-child:nth-child(3n + 1) .feature-body {
|
||||
max-width: 720px;
|
||||
}
|
||||
}
|
||||
|
||||
.feature-title {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
|
||||
@@ -7,7 +7,8 @@ function toggleTheme() {
|
||||
}
|
||||
|
||||
function scrollToTop() {
|
||||
window.scrollTo({ top: 0, behavior: "smooth" })
|
||||
const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches
|
||||
window.scrollTo({ top: 0, behavior: reducedMotion ? "auto" : "smooth" })
|
||||
}
|
||||
|
||||
function onScroll() {
|
||||
@@ -24,10 +25,11 @@ onUnmounted(() => window.removeEventListener("scroll", onScroll))
|
||||
class="float-btn float-theme"
|
||||
:class="{ raised: showScrollTop }"
|
||||
:title="colorMode.value === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'"
|
||||
:aria-label="colorMode.value === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'"
|
||||
@click="toggleTheme"
|
||||
>
|
||||
<span v-if="colorMode.value === 'dark'">☀</span>
|
||||
<span v-else>◐</span>
|
||||
<span v-if="colorMode.value === 'dark'" aria-hidden="true">☀</span>
|
||||
<span v-else aria-hidden="true">◐</span>
|
||||
</button>
|
||||
|
||||
<Transition name="slide-up">
|
||||
@@ -36,6 +38,7 @@ onUnmounted(() => window.removeEventListener("scroll", onScroll))
|
||||
v-if="showScrollTop"
|
||||
class="float-btn float-scroll"
|
||||
title="Scroll to top"
|
||||
aria-label="Scroll to top"
|
||||
@click="scrollToTop"
|
||||
>
|
||||
↑
|
||||
|
||||
@@ -16,10 +16,11 @@ function closeMenu() {
|
||||
<span class="brand-dot" />
|
||||
</a>
|
||||
|
||||
<nav class="nav-links">
|
||||
<nav class="nav-links" aria-label="Primary navigation">
|
||||
<a href="#features" class="nav-link">features</a>
|
||||
<a href="#captcha" class="nav-link">captcha</a>
|
||||
<a href="#how-it-works" class="nav-link">tiers</a>
|
||||
<a href="#proxy" class="nav-link">proxy</a>
|
||||
<a href="#compare" class="nav-link">benchmarks</a>
|
||||
<a href="#code" class="nav-link">quickstart</a>
|
||||
<a :href="docsUrl" class="nav-link">docs</a>
|
||||
@@ -29,6 +30,7 @@ function closeMenu() {
|
||||
rel="noopener"
|
||||
class="nav-link nav-github"
|
||||
title="GitHub"
|
||||
aria-label="TRAWL on GitHub"
|
||||
>
|
||||
<svg viewBox="0 0 24 24" width="16" height="16" fill="currentColor" aria-hidden="true">
|
||||
<path
|
||||
@@ -47,6 +49,8 @@ function closeMenu() {
|
||||
:class="{ open: menuOpen }"
|
||||
@click="menuOpen = !menuOpen"
|
||||
aria-label="Toggle menu"
|
||||
:aria-expanded="menuOpen"
|
||||
aria-controls="mobile-navigation"
|
||||
>
|
||||
<span /><span /><span />
|
||||
</button>
|
||||
@@ -55,10 +59,11 @@ function closeMenu() {
|
||||
|
||||
<!-- Mobile menu -->
|
||||
<Transition name="menu">
|
||||
<div v-if="menuOpen" class="mobile-menu">
|
||||
<nav v-show="menuOpen" id="mobile-navigation" class="mobile-menu" aria-label="Mobile navigation">
|
||||
<a href="#features" class="mobile-link" @click="closeMenu">features</a>
|
||||
<a href="#captcha" class="mobile-link" @click="closeMenu">captcha</a>
|
||||
<a href="#how-it-works" class="mobile-link" @click="closeMenu">tiers</a>
|
||||
<a href="#proxy" class="mobile-link" @click="closeMenu">proxy</a>
|
||||
<a href="#compare" class="mobile-link" @click="closeMenu">benchmarks</a>
|
||||
<a href="#code" class="mobile-link" @click="closeMenu">quickstart</a>
|
||||
<a :href="docsUrl" class="mobile-link" @click="closeMenu" @keydown.enter="closeMenu">docs</a>
|
||||
@@ -77,7 +82,7 @@ function closeMenu() {
|
||||
GitHub
|
||||
<span v-if="stars" class="mobile-github-stars">★ {{ stars }}</span>
|
||||
</a>
|
||||
</div>
|
||||
</nav>
|
||||
</Transition>
|
||||
</header>
|
||||
|
||||
@@ -196,7 +201,9 @@ function closeMenu() {
|
||||
display: block;
|
||||
height: 1px;
|
||||
background: var(--text-muted);
|
||||
transition: all 0.2s;
|
||||
transition:
|
||||
opacity 0.2s,
|
||||
transform 0.2s;
|
||||
transform-origin: center;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
<script lang="ts" setup>
|
||||
const docsUrl = useDocsUrl()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<section id="proxy" class="section proxy-section">
|
||||
<div class="container">
|
||||
<p class="eyebrow">challenge-aware proxy</p>
|
||||
<h2 class="section-title">one proxy. challenge-aware by default.</h2>
|
||||
<p class="section-sub">
|
||||
Point any HTTP-compatible client at port 8192. Normal traffic takes the direct path; detected challenge pages
|
||||
automatically escalate through TRAWL's browser solver and return the resolved response.
|
||||
</p>
|
||||
|
||||
<div class="proxy-panel">
|
||||
<div
|
||||
v-motion
|
||||
:initial="{ opacity: 0, x: -20 }"
|
||||
:visible-once="{ opacity: 1, x: 0, transition: { duration: 450 } }"
|
||||
class="proxy-flow"
|
||||
>
|
||||
<div class="flow-row">
|
||||
<span class="flow-node">your client</span>
|
||||
<span class="flow-arrow">→</span>
|
||||
<span class="flow-node flow-node-accent">:8192</span>
|
||||
<span class="flow-arrow">→</span>
|
||||
<span class="flow-node">direct web</span>
|
||||
</div>
|
||||
<div class="flow-branch">
|
||||
<span class="branch-line">└─ challenge detected</span>
|
||||
<span class="flow-arrow">→</span>
|
||||
<span class="flow-node flow-node-accent">tier solver</span>
|
||||
<span class="flow-arrow">→</span>
|
||||
<span class="flow-node">resolved response</span>
|
||||
</div>
|
||||
|
||||
<div class="proxy-code">
|
||||
<span class="code-prompt">$</span>
|
||||
<code>curl --proxy http://localhost:8192 https://example.com</code>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-motion
|
||||
:initial="{ opacity: 0, x: 20 }"
|
||||
:visible-once="{ opacity: 1, x: 0, transition: { duration: 450, delay: 100 } }"
|
||||
class="proxy-capabilities"
|
||||
>
|
||||
<div class="capability">
|
||||
<span class="capability-mark">✓</span>
|
||||
<span><strong>General traffic</strong> HTTP, HTTPS, WebSockets, binary files, and downloads.</span>
|
||||
</div>
|
||||
<div class="capability">
|
||||
<span class="capability-mark">✓</span>
|
||||
<span><strong>Complete requests</strong> Authentication, cookies, custom headers, and request bodies.</span>
|
||||
</div>
|
||||
<div class="capability">
|
||||
<span class="capability-mark">✓</span>
|
||||
<span><strong>Download semantics</strong> Range requests, 206 responses, and large-file streaming.</span>
|
||||
</div>
|
||||
<div class="capability">
|
||||
<span class="capability-mark">✓</span>
|
||||
<span><strong>Automatic escalation</strong> Browser work starts only after a challenge is detected.</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="proxy-note">
|
||||
HTTPS interception requires the generated TRAWL CA certificate to be trusted by the client.
|
||||
<a :href="`${docsUrl}/proxy/ca-installation`" target="_blank" rel="noopener noreferrer">
|
||||
installation guide ↗
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.section-title {
|
||||
margin-bottom: 12px;
|
||||
color: var(--text);
|
||||
font-size: clamp(22px, 3vw, 34px);
|
||||
font-weight: 700;
|
||||
letter-spacing: -0.02em;
|
||||
}
|
||||
|
||||
.section-sub {
|
||||
max-width: 720px;
|
||||
margin-bottom: 42px;
|
||||
color: var(--text-muted);
|
||||
font-size: 13px;
|
||||
line-height: 1.75;
|
||||
}
|
||||
|
||||
.proxy-panel {
|
||||
display: grid;
|
||||
grid-template-columns: 1.15fr 0.85fr;
|
||||
border: 1px solid var(--border-strong);
|
||||
background: var(--bg);
|
||||
}
|
||||
|
||||
.proxy-flow,
|
||||
.proxy-capabilities {
|
||||
padding: 34px;
|
||||
}
|
||||
|
||||
.proxy-flow {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
border-right: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.flow-row,
|
||||
.flow-branch {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.flow-branch {
|
||||
margin: 16px 0 0 72px;
|
||||
}
|
||||
|
||||
.flow-node {
|
||||
padding: 7px 10px;
|
||||
border: 1px solid var(--border-strong);
|
||||
color: var(--text);
|
||||
font-size: 10px;
|
||||
letter-spacing: 0.04em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.flow-node-accent {
|
||||
border-color: var(--accent);
|
||||
background: var(--accent-tint);
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.flow-arrow,
|
||||
.branch-line {
|
||||
color: var(--text-muted);
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.proxy-code {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
overflow-x: auto;
|
||||
margin-top: 32px;
|
||||
padding: 14px 16px;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--bg-subtle);
|
||||
color: var(--text-muted);
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.code-prompt,
|
||||
.capability-mark {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.proxy-capabilities {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
gap: 22px;
|
||||
}
|
||||
|
||||
.capability {
|
||||
display: grid;
|
||||
grid-template-columns: 16px 1fr;
|
||||
gap: 10px;
|
||||
color: var(--text-muted);
|
||||
font-size: 11px;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.capability strong {
|
||||
display: block;
|
||||
color: var(--text);
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.proxy-note {
|
||||
margin-top: 16px;
|
||||
color: var(--text-muted);
|
||||
font-size: 10px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.proxy-note a {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
@media (max-width: 800px) {
|
||||
.proxy-panel {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.proxy-flow {
|
||||
border-right: 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.proxy-flow,
|
||||
.proxy-capabilities {
|
||||
padding: 24px 20px;
|
||||
}
|
||||
|
||||
.flow-row,
|
||||
.flow-branch {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.flow-branch {
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.flow-arrow {
|
||||
transform: rotate(90deg);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -12,21 +12,21 @@ const tiers = [
|
||||
name: "Cached Browser Session",
|
||||
time: "~500ms",
|
||||
tag: "redis hit",
|
||||
desc: "Restores Cloudflare cookies from Redis into a warm pooled browser and navigates. No challenge re-solve. Returns the page in under a second. Stays cached for 1 hour.",
|
||||
desc: "Restores solved cookies from Redis into a warm pooled browser. No challenge re-solve while the target accepts the session. Returns the page in under a second. Cache TTL is configurable.",
|
||||
},
|
||||
{
|
||||
num: "03",
|
||||
name: "Live Cloudflare Solve",
|
||||
name: "Live Challenge Solve",
|
||||
time: "4–15s",
|
||||
tag: "fresh context",
|
||||
desc: "A fresh browser context triggers CF managed-mode — the fastest possible challenge path. Turnstile, reCAPTCHA, hCaptcha, and GeeTest are solved automatically. Cookies are cached on success.",
|
||||
desc: "A fresh Camoufox context runs the matching Cloudflare, Akamai, or Imperva flow. Turnstile, reCAPTCHA, hCaptcha, and GeeTest are solved automatically when encountered. Cookies are cached on success.",
|
||||
},
|
||||
{
|
||||
num: "04",
|
||||
name: "Residential Proxy",
|
||||
time: "8–25s",
|
||||
tag: "optional",
|
||||
desc: "Same as Tier 3 but routes through a residential proxy. Only triggered when the datacenter IP is reputation-flagged by Cloudflare. Bandwidth cost is incurred only when truly necessary.",
|
||||
desc: "Same as Tier 3 but routed through a residential proxy. Only triggered after the direct or datacenter path fails, so paid residential bandwidth is used only when necessary.",
|
||||
},
|
||||
]
|
||||
</script>
|
||||
|
||||
+63
-40
@@ -1,8 +1,18 @@
|
||||
const title = "Self-Hosted Web Scraping & Challenge Bypass"
|
||||
const description =
|
||||
"Adaptive, self-hosted web scraping engine for Cloudflare, Akamai, Imperva, and CAPTCHA-protected sites, with a challenge-aware HTTP/HTTPS proxy."
|
||||
const socialDescription =
|
||||
"Scrape protected websites with adaptive HTTP and browser tiers, CAPTCHA handling, session caching, and a challenge-aware proxy."
|
||||
|
||||
export default defineNuxtConfig({
|
||||
compatibilityDate: "2026-06-23",
|
||||
|
||||
modules: ["@nuxt/fonts", "@nuxtjs/color-mode", "@nuxtjs/seo", "@vueuse/motion/nuxt"],
|
||||
|
||||
alias: {
|
||||
tslib: "tslib/tslib.es6.js",
|
||||
},
|
||||
|
||||
colorMode: {
|
||||
classSuffix: "",
|
||||
fallback: "dark",
|
||||
@@ -14,7 +24,8 @@ export default defineNuxtConfig({
|
||||
{
|
||||
name: "Geist Mono",
|
||||
provider: "google",
|
||||
weights: ["300", "400", "500", "600", "700"],
|
||||
weights: ["400", "500", "600", "700"],
|
||||
subsets: ["latin"],
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -22,54 +33,66 @@ export default defineNuxtConfig({
|
||||
site: {
|
||||
url: "https://trawl.dev",
|
||||
name: "TRAWL",
|
||||
description:
|
||||
"Self-hosted web scraping engine with adaptive tier execution. Solves Cloudflare challenges natively in 4–15s. Returns cached results in under 500ms. Drop-in FlareSolverr replacement for Prowlarr, Jackett, Sonarr, and Radarr.",
|
||||
description,
|
||||
defaultLocale: "en",
|
||||
},
|
||||
|
||||
seo: {
|
||||
meta: {
|
||||
description,
|
||||
themeColor: "#00e87a",
|
||||
ogType: "website",
|
||||
ogTitle: `TRAWL — ${title}`,
|
||||
ogDescription: socialDescription,
|
||||
twitterCard: "summary_large_image",
|
||||
twitterTitle: `TRAWL — ${title}`,
|
||||
twitterDescription:
|
||||
"An adaptive web scraping engine for protected sites, with CAPTCHA handling and a general HTTP/HTTPS proxy.",
|
||||
},
|
||||
},
|
||||
|
||||
ogImage: {
|
||||
enabled: false,
|
||||
},
|
||||
|
||||
robots: {
|
||||
credits: false,
|
||||
},
|
||||
|
||||
sitemap: {
|
||||
zeroRuntime: true,
|
||||
},
|
||||
|
||||
nitro: {
|
||||
compressPublicAssets: true,
|
||||
routeRules: {
|
||||
"/**": {
|
||||
headers: {
|
||||
"Content-Security-Policy":
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self' https://api.github.com; frame-ancestors 'none'; base-uri 'self'; form-action 'self'",
|
||||
"Cache-Control": "public, max-age=0, must-revalidate",
|
||||
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
||||
"Referrer-Policy": "strict-origin-when-cross-origin",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"X-Frame-Options": "DENY",
|
||||
},
|
||||
},
|
||||
"/_nuxt/**": {
|
||||
headers: {
|
||||
"Cache-Control": "public, max-age=31536000, immutable",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
app: {
|
||||
head: {
|
||||
title: "TRAWL — Adaptive Web Scraping Engine",
|
||||
title,
|
||||
templateParams: {
|
||||
siteName: "TRAWL",
|
||||
separator: "—",
|
||||
},
|
||||
meta: [
|
||||
{ charset: "utf-8" },
|
||||
{ name: "viewport", content: "width=device-width, initial-scale=1" },
|
||||
{ name: "theme-color", content: "#00e87a" },
|
||||
{ name: "robots", content: "index, follow" },
|
||||
{
|
||||
name: "keywords",
|
||||
content:
|
||||
"cloudflare bypass, web scraping, flaresolver replacement, self-hosted scraper, browser automation, captcha solver, prowlarr, jackett, *arr",
|
||||
},
|
||||
{ property: "og:type", content: "website" },
|
||||
{
|
||||
property: "og:title",
|
||||
content: "TRAWL — Adaptive Web Scraping Engine",
|
||||
},
|
||||
{
|
||||
property: "og:description",
|
||||
content:
|
||||
"Solve Cloudflare challenges natively. Return cached results in <500ms. Self-hosted, zero external APIs, FlareSolverr-compatible.",
|
||||
},
|
||||
{ property: "og:url", content: "https://trawl.dev" },
|
||||
{ name: "twitter:card", content: "summary" },
|
||||
{
|
||||
name: "twitter:title",
|
||||
content: "TRAWL — Adaptive Web Scraping Engine",
|
||||
},
|
||||
{
|
||||
name: "twitter:description",
|
||||
content:
|
||||
"Self-hosted scraping engine. Cloudflare bypass, captcha solving, session caching. Drop-in FlareSolverr replacement.",
|
||||
},
|
||||
],
|
||||
link: [
|
||||
{ rel: "canonical", href: "https://trawl.dev" },
|
||||
{ rel: "icon", type: "image/svg+xml", href: "/favicon.svg" },
|
||||
],
|
||||
link: [{ rel: "icon", type: "image/svg+xml", href: "/favicon.svg" }],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
User-agent: *
|
||||
Allow: /
|
||||
Sitemap: https://trawl.dev/sitemap.xml
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
@@ -1,8 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url>
|
||||
<loc>https://trawl.dev/</loc>
|
||||
<changefreq>weekly</changefreq>
|
||||
<priority>1.0</priority>
|
||||
</url>
|
||||
</urlset>
|
||||
Reference in New Issue
Block a user