2026-06-17 11:45:00 +02:00
|
|
|
|
---
|
|
|
|
|
|
title: Native API
|
|
|
|
|
|
description: POST /scrape — the native TRAWL endpoint with full tier control.
|
|
|
|
|
|
---
|
|
|
|
|
|
|
|
|
|
|
|
# `POST /scrape` — Native API
|
|
|
|
|
|
|
|
|
|
|
|
The native endpoint exposes TRAWL's full feature set: tier capping, session IDs, and rich timing metadata.
|
|
|
|
|
|
|
|
|
|
|
|
## Request
|
|
|
|
|
|
|
|
|
|
|
|
```typescript
|
|
|
|
|
|
interface ScrapeRequest {
|
|
|
|
|
|
url: string
|
2026-06-26 22:05:47 +02:00
|
|
|
|
maxTimeout?: number // ms, default 60000
|
|
|
|
|
|
skipHttp?: boolean // skip Tier 1 (plain fetch), default false
|
|
|
|
|
|
maxTier?: 1 | 2 | 3 | 4 // cap escalation at this tier
|
|
|
|
|
|
sessionId?: string // sticky session override key
|
|
|
|
|
|
headers?: Record<string, string> // custom headers forwarded to the target
|
2026-06-17 11:45:00 +02:00
|
|
|
|
}
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
### Fields
|
|
|
|
|
|
|
|
|
|
|
|
| Field | Type | Default | Description |
|
|
|
|
|
|
|-------|------|---------|-------------|
|
|
|
|
|
|
| `url` | string | — | The URL to scrape |
|
|
|
|
|
|
| `maxTimeout` | number | 60000 | Max total time in milliseconds |
|
|
|
|
|
|
| `skipHttp` | boolean | false | Skip Tier 1 (go straight to browser) |
|
|
|
|
|
|
| `maxTier` | 1–4 | 4 | Never escalate beyond this tier |
|
|
|
|
|
|
| `sessionId` | string | hostname | Override the Redis session key |
|
2026-06-26 22:05:47 +02:00
|
|
|
|
| `headers` | object | — | Custom headers forwarded to the target across all tiers — see [Custom Headers](/api-reference/custom-headers) |
|
2026-06-17 11:45:00 +02:00
|
|
|
|
|
|
|
|
|
|
## Response
|
|
|
|
|
|
|
|
|
|
|
|
```typescript
|
|
|
|
|
|
interface ScrapeResult {
|
|
|
|
|
|
url: string
|
|
|
|
|
|
html: string
|
|
|
|
|
|
cookies: Cookie[]
|
|
|
|
|
|
userAgent: string
|
|
|
|
|
|
statusCode: number
|
|
|
|
|
|
tier: 1 | 2 | 3 | 4 // which tier succeeded
|
|
|
|
|
|
sessionCached: boolean // true if a cached session was used
|
|
|
|
|
|
timings: TierResult[] // per-tier attempt history
|
|
|
|
|
|
totalMs: number
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
interface TierResult {
|
|
|
|
|
|
tier: 1 | 2 | 3 | 4
|
|
|
|
|
|
status: 'success' | 'blocked' | 'needs-js' | 'timeout' | 'error' | 'skipped'
|
|
|
|
|
|
durationMs: number
|
|
|
|
|
|
reason?: string
|
|
|
|
|
|
}
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
## Examples
|
|
|
|
|
|
|
|
|
|
|
|
### Minimal request
|
|
|
|
|
|
|
|
|
|
|
|
```bash
|
|
|
|
|
|
curl -s -X POST http://localhost:8191/scrape \
|
|
|
|
|
|
-H "Content-Type: application/json" \
|
|
|
|
|
|
-d '{ "url": "https://nowsecure.nl" }' | jq '{tier, totalMs, sessionCached}'
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
### Force browser only (skip plain HTTP)
|
|
|
|
|
|
|
|
|
|
|
|
```bash
|
|
|
|
|
|
curl -s -X POST http://localhost:8191/scrape \
|
|
|
|
|
|
-H "Content-Type: application/json" \
|
|
|
|
|
|
-d '{
|
|
|
|
|
|
"url": "https://nowsecure.nl",
|
|
|
|
|
|
"skipHttp": true,
|
|
|
|
|
|
"maxTier": 3
|
|
|
|
|
|
}'
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
### Inspect timing breakdown
|
|
|
|
|
|
|
|
|
|
|
|
```javascript
|
|
|
|
|
|
const res = await fetch('http://localhost:8191/scrape', {
|
|
|
|
|
|
method: 'POST',
|
|
|
|
|
|
headers: { 'Content-Type': 'application/json' },
|
|
|
|
|
|
body: JSON.stringify({ url: 'https://nowsecure.nl' }),
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
|
|
const result = await res.json()
|
|
|
|
|
|
|
|
|
|
|
|
console.log(`Tier used: ${result.tier}`)
|
|
|
|
|
|
console.log(`Session cached: ${result.sessionCached}`)
|
|
|
|
|
|
console.log(`Total: ${result.totalMs}ms`)
|
|
|
|
|
|
|
|
|
|
|
|
for (const t of result.timings) {
|
|
|
|
|
|
console.log(` Tier ${t.tier}: ${t.status} in ${t.durationMs}ms`)
|
|
|
|
|
|
}
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
### Example response
|
|
|
|
|
|
|
|
|
|
|
|
```json
|
|
|
|
|
|
{
|
|
|
|
|
|
"url": "https://nowsecure.nl",
|
|
|
|
|
|
"html": "<!DOCTYPE html>...",
|
|
|
|
|
|
"cookies": [
|
|
|
|
|
|
{ "name": "cf_clearance", "value": "abc123...", "domain": ".nowsecure.nl", "path": "/", "expires": 1700003600, "httpOnly": false, "secure": true }
|
|
|
|
|
|
],
|
|
|
|
|
|
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...",
|
|
|
|
|
|
"statusCode": 200,
|
|
|
|
|
|
"tier": 2,
|
|
|
|
|
|
"sessionCached": true,
|
|
|
|
|
|
"timings": [
|
|
|
|
|
|
{ "tier": 1, "status": "needs-js", "durationMs": 85 },
|
|
|
|
|
|
{ "tier": 2, "status": "success", "durationMs": 512 }
|
|
|
|
|
|
],
|
|
|
|
|
|
"totalMs": 600
|
|
|
|
|
|
}
|
|
|
|
|
|
```
|
2026-06-30 00:22:57 +02:00
|
|
|
|
|
|
|
|
|
|
## Error response
|
|
|
|
|
|
|
|
|
|
|
|
HTTP status codes:
|
|
|
|
|
|
|
|
|
|
|
|
| Code | Meaning |
|
|
|
|
|
|
|------|---------|
|
|
|
|
|
|
| 200 | `tier` succeeded |
|
|
|
|
|
|
| 400 | Malformed request body |
|
|
|
|
|
|
| 429 | Pool exhausted — all browsers busy past `BROWSER_ACQUIRE_TIMEOUT_MS` |
|
|
|
|
|
|
| 503 | Browser pool initializing |
|
|
|
|
|
|
| 500 | Internal error |
|
|
|
|
|
|
|
|
|
|
|
|
For 429 pool-exhaustion errors, the body is a **FlareSolverr v2 envelope** (same shape `/v1` uses) so clients can parse both endpoints uniformly:
|
|
|
|
|
|
|
|
|
|
|
|
```json
|
|
|
|
|
|
{
|
|
|
|
|
|
"status": "error",
|
|
|
|
|
|
"message": "Browser pool saturated, retry shortly",
|
|
|
|
|
|
"startTimestamp": 1700000000000,
|
|
|
|
|
|
"endTimestamp": 1700000015000,
|
|
|
|
|
|
"version": "2.0.0",
|
|
|
|
|
|
"solution": {
|
|
|
|
|
|
"url": "https://nowsecure.nl",
|
|
|
|
|
|
"status": 0,
|
|
|
|
|
|
"headers": {},
|
|
|
|
|
|
"response": "",
|
|
|
|
|
|
"cookies": [],
|
|
|
|
|
|
"userAgent": ""
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
For 400 / 503 / 500 the body is the native shape `{ "error": "Human-readable message" }`.
|
