mirror of
https://github.com/germondai/trawl.git
synced 2026-08-17 12:11:23 +02:00
117 lines
2.9 KiB
Markdown
117 lines
2.9 KiB
Markdown
|
---
|
|||
|
|
title: Native API
|
|||
|
|
description: POST /scrape — the native TRAWL endpoint with full tier control.
|
|||
|
|
---
|
|||
|
|
|
|||
|
|
# `POST /scrape` — Native API
|
|||
|
|
|
|||
|
|
The native endpoint exposes TRAWL's full feature set: tier capping, session IDs, and rich timing metadata.
|
|||
|
|
|
|||
|
|
## Request
|
|||
|
|
|
|||
|
|
```typescript
|
|||
|
|
interface ScrapeRequest {
|
|||
|
|
url: string
|
|||
|
|
maxTimeout?: number // ms, default 60000
|
|||
|
|
skipHttp?: boolean // skip Tier 1 (plain fetch), default false
|
|||
|
|
maxTier?: 1 | 2 | 3 | 4 // cap escalation at this tier
|
|||
|
|
sessionId?: string // sticky session override key
|
|||
|
|
}
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
### Fields
|
|||
|
|
|
|||
|
|
| Field | Type | Default | Description |
|
|||
|
|
|-------|------|---------|-------------|
|
|||
|
|
| `url` | string | — | The URL to scrape |
|
|||
|
|
| `maxTimeout` | number | 60000 | Max total time in milliseconds |
|
|||
|
|
| `skipHttp` | boolean | false | Skip Tier 1 (go straight to browser) |
|
|||
|
|
| `maxTier` | 1–4 | 4 | Never escalate beyond this tier |
|
|||
|
|
| `sessionId` | string | hostname | Override the Redis session key |
|
|||
|
|
|
|||
|
|
## Response
|
|||
|
|
|
|||
|
|
```typescript
|
|||
|
|
interface ScrapeResult {
|
|||
|
|
url: string
|
|||
|
|
html: string
|
|||
|
|
cookies: Cookie[]
|
|||
|
|
userAgent: string
|
|||
|
|
statusCode: number
|
|||
|
|
tier: 1 | 2 | 3 | 4 // which tier succeeded
|
|||
|
|
sessionCached: boolean // true if a cached session was used
|
|||
|
|
timings: TierResult[] // per-tier attempt history
|
|||
|
|
totalMs: number
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
interface TierResult {
|
|||
|
|
tier: 1 | 2 | 3 | 4
|
|||
|
|
status: 'success' | 'blocked' | 'needs-js' | 'timeout' | 'error' | 'skipped'
|
|||
|
|
durationMs: number
|
|||
|
|
reason?: string
|
|||
|
|
}
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
## Examples
|
|||
|
|
|
|||
|
|
### Minimal request
|
|||
|
|
|
|||
|
|
```bash
|
|||
|
|
curl -s -X POST http://localhost:8191/scrape \
|
|||
|
|
-H "Content-Type: application/json" \
|
|||
|
|
-d '{ "url": "https://nowsecure.nl" }' | jq '{tier, totalMs, sessionCached}'
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
### Force browser only (skip plain HTTP)
|
|||
|
|
|
|||
|
|
```bash
|
|||
|
|
curl -s -X POST http://localhost:8191/scrape \
|
|||
|
|
-H "Content-Type: application/json" \
|
|||
|
|
-d '{
|
|||
|
|
"url": "https://nowsecure.nl",
|
|||
|
|
"skipHttp": true,
|
|||
|
|
"maxTier": 3
|
|||
|
|
}'
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
### Inspect timing breakdown
|
|||
|
|
|
|||
|
|
```javascript
|
|||
|
|
const res = await fetch('http://localhost:8191/scrape', {
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: { 'Content-Type': 'application/json' },
|
|||
|
|
body: JSON.stringify({ url: 'https://nowsecure.nl' }),
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
const result = await res.json()
|
|||
|
|
|
|||
|
|
console.log(`Tier used: ${result.tier}`)
|
|||
|
|
console.log(`Session cached: ${result.sessionCached}`)
|
|||
|
|
console.log(`Total: ${result.totalMs}ms`)
|
|||
|
|
|
|||
|
|
for (const t of result.timings) {
|
|||
|
|
console.log(` Tier ${t.tier}: ${t.status} in ${t.durationMs}ms`)
|
|||
|
|
}
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
### Example response
|
|||
|
|
|
|||
|
|
```json
|
|||
|
|
{
|
|||
|
|
"url": "https://nowsecure.nl",
|
|||
|
|
"html": "<!DOCTYPE html>...",
|
|||
|
|
"cookies": [
|
|||
|
|
{ "name": "cf_clearance", "value": "abc123...", "domain": ".nowsecure.nl", "path": "/", "expires": 1700003600, "httpOnly": false, "secure": true }
|
|||
|
|
],
|
|||
|
|
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)...",
|
|||
|
|
"statusCode": 200,
|
|||
|
|
"tier": 2,
|
|||
|
|
"sessionCached": true,
|
|||
|
|
"timings": [
|
|||
|
|
{ "tier": 1, "status": "needs-js", "durationMs": 85 },
|
|||
|
|
{ "tier": 2, "status": "success", "durationMs": 512 }
|
|||
|
|
],
|
|||
|
|
"totalMs": 600
|
|||
|
|
}
|
|||
|
|
```
|