Merge pull request #23 from germondai/22-bug-js-only-challenge-pages-bypass-tier-1-escalation

22 bug js only challenge pages bypass tier 1 escalation

packages/tiers/src/solvers/hcaptcha.ts

@@ -1,20 +1,33 @@
-// hCaptcha solver — browser-only, no external services.
+// hCaptcha solver — checkbox auto-pass + audio STT fallback.
 //
-// With a real Chrome fingerprint and a non-datacenter IP, hCaptcha's risk
-// scoring sometimes auto-passes the checkbox without showing an image challenge.
-// Success rate varies by site sitekey difficulty and IP reputation.
+// Flow:
+//   1. Click the hCaptcha checkbox. With a good IP and a Camoufox Firefox fingerprint,
+//      hCaptcha's risk scoring sometimes auto-passes without showing an image challenge.
+//   2. If a visual image challenge appears, switch to the audio challenge and solve via
+//      speech-to-text (Google's free API or a configured Whisper-compatible endpoint).
+//   3. Submit the transcribed digit string and verify.
 //
-// There is no fully free, reliable way to solve hCaptcha image grids without
-// an AI/ML model or a paid solving service. We attempt the checkbox and return
-// whether it auto-passed.
+// Site owners can disable the audio option per sitekey — when that happens the solver
+// gives up cleanly and returns false. There is no fully free, reliable way to solve
+// hCaptcha image grids without an AI/ML model or a paid solving service.
 
-import type { Page } from "patchright"
+import type { FrameLocator, Page } from "patchright"
+import { transcribeAudio } from "./stt"
 
 // hCaptcha widget iframe. newassets.hcaptcha.com is their CDN; don't filter by title
 // since the title attribute may not be set yet or may vary across versions.
 const WIDGET_FRAME = 'iframe[src*="hcaptcha.com"]'
 
-export async function solveHcaptcha(page: Page, timeoutMs = 15_000): Promise<boolean> {
+// Selectors within the hCaptcha challenge UI. Source: Asmodei513/hcaptcha-solver,
+// NotHarshhaa/hc_audio_challenger, dev1siN/hc-audio-solver (cross-verified).
+const AUDIO_BUTTON = "#audio-button"
+const AUDIO_RESPONSE = "textarea#audio-response"
+const AUDIO_SUBMIT = "#audio-submit"
+const RELOAD_BUTTON = 'button[aria-label="Get a new challenge"]'
+
+const MAX_AUDIO_ATTEMPTS = 3
+
+export async function solveHcaptcha(page: Page, timeoutMs = 30_000): Promise<boolean> {
   try {
     const hasWidget = await page
       .waitForSelector(WIDGET_FRAME, { timeout: 8000, state: "attached" })
@@ -25,29 +38,142 @@ export async function solveHcaptcha(page: Page, timeoutMs = 15_000): Promise<boo
     // Pick the first hCaptcha iframe (may be multiple on demo pages with difficulty tabs)
     const widget = page.frameLocator(WIDGET_FRAME).first()
 
-    // Click the checkbox — force:true handles widgets inside hidden tab containers
+    // Step 1: click the checkbox
     await widget.locator("#checkbox").click({ timeout: 5000, force: true })
     console.log("[hcaptcha] clicked checkbox")
 
-    await new Promise((r) => setTimeout(r, Math.min(timeoutMs - 1000, 3000)))
+    // Step 2: give hCaptcha's risk scoring time to run
+    await new Promise((r) => setTimeout(r, 2500))
 
-    const passed = await widget
-      .locator('[aria-checked="true"]')
-      .isVisible({ timeout: 1000 })
-      .catch(() => false)
-    if (passed) {
+    // Step 3: check for auto-pass
+    if (
+      await widget
+        .locator('[aria-checked="true"]')
+        .isVisible({ timeout: 1000 })
+        .catch(() => false)
+    ) {
       console.log("[hcaptcha] auto-passed ✓")
       return true
     }
 
-    console.log("[hcaptcha] image challenge appeared — cannot solve without AI")
-    return false
+    // Step 4: image challenge appeared — try audio fallback within remaining budget
+    const remaining = Math.max(timeoutMs - 3000, 5000)
+    return await solveHcaptchaAudio(widget, remaining)
   } catch (err) {
     console.log("[hcaptcha] error:", err instanceof Error ? err.message : err)
     return false
   }
 }
 
+async function solveHcaptchaAudio(widget: FrameLocator, remainingMs: number): Promise<boolean> {
+  if (remainingMs < 5000) {
+    console.log("[hcaptcha] not enough time for audio attempt")
+    return false
+  }
+
+  // Click the audio toggle. Some sitekeys disable audio entirely — fail cleanly.
+  const hasAudioButton = await widget
+    .locator(AUDIO_BUTTON)
+    .waitFor({ timeout: 3000, state: "attached" })
+    .then(() => true)
+    .catch(() => false)
+  if (!hasAudioButton) {
+    console.log("[hcaptcha] audio challenge not available for this sitekey")
+    return false
+  }
+  await widget
+    .locator(AUDIO_BUTTON)
+    .click({ timeout: 5000, force: true })
+    .catch(() => {})
+  console.log("[hcaptcha] switching to audio challenge")
+
+  const deadline = Date.now() + remainingMs - 1000
+  let attempt = 0
+
+  while (Date.now() < deadline && attempt < MAX_AUDIO_ATTEMPTS) {
+    attempt++
+
+    // Wait for the audio element to appear. Some hCaptcha versions render it lazily
+    // after the button click.
+    const hasAudio = await widget
+      .locator("audio")
+      .waitFor({ timeout: 8000 })
+      .then(() => true)
+      .catch(() => false)
+    if (!hasAudio) {
+      console.log(`[hcaptcha] audio element not found (attempt ${attempt})`)
+      continue
+    }
+
+    // Get the audio URL via the JS property — more reliable than getAttribute("src")
+    // because hCaptcha sets src dynamically after the audio challenge loads.
+    const audioHref = await widget
+      .locator("audio")
+      .evaluate((el) => (el as HTMLAudioElement).src || "")
+      .catch(() => "")
+
+    if (!audioHref || audioHref.startsWith("blob:")) {
+      console.log(`[hcaptcha] audio URL not usable: ${audioHref?.slice(0, 60) ?? "empty"}`)
+      await widget
+        .locator(RELOAD_BUTTON)
+        .click({ timeout: 3000, force: true })
+        .catch(() => {})
+      await new Promise((r) => setTimeout(r, 2000))
+      continue
+    }
+
+    console.log(`[hcaptcha] transcribing audio (attempt ${attempt})`)
+
+    const signal = AbortSignal.timeout(Math.max(deadline - Date.now() - 3000, 5000))
+    const answer = await transcribeAudio(audioHref, signal)
+
+    if (!answer) {
+      console.log(`[hcaptcha] transcription empty, reloading audio`)
+      await widget
+        .locator(RELOAD_BUTTON)
+        .click({ timeout: 3000, force: true })
+        .catch(() => {})
+      await new Promise((r) => setTimeout(r, 1500))
+      continue
+    }
+
+    console.log(`[hcaptcha] answer: ${answer}`)
+
+    // Submit
+    await widget
+      .locator(AUDIO_RESPONSE)
+      .fill(answer, { timeout: 3000 })
+      .catch(() => {})
+    await widget
+      .locator(AUDIO_SUBMIT)
+      .click({ timeout: 3000 })
+      .catch(() => {})
+    await new Promise((r) => setTimeout(r, 2000))
+
+    // Verify pass — hCaptcha marks the widget via aria-checked when solved
+    if (
+      await widget
+        .locator('[aria-checked="true"]')
+        .isVisible({ timeout: 2000 })
+        .catch(() => false)
+    ) {
+      console.log("[hcaptcha] solved via audio ✓")
+      return true
+    }
+
+    // Wrong answer — reload the challenge and try again
+    console.log(`[hcaptcha] wrong answer, reloading challenge`)
+    await widget
+      .locator(RELOAD_BUTTON)
+      .click({ timeout: 3000, force: true })
+      .catch(() => {})
+    await new Promise((r) => setTimeout(r, 1500))
+  }
+
+  console.log(`[hcaptcha] exhausted retries (${attempt}/${MAX_AUDIO_ATTEMPTS})`)
+  return false
+}
+
 export async function hasHcaptchaWidget(page: Page, timeout = 2000): Promise<boolean> {
   return page
     .waitForSelector(WIDGET_FRAME, { timeout, state: "attached" })

packages/tiers/src/tier1.ts

@@ -1,6 +1,6 @@
 import { FINGERPRINT } from "@trawl/browser"
 import type { TierResult } from "@trawl/types"
-import { isBlocked, isCloudflarePage } from "./detect"
+import { hasHcaptcha, hasRecaptcha, hasTurnstile, isBlocked, isCloudflarePage } from "./detect"
 import { normalizeHtml } from "./html"
 
 export interface Tier1Result extends TierResult {
@@ -42,6 +42,21 @@ export async function runTier1(
       return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "cloudflare-challenge" }
     }
 
+    // JS-only challenges: the page's static HTML is just a shell that loads the
+    // captcha widget via <script src="...api.js">. Plain fetch sees the shell and
+    // would otherwise report success — but the real content (including the widget)
+    // only renders after JS executes. Escalate so Tier 3 runs the page in a browser,
+    // executes JS, and the solver can engage the actual widget.
+    if (hasHcaptcha(html)) {
+      return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "hcaptcha-shell" }
+    }
+    if (hasRecaptcha(html)) {
+      return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "recaptcha-shell" }
+    }
+    if (hasTurnstile(html)) {
+      return { tier: 1, status: "needs-js", durationMs: Date.now() - start, reason: "turnstile-shell" }
+    }
+
     if (isBlocked(res.status, html)) {
       return { tier: 1, status: "blocked", durationMs: Date.now() - start, reason: `http-${res.status}` }
     }

scripts/hcaptcha-audio-test.ts

@@ -0,0 +1,82 @@
+// Direct end-to-end test of the new hCaptcha audio STT fallback.
+// Uses the production BrowserPool from @trawl/browser (same code path the API uses)
+// so bun's module resolution can't trip over camoufox-js's internal playwright-core dep.
+//
+// Run inside the running container:
+//   docker exec -w /app trawl-hcaptcha-test bun run /tmp/hcaptcha-audio-test.ts
+
+import { BrowserPool } from "@trawl/browser"
+import { solveHcaptcha } from "../packages/tiers/src/solvers/hcaptcha.ts"
+
+process.on("uncaughtException", (err) => {
+  console.error("[uncaught]", err.message)
+  process.exit(1)
+})
+
+async function main() {
+  const pool = new BrowserPool({
+    size: 1,
+    headless: true,
+    geoip: true,
+    humanize: true,
+    block_webrtc: true,
+    disable_coop: true,
+  })
+
+  console.log("[test] initializing BrowserPool (size=1)...")
+  await pool.init()
+  console.log("[test] pool ready")
+
+  const handle = await pool.acquire("nopecha.com")
+  console.log("[test] acquired browser handle id=" + handle.id)
+
+  // BrowserHandle exposes context + browser, not page — create one from the context
+  // (same pattern apps/api/src/index.ts uses).
+  const page = await handle.context.newPage()
+
+  try {
+    const target = "https://nopecha.com/demo/hcaptcha"
+    console.log(`[test] visiting ${target}...`)
+    await page
+      .goto(target, { waitUntil: "domcontentloaded", timeout: 30000 })
+      .catch((e: Error) => console.log("[test] goto error:", e.message.slice(0, 120)))
+
+    // Give hCaptcha's api.js time to bootstrap the widget iframe.
+    console.log("[test] waiting 8s for hCaptcha widget to render...")
+    await new Promise((r) => setTimeout(r, 8000))
+    await page.waitForLoadState("networkidle", { timeout: 15000 }).catch(() => {})
+
+    const html = await page.content().catch(() => "")
+    console.log("[test] page.content() length:", html.length)
+    console.log("[test] URL:", page.url())
+    console.log("[test] Title:", await page.title().catch(() => "?"))
+
+    const frames = page
+      .frames()
+      .map((f: { url: () => string }) => f.url())
+      .filter((u: string) => u && u !== "about:blank")
+    console.log("[test] frames:", frames.slice(0, 10))
+
+    // ─── Run the new solver ────────────────────────────────────────────────
+    console.log("[test] calling solveHcaptcha(page, 45000)...")
+    const t0 = Date.now()
+    const solved = await solveHcaptcha(page, 45_000)
+    const elapsed = Date.now() - t0
+    console.log(`[test] solveHcaptcha returned: ${solved} (took ${elapsed}ms)`)
+
+    await pool.release(handle.id)
+    await pool.shutdown()
+    console.log("[test] DONE")
+    process.exit(solved ? 0 : 2)
+  } catch (e) {
+    console.error("[test] FAIL:", (e as Error).message)
+    await pool.release(handle.id).catch(() => {})
+    await pool.shutdown().catch(() => {})
+    process.exit(1)
+  }
+}
+
+main().catch((e: Error) => {
+  console.error("[test] FATAL:", e.message)
+  process.exit(1)
+})