fix: always retry https on any http failure, unify timeouts

- Any http failure (ConnectError, ConnectTimeout, ReadTimeout, TLS error
  inside a redirect chain) now falls through to an https retry instead
  of immediately marking dead.
- Use one unified 8s connect / 12s read timeout for both schemes so that
  http→https redirects followed inside the same client get a full TLS
  handshake window (previously 4s http timeout was too short for the
  https redirect hop).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-20 19:50:56 +02:00
parent 54c781773d
commit 989717e479

View File

@@ -108,15 +108,14 @@ async def _check_domain(domain: str) -> dict:
# will too, so we don't bother retrying on timeout errors. # will too, so we don't bother retrying on timeout errors.
# We DO retry https on ConnectError/RemoteProtocolError (port 80 closed or # We DO retry https on ConnectError/RemoteProtocolError (port 80 closed or
# speaking wrong protocol) because those servers are often https-only. # speaking wrong protocol) because those servers are often https-only.
timeouts = { # Use the same generous timeout for both schemes so that http→https redirects
"http": httpx.Timeout(connect=4, read=6, write=3, pool=5), # (followed inside the same client) also get enough time for the TLS handshake.
"https": httpx.Timeout(connect=6, read=10, write=3, pool=5), timeout = httpx.Timeout(connect=8, read=12, write=5, pool=8)
}
for scheme in ("http", "https"): for scheme in ("http", "https"):
try: try:
async with httpx.AsyncClient( async with httpx.AsyncClient(
timeout=timeouts[scheme], timeout=timeout,
follow_redirects=True, follow_redirects=True,
headers=_headers(), headers=_headers(),
verify=False, verify=False,
@@ -152,16 +151,14 @@ async def _check_domain(domain: str) -> dict:
result["prescreen_status"] = "live" result["prescreen_status"] = "live"
return result return result
except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.ConnectTimeout) as e: except Exception as e:
# Port refused, wrong protocol, or port 80 firewalled/timed-out # Any failure on http (connection refused, timeout, TLS error in redirect
# → server may be https-only, always retry on 443 # chain, etc.) → server may be https-only, always try 443 once.
if scheme == "http": if scheme == "http":
logger.debug("Validator %s: %s on http, trying https", domain, type(e).__name__) logger.debug("Validator %s: %s on http, trying https", domain, type(e).__name__)
continue continue
break # Both schemes failed → dead
except Exception as e: logger.debug("Validator %s (https): %s", domain, type(e).__name__)
# ReadTimeout or other error — connected but server too slow; mark dead
logger.debug("Validator %s (%s): %s", domain, scheme, type(e).__name__)
break break
result["load_time_ms"] = int((time.monotonic() - t0) * 1000) result["load_time_ms"] = int((time.monotonic() - t0) * 1000)