fix: always retry https on any http failure, unify timeouts
- Any http failure (ConnectError, ConnectTimeout, ReadTimeout, TLS error inside a redirect chain) now falls through to an https retry instead of immediately marking dead. - Use one unified 8s connect / 12s read timeout for both schemes so that http→https redirects followed inside the same client get a full TLS handshake window (previously 4s http timeout was too short for the https redirect hop). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -108,15 +108,14 @@ async def _check_domain(domain: str) -> dict:
|
|||||||
# will too, so we don't bother retrying on timeout errors.
|
# will too, so we don't bother retrying on timeout errors.
|
||||||
# We DO retry https on ConnectError/RemoteProtocolError (port 80 closed or
|
# We DO retry https on ConnectError/RemoteProtocolError (port 80 closed or
|
||||||
# speaking wrong protocol) because those servers are often https-only.
|
# speaking wrong protocol) because those servers are often https-only.
|
||||||
timeouts = {
|
# Use the same generous timeout for both schemes so that http→https redirects
|
||||||
"http": httpx.Timeout(connect=4, read=6, write=3, pool=5),
|
# (followed inside the same client) also get enough time for the TLS handshake.
|
||||||
"https": httpx.Timeout(connect=6, read=10, write=3, pool=5),
|
timeout = httpx.Timeout(connect=8, read=12, write=5, pool=8)
|
||||||
}
|
|
||||||
|
|
||||||
for scheme in ("http", "https"):
|
for scheme in ("http", "https"):
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(
|
||||||
timeout=timeouts[scheme],
|
timeout=timeout,
|
||||||
follow_redirects=True,
|
follow_redirects=True,
|
||||||
headers=_headers(),
|
headers=_headers(),
|
||||||
verify=False,
|
verify=False,
|
||||||
@@ -152,16 +151,14 @@ async def _check_domain(domain: str) -> dict:
|
|||||||
result["prescreen_status"] = "live"
|
result["prescreen_status"] = "live"
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.ConnectTimeout) as e:
|
except Exception as e:
|
||||||
# Port refused, wrong protocol, or port 80 firewalled/timed-out
|
# Any failure on http (connection refused, timeout, TLS error in redirect
|
||||||
# → server may be https-only, always retry on 443
|
# chain, etc.) → server may be https-only, always try 443 once.
|
||||||
if scheme == "http":
|
if scheme == "http":
|
||||||
logger.debug("Validator %s: %s on http, trying https", domain, type(e).__name__)
|
logger.debug("Validator %s: %s on http, trying https", domain, type(e).__name__)
|
||||||
continue
|
continue
|
||||||
break
|
# Both schemes failed → dead
|
||||||
except Exception as e:
|
logger.debug("Validator %s (https): %s", domain, type(e).__name__)
|
||||||
# ReadTimeout or other error — connected but server too slow; mark dead
|
|
||||||
logger.debug("Validator %s (%s): %s", domain, scheme, type(e).__name__)
|
|
||||||
break
|
break
|
||||||
|
|
||||||
result["load_time_ms"] = int((time.monotonic() - t0) * 1000)
|
result["load_time_ms"] = int((time.monotonic() - t0) * 1000)
|
||||||
|
|||||||
Reference in New Issue
Block a user