fix: SQLite database locked errors + add error status for 4xx/5xx
SQLite locking: - Enable WAL journal mode in init_db (readers don't block writers) - Set busy_timeout=30000ms in init_db - Add timeout=30 to every aiosqlite.connect() across db.py, validator.py, enricher.py, main.py so connections wait up to 30s instead of crashing Error status: - 4xx/5xx HTTP responses are now prescreen_status='error' (server alive but broken/blocking) instead of 'live' - Added 'error' counter to validator stats and orange Error stat box in UI - Added ps-error CSS class (orange) and filter option in Browse tab Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -66,6 +66,7 @@ _val_stats: dict = {
|
||||
"processed": 0,
|
||||
"live": 0,
|
||||
"dead": 0,
|
||||
"error": 0,
|
||||
"parked": 0,
|
||||
"redirect": 0,
|
||||
"skipped": 0,
|
||||
@@ -148,6 +149,11 @@ async def _check_domain(domain: str) -> dict:
|
||||
result["prescreen_status"] = "parked"
|
||||
return result
|
||||
|
||||
# 4xx / 5xx = server is alive but site is broken/blocking
|
||||
if resp.status_code >= 400:
|
||||
result["prescreen_status"] = "error"
|
||||
return result
|
||||
|
||||
result["prescreen_status"] = "live"
|
||||
return result
|
||||
|
||||
@@ -201,7 +207,7 @@ async def _filter_unvalidated(domains: list[str], rescan_dead: bool = False) ->
|
||||
condition = "prescreen_status IS NOT NULL AND prescreen_status != 'dead'"
|
||||
else:
|
||||
condition = "prescreen_status IS NOT NULL"
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
|
||||
async with db.execute(
|
||||
f"SELECT domain FROM enriched_domains "
|
||||
f"WHERE domain IN ({placeholders}) AND {condition}",
|
||||
@@ -212,7 +218,7 @@ async def _filter_unvalidated(domains: list[str], rescan_dead: bool = False) ->
|
||||
|
||||
|
||||
async def _save_batch(results: list[dict]):
|
||||
async with aiosqlite.connect(SQLITE_PATH) as db:
|
||||
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
|
||||
for r in results:
|
||||
await db.execute(
|
||||
"""INSERT INTO enriched_domains
|
||||
@@ -319,7 +325,7 @@ def start_validator(tld_filter: Optional[str] = None, rescan_dead: bool = False)
|
||||
# domains, so restarting from 0 is safe and fast even for the same TLD.
|
||||
_val_stats.update(
|
||||
running=True,
|
||||
processed=0, live=0, dead=0, parked=0,
|
||||
processed=0, live=0, dead=0, error=0, parked=0,
|
||||
redirect=0, skipped=0, offset=0, rate=0.0,
|
||||
tld_filter=tld_filter,
|
||||
rescan_dead=rescan_dead,
|
||||
|
||||
Reference in New Issue
Block a user