fix: SQLite database locked errors + add error status for 4xx/5xx

SQLite locking:
- Enable WAL journal mode in init_db (readers don't block writers)
- Set busy_timeout=30000ms in init_db
- Add timeout=30 to every aiosqlite.connect() across db.py, validator.py,
  enricher.py, main.py so connections wait up to 30s instead of crashing

Error status:
- 4xx/5xx HTTP responses are now prescreen_status='error' (server alive
  but broken/blocking) instead of 'live'
- Added 'error' counter to validator stats and orange Error stat box in UI
- Added ps-error CSS class (orange) and filter option in Browse tab

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-21 07:10:45 +02:00
parent 989717e479
commit db95876db2
5 changed files with 33 additions and 23 deletions

View File

@@ -66,6 +66,7 @@ _val_stats: dict = {
"processed": 0,
"live": 0,
"dead": 0,
"error": 0,
"parked": 0,
"redirect": 0,
"skipped": 0,
@@ -148,6 +149,11 @@ async def _check_domain(domain: str) -> dict:
result["prescreen_status"] = "parked"
return result
# 4xx / 5xx = server is alive but site is broken/blocking
if resp.status_code >= 400:
result["prescreen_status"] = "error"
return result
result["prescreen_status"] = "live"
return result
@@ -201,7 +207,7 @@ async def _filter_unvalidated(domains: list[str], rescan_dead: bool = False) ->
condition = "prescreen_status IS NOT NULL AND prescreen_status != 'dead'"
else:
condition = "prescreen_status IS NOT NULL"
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
async with db.execute(
f"SELECT domain FROM enriched_domains "
f"WHERE domain IN ({placeholders}) AND {condition}",
@@ -212,7 +218,7 @@ async def _filter_unvalidated(domains: list[str], rescan_dead: bool = False) ->
async def _save_batch(results: list[dict]):
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
for r in results:
await db.execute(
"""INSERT INTO enriched_domains
@@ -319,7 +325,7 @@ def start_validator(tld_filter: Optional[str] = None, rescan_dead: bool = False)
# domains, so restarting from 0 is safe and fast even for the same TLD.
_val_stats.update(
running=True,
processed=0, live=0, dead=0, parked=0,
processed=0, live=0, dead=0, error=0, parked=0,
redirect=0, skipped=0, offset=0, rate=0.0,
tld_filter=tld_filter,
rescan_dead=rescan_dead,