fix: SQLite database locked errors + add error status for 4xx/5xx

SQLite locking:
- Enable WAL journal mode in init_db (readers don't block writers)
- Set busy_timeout=30000ms in init_db
- Add timeout=30 to every aiosqlite.connect() across db.py, validator.py,
  enricher.py, main.py so connections wait up to 30s instead of crashing

Error status:
- 4xx/5xx HTTP responses are now prescreen_status='error' (server alive
  but broken/blocking) instead of 'live'
- Added 'error' counter to validator stats and orange Error stat box in UI
- Added ps-error CSS class (orange) and filter option in Browse tab

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-21 07:10:45 +02:00
parent 989717e479
commit db95876db2
5 changed files with 33 additions and 23 deletions

View File

@@ -251,7 +251,7 @@ async def enrich_domain(domain: str) -> dict:
async def save_enriched(data: dict):
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
await db.execute(
"""INSERT INTO enriched_domains
(domain, is_live, status_code, ssl_valid, ssl_expiry_days, cms,
@@ -284,7 +284,7 @@ async def save_enriched(data: dict):
async def mark_job(domain: str, status: str, error: str = None):
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
if status == "running":
await db.execute(
"UPDATE job_queue SET status=?, started_at=datetime('now') WHERE domain=?",
@@ -315,7 +315,7 @@ async def worker_loop():
if _paused:
await asyncio.sleep(1)
continue
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
async with db.execute(
"SELECT domain FROM job_queue WHERE status='pending' LIMIT 100"
) as cur:
@@ -348,7 +348,7 @@ async def _assess_one(domain: str, language: str = "ES") -> None:
except Exception as e:
logger.error("AI: failed %s%s", domain, e, exc_info=True)
try:
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
await db.execute(
"UPDATE ai_queue SET status='failed', completed_at=datetime('now'), error=? WHERE domain=?",
(str(e)[:400], domain),
@@ -362,7 +362,7 @@ async def ai_worker_loop():
logger.info("AI worker loop starting")
# Reset any jobs left in 'running' state from a previous crashed worker
try:
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
result = await db.execute(
"UPDATE ai_queue SET status='pending' WHERE status='running'"
)
@@ -375,7 +375,7 @@ async def ai_worker_loop():
while True:
rows = []
try:
async with aiosqlite.connect(SQLITE_PATH) as db:
async with aiosqlite.connect(SQLITE_PATH, timeout=30) as db:
async with db.execute(
"SELECT domain, COALESCE(language,'ES') FROM ai_queue WHERE status='pending' LIMIT 5"
) as cur: