mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
d7a1c2d203
* fix(db): bound lock waits and idle transactions so a parked coroutine can't wedge the pool 2026-07-29 production incident: verb/evidence handlers and background dispatch coroutines held an open DB transaction across minutes of git subprocess work and asyncio lock queues (per-workspace ensure locks). Early writes in those transactions held tasks/agents row locks, every other write convoyed behind them, and blocked statements camped on pool connections until all 30 were waiters — 1000+ QueuePool timeouts per hour, one transaction open 1h20m. Two layers: - get_engine now passes asyncpg server_settings: idle_in_transaction_session_timeout (default 120s) kills any session parked mid-transaction on non-DB work, releasing its locks and pool slot; lock_timeout (default 30s) makes a statement queued on someone else's row lock give up instead of holding a connection for the wait. Both env-tunable (ROBOCO_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS / ROBOCO_DATABASE_LOCK_TIMEOUT_MS), 0 disables. Alembic runs its own sync engine and is untouched; best-effort writers (proactive-context injection) already swallow errors and now fail in 30s instead of camping for an hour. - ContentActions.evidence commits the request session before its fetch/diff git work, so a multi-minute evidence call no longer pins a pool connection for the duration (expire_on_commit=False keeps the loaded task usable; later reads reopen a transaction on demand). The deeper restructuring — claim flows committing their transition before briefing/workspace assembly — is scoped to the existing evidence-assembly-timeout task and not attempted here. * fix(db): bound lock waits and idle transactions so a parked coroutine can't wedge the pool 2026-07-29 production incident: verb/evidence handlers and background dispatch coroutines held an open DB transaction across minutes of git subprocess work and asyncio lock queues (per-workspace ensure locks). Early writes in those transactions held tasks/agents row locks, every other write convoyed behind them, and blocked statements camped on pool connections until all 30 were waiters — 1000+ QueuePool timeouts per hour, one transaction open 1h20m. Two layers: - get_engine now passes asyncpg server_settings: idle_in_transaction_session_timeout (default 20 min) kills any session parked mid-transaction on non-DB work, releasing its locks and pool slot; lock_timeout (default 60s) makes a statement queued on someone else's row lock give up with a clean retryable error instead of camping on a pool connection for the wait. Both env-tunable (ROBOCO_DATABASE_IDLE_IN_TRANSACTION_TIMEOUT_MS / ROBOCO_DATABASE_LOCK_TIMEOUT_MS), 0 disables. The idle default deliberately clears the longest LEGITIMATE in-transaction window — a cold-workspace claim holds its transaction across the clone (300s budget) + dep install (600s budget) under the 900s slow-verb wall — so routine claims never trip it while today's 80-minute parked transaction dies at 20 min. Alembic's env.py builds its own engine and never carries these; best-effort writers (proactive-context injection) already swallow errors and now fail in 60s instead of camping for an hour. - ContentActions.evidence ends the request transaction (commit, or rollback on a poisoned session) before its fetch/diff git work, so a multi-minute evidence call no longer pins a pool connection for the duration (expire_on_commit=False keeps the loaded task usable; later reads reopen a transaction on demand). The deeper restructuring — claim flows committing their transition before briefing/workspace assembly — is scoped to the existing evidence-assembly-timeout task and not attempted here. --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
"""Engine-level Postgres session timeouts (2026-07-29 pool-exhaustion class).
|
|
|
|
A session parked mid-transaction on non-DB work (a git subprocess, an
|
|
asyncio lock queue) holds its row locks and pooled connection until Postgres
|
|
kills it; a statement queued on someone else's row lock must give up instead
|
|
of camping on a pool slot. ``get_engine`` passes both as asyncpg
|
|
``server_settings`` so every app connection carries them; a 0 value drops
|
|
the setting entirely (Postgres-default behavior, operator off-switch).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
from roboco.db import base as db_base
|
|
|
|
|
|
@pytest.fixture
|
|
def _holder_reset(monkeypatch: pytest.MonkeyPatch) -> Any:
|
|
"""Isolate _DbHolder so the test never touches the real engine."""
|
|
monkeypatch.setattr(db_base._DbHolder, "engine", None)
|
|
monkeypatch.setattr(db_base._DbHolder, "session_factory", None)
|
|
monkeypatch.setattr(db_base._DbHolder, "loop", None)
|
|
return db_base._DbHolder
|
|
|
|
|
|
def _capture_engine_kwargs(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
captured: dict[str, Any] = {}
|
|
|
|
def _fake_create(_url: str, **kwargs: Any) -> MagicMock:
|
|
captured.update(kwargs)
|
|
return MagicMock()
|
|
|
|
monkeypatch.setattr(db_base, "create_async_engine", _fake_create)
|
|
return captured
|
|
|
|
|
|
def test_engine_carries_server_side_timeouts(
|
|
monkeypatch: pytest.MonkeyPatch, _holder_reset: Any
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
db_base.settings, "database_idle_in_transaction_timeout_ms", 120_000
|
|
)
|
|
monkeypatch.setattr(db_base.settings, "database_lock_timeout_ms", 30_000)
|
|
captured = _capture_engine_kwargs(monkeypatch)
|
|
|
|
db_base.get_engine()
|
|
|
|
assert captured["connect_args"]["server_settings"] == {
|
|
"idle_in_transaction_session_timeout": "120000",
|
|
"lock_timeout": "30000",
|
|
}
|
|
|
|
|
|
def test_zero_disables_each_timeout_individually(
|
|
monkeypatch: pytest.MonkeyPatch, _holder_reset: Any
|
|
) -> None:
|
|
monkeypatch.setattr(db_base.settings, "database_idle_in_transaction_timeout_ms", 0)
|
|
monkeypatch.setattr(db_base.settings, "database_lock_timeout_ms", 5_000)
|
|
captured = _capture_engine_kwargs(monkeypatch)
|
|
|
|
db_base.get_engine()
|
|
|
|
assert captured["connect_args"]["server_settings"] == {"lock_timeout": "5000"}
|
|
|
|
|
|
def test_both_zero_sends_no_server_settings(
|
|
monkeypatch: pytest.MonkeyPatch, _holder_reset: Any
|
|
) -> None:
|
|
monkeypatch.setattr(db_base.settings, "database_idle_in_transaction_timeout_ms", 0)
|
|
monkeypatch.setattr(db_base.settings, "database_lock_timeout_ms", 0)
|
|
captured = _capture_engine_kwargs(monkeypatch)
|
|
|
|
db_base.get_engine()
|
|
|
|
assert captured["connect_args"] == {}
|