fix(api): default event loop to asyncio + cancellation-safe commit — kills the CI segfault (#340)

* fix(api): default the event loop to asyncio + cancellation-safe commit

The recurring CI e2e segfault traced to uvloop: the harness's
uvicorn.run() auto-selected it while production's serve() path never
consulted Config.loop (stock asyncio, accidentally safe). Every launch
site now resolves ROBOCO_UVICORN_LOOP (default asyncio; uvloop opt-in),
and DbCommitMiddleware's commit-in-send can no longer be interrupted
mid-wire: on cancellation it gets a bounded grace to finish (committed
data survives the 504), else invalidate-and-reraise.

* feat(runtime): expected-stop breadcrumbs attribute container deaths

Two production exit-143s had no attributable source: every orchestrator
kill path now records a short reason breadcrumb, and the exit monitor
consumes it -- an expected stop logs its reason at info, a genuinely
unexpected one logs none_recorded plus docker-inspect diagnostics
(OOMKilled, timestamps) so the next mystery SIGTERM self-identifies.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>

tests/e2e_smoke/harness.py

@@ -369,8 +369,18 @@ def build_e2e_stack(
     mp.setattr(settings, "github_api_base_url", f"{base_url}/_github")
 
     app = _build_app(gh)
+    # loop=settings.uvicorn_loop ("asyncio" by default): uvicorn auto-selects
+    # uvloop when installed, and this in-thread server has crashed CI with a
+    # uvloop/asyncpg segfault (uvloop 0.22 + asyncpg 0.31 + Python 3.13) —
+    # mirror the production default instead of picking up uvloop implicitly.
     server = uvicorn.Server(
-        uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning")
+        uvicorn.Config(
+            app,
+            host="127.0.0.1",
+            port=port,
+            log_level="warning",
+            loop=settings.uvicorn_loop,
+        )
     )
     thread = threading.Thread(target=server.run, daemon=True)
     thread.start()

tests/unit/api/test_middleware.py

@@ -662,7 +662,46 @@ class _CancelableCommitSession:
         self._txn = False
 
     async def invalidate(self) -> None:
-        self._order.append("invalidate")
+        # Mirrors real SQLAlchemy Session.invalidate(): a no-op past the
+        # first call (no open transaction left to touch) — both
+        # _commit_shielded and get_db's own cancel handler call this for the
+        # same cancelled request, and only the first should count.
+        if self._txn:
+            self._order.append("invalidate")
+        self._txn = False
+
+
+class _SlowButFinishingCommitSession:
+    """get_db-style fake session whose ``commit()`` outlives the server-side
+    timeout but finishes well within the shield's grace period — proving a
+    cancelled-mid-commit request still lets the commit land instead of
+    severing it on the spot."""
+
+    def __init__(self, order: list[str], delay: float) -> None:
+        self._order = order
+        self._delay = delay
+        self._txn = True
+
+    def in_transaction(self) -> bool:
+        return self._txn
+
+    async def commit(self) -> None:
+        self._order.append("commit_start")
+        await asyncio.sleep(self._delay)
+        self._order.append("commit_end")
+        self._txn = False
+
+    async def rollback(self) -> None:
+        self._order.append("rollback")
+        self._txn = False
+
+    async def invalidate(self) -> None:
+        # Mirrors real SQLAlchemy Session.invalidate(): a no-op once the
+        # transaction is already gone (a successful commit clears it) — the
+        # cancel-completes-successfully test relies on this, exactly like the
+        # double-invalidate case above.
+        if self._txn:
+            self._order.append("invalidate")
         self._txn = False
 
 
@@ -701,8 +740,9 @@ def _make_cancel_during_commit_app(order: list[str]) -> FastAPI:
 
 def test_cancellation_mid_commit_invalidates_not_rollback(monkeypatch: Any) -> None:
     """A flow-verb request that blows its server-side timeout WHILE
-    DbCommitMiddleware's commit is in flight must: propagate CancelledError
-    cleanly to a 504 (not hang, not a raw 500), discard the session via
+    DbCommitMiddleware's commit is in flight, and the commit is STILL stuck
+    past the shield's grace period, must: propagate CancelledError cleanly
+    to a 504 (not hang, not a raw 500), discard the session via
     ``invalidate()`` — NOT ``rollback()`` (SQLAlchemy's own docs: rolling
     back a cancelled/timed-out operation risks issuing another command over
     a connection whose wire-protocol state is now undefined, which is what
@@ -710,6 +750,7 @@ def test_cancellation_mid_commit_invalidates_not_rollback(monkeypatch: Any) -> N
     and never resume/complete the cancelled commit.
     """
     monkeypatch.setattr(settings, "flow_verb_timeout_seconds", 0.05)
+    monkeypatch.setattr(settings, "db_commit_cancel_grace_seconds", 0.05)
     order: list[str] = []
     app = _make_cancel_during_commit_app(order)
 
@@ -719,3 +760,58 @@ def test_cancellation_mid_commit_invalidates_not_rollback(monkeypatch: Any) -> N
     assert response.status_code == HTTPStatus.GATEWAY_TIMEOUT
     assert response.json()["error"] == "gateway_timeout"
     assert order == ["route_body", "commit_start", "invalidate"], order
+
+
+async def _fake_get_db_slow_commit(request: Request) -> Any:
+    """Module-level for the same reason as ``_fake_get_db_cancel_safe`` above —
+    a local closure isn't resolvable as a FastAPI dependency under this file's
+    ``from __future__ import annotations``."""
+    order: list[str] = request.app.state.db_commit_order
+    delay: float = request.app.state.db_commit_delay
+    session = _SlowButFinishingCommitSession(order, delay)
+    request.state.db_session = session
+    try:
+        yield session
+        await session.commit()
+    except asyncio.CancelledError:
+        await _discard_on_cancel(cast("AsyncSession", session))
+        raise
+    except Exception:
+        await session.rollback()
+        raise
+
+
+def _make_slow_commit_app(order: list[str], delay: float) -> FastAPI:
+    app = FastAPI()
+    app.state.db_commit_order = order
+    app.state.db_commit_delay = delay
+
+    @app.post("/api/v1/flow/developer/give_me_work")
+    async def _write(_db: Annotated[Any, Depends(_fake_get_db_slow_commit)]) -> Any:
+        order.append("route_body")
+        return {"status": "ok"}
+
+    setup_middleware(app)
+    return app
+
+
+def test_cancellation_mid_commit_lets_commit_finish_within_grace(
+    monkeypatch: Any,
+) -> None:
+    """A commit already in flight when the server-side timeout fires, but
+    that finishes on its own well within the shield's grace period, must be
+    allowed to land — never severed on the spot. No rollback, no invalidate
+    (nothing to undo — the data is durably committed), and the timeout's own
+    CancelledError still propagates to the client as a clean 504 (the
+    client's retry is idempotent-safe regardless)."""
+    monkeypatch.setattr(settings, "flow_verb_timeout_seconds", 0.05)
+    monkeypatch.setattr(settings, "db_commit_cancel_grace_seconds", 2.0)
+    order: list[str] = []
+    app = _make_slow_commit_app(order, delay=0.15)
+
+    client = TestClient(app)
+    response = client.post("/api/v1/flow/developer/give_me_work")
+
+    assert response.status_code == HTTPStatus.GATEWAY_TIMEOUT
+    assert response.json()["error"] == "gateway_timeout"
+    assert order == ["route_body", "commit_start", "commit_end"], order

tests/unit/llm/test_providers.py

@@ -62,6 +62,7 @@ class _FakeHost:
 
     def __init__(self) -> None:
         self.removed: list[str] = []
+        self.remove_stop_reasons: list[str | None] = []
         self.spawn_args: tuple[object, ...] | None = None
         self.mount_config: OrchestratorAgentConfig | None = None
         self.data_dirs_ensured: list[str] = []
@@ -75,8 +76,11 @@ class _FakeHost:
         self.spawn_args = (config, initial_prompt, agent_settings_path)
         return "container-id-abc123"
 
-    async def _remove_container(self, container_name: str) -> None:
+    async def _remove_container(
+        self, container_name: str, *, stop_reason: str | None = None
+    ) -> None:
         self.removed.append(container_name)
+        self.remove_stop_reasons.append(stop_reason)
 
     def _ensure_grok_usage_dir(self, agent_id: str) -> None:
         self.data_dirs_ensured.append(agent_id)
@@ -227,6 +231,7 @@ async def test_grok_spawn_wires_gateway_env_and_image_last() -> None:
     # The image is the final docker-run argument.
     assert cmd[-1] == "roboco-agent-grok:test"
     assert host.removed == ["roboco-agent-be-dev-1"]
+    assert host.remove_stop_reasons == ["pre_spawn_stale_clear"]
     assert result == SpawnResult(
         instance_id="roboco-agent-be-dev-1",
         extra={"container_id": "cid", "model": "grok-build"},

tests/unit/runtime/test_expected_stop_breadcrumb.py

@@ -0,0 +1,199 @@
+"""Expected-stop breadcrumb registry.
+
+Production containers can exit 143 (SIGTERM) with the orchestrator's exit
+monitor logging only "Agent container stopped unexpectedly" — no line
+identifies who stopped it, and containers are gone by the time anyone looks
+(docker events empty). Every orchestrator-initiated stop/kill path now
+breadcrumbs the agent_id (_record_expected_stop) before it acts; the monitor
+consumes it (_consume_expected_stop) when the container turns up dead and
+downgrades an attributed death to an info "(expected)" line, keeping the
+warning meaningful for genuinely unexplained SIGTERMs/crashes.
+"""
+
+from __future__ import annotations
+
+import time
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from roboco.runtime.orchestrator import AgentOrchestrator, AgentState
+from structlog.testing import capture_logs
+
+
+def _make_orchestrator() -> AgentOrchestrator:
+    with patch.object(AgentOrchestrator, "__init__", return_value=None):
+        orch = AgentOrchestrator.__new__(AgentOrchestrator)
+    orch._instances = {}
+    orch._lock = MagicMock()
+    return orch
+
+
+def _instance() -> MagicMock:
+    inst = MagicMock()
+    inst.state = AgentState.ACTIVE
+    inst.container_id = "deadbeef1234"
+    inst.current_task_id = None
+    inst.error_count = 0
+    inst.config = MagicMock(git_context=None)
+    return inst
+
+
+# ---------------------------------------------------------------------------
+# _record_expected_stop / _consume_expected_stop
+# ---------------------------------------------------------------------------
+
+
+def test_record_then_consume_returns_the_reason() -> None:
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    assert orch._consume_expected_stop("be-dev-1") == "budget_sweep"
+
+
+def test_consume_pops_the_entry() -> None:
+    """A second consume for the same agent finds nothing — one-shot breadcrumb."""
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    orch._consume_expected_stop("be-dev-1")
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_no_breadcrumb_is_none_recorded() -> None:
+    orch = _make_orchestrator()
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_stale_breadcrumb_is_ignored() -> None:
+    """A breadcrumb older than the freshness window can't attribute a later,
+    unrelated exit — treated the same as never having been recorded."""
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    reason, _ts = orch._expected_stops["be-dev-1"]
+    orch._expected_stops["be-dev-1"] = (reason, time.monotonic() - 121.0)
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_registry_defensive_on_bare_new_instance() -> None:
+    """A __new__-constructed instance (many existing test fixtures across the
+    suite bypass __init__ this way) has no _expected_stops attribute until
+    first use — both helpers must self-heal it rather than raise
+    AttributeError."""
+    orch = AgentOrchestrator.__new__(AgentOrchestrator)
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+    orch._record_expected_stop("be-dev-1", "stop_agent_api")
+    assert orch._consume_expected_stop("be-dev-1") == "stop_agent_api"
+
+
+# ---------------------------------------------------------------------------
+# _check_health / _handle_stopped_container attribution
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_kill_path_breadcrumb_downgrades_the_monitor_log() -> None:
+    """A kill path (e.g. the budget sweep) records a breadcrumb; when the
+    monitor later observes the same container gone, it logs "(expected)" at
+    info with the recorded reason instead of "unexpectedly" at warning."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+    # The exact call a kill path makes (_sweep_budget_exceeded -> stop_agent)
+    # before it issues its own docker stop/kill.
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    expected = [e for e in logs if e["event"] == "Agent container stopped (expected)"]
+    assert expected, logs
+    assert expected[0]["log_level"] == "info"
+    assert expected[0]["expected_stop_reason"] == "budget_sweep"
+    assert not [e for e in logs if e["event"] == "Agent container stopped unexpectedly"]
+
+
+@pytest.mark.asyncio
+async def test_no_breadcrumb_stays_a_warning() -> None:
+    """No breadcrumb recorded: the death is genuinely unattributed, so the
+    line stays a warning carrying expected_stop_reason="none_recorded"."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert unexpected[0]["log_level"] == "warning"
+    assert unexpected[0]["expected_stop_reason"] == "none_recorded"
+
+
+@pytest.mark.asyncio
+async def test_stale_breadcrumb_does_not_suppress_the_warning() -> None:
+    """A breadcrumb from a much older stop must not attribute an unrelated,
+    later exit — the warning line still fires with none_recorded."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    orch._expected_stops["be-dev-1"] = ("budget_sweep", time.monotonic() - 121.0)
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert unexpected[0]["expected_stop_reason"] == "none_recorded"
+
+
+@pytest.mark.asyncio
+async def test_inspect_diagnostics_failure_is_tolerated() -> None:
+    """A failed/timed-out extra `docker inspect` for OOMKilled/StartedAt/etc
+    must not break the monitor — the log line still emits, just without
+    those fields (best-effort, never blocks the log)."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+
+    async def _create_subprocess_exec(*args: object, **_kw: object) -> MagicMock:
+        if any(isinstance(a, str) and "OOMKilled" in a for a in args):
+            raise RuntimeError("docker daemon unreachable")
+        proc = MagicMock()
+        proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+        return proc
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch(
+            "asyncio.create_subprocess_exec",
+            AsyncMock(side_effect=_create_subprocess_exec),
+        ),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert "oom_killed" not in unexpected[0]

tests/unit/runtime/test_gateway_health.py

@@ -121,7 +121,9 @@ async def test_broken_past_grace_is_killed(monkeypatch: pytest.MonkeyPatch) -> N
     monkeypatch.setattr(orch, "_remove_container", remove)
     monkeypatch.setattr(orch, "_probe_gateway_health", AsyncMock(return_value=False))
     assert await orch._maybe_recover_broken_gateway(_task()) is True
-    remove.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="gateway_health_recovery"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
 
 

tests/unit/runtime/test_grok_cost_budget.py

@@ -52,7 +52,9 @@ async def test_cost_over_cap_kills_and_evicts(monkeypatch: pytest.MonkeyPatch) -
 
     await orch._enforce_grok_cost_budget()
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="grok_cost_cap"
+    )
     assert "be-dev-1" not in orch._instances
 
 
@@ -140,7 +142,9 @@ async def test_interactive_kill_closes_the_relay(
 
     await orch._enforce_grok_cost_budget()
 
-    remove_mock.assert_awaited_once_with(f"roboco-agent-{INTAKE_AGENT_ID}")
+    remove_mock.assert_awaited_once_with(
+        f"roboco-agent-{INTAKE_AGENT_ID}", stop_reason="grok_cost_cap"
+    )
     assert INTAKE_AGENT_ID not in orch._instances
     assert len(registry.calls) == 1
     assert registry.calls[0][0] == INTAKE_AGENT_ID

tests/unit/runtime/test_intake_spawn.py

@@ -1040,7 +1040,7 @@ class TestSpawnIntakeShutdownNoOrphan:
         _wire_spawn_mocks(monkeypatch, orch, run_calls)
         removed: list[str] = []
 
-        async def _remove(name: str) -> None:
+        async def _remove(name: str, **_kw: Any) -> None:
             removed.append(name)
 
         async def _run(cmd: list[str]) -> str:
@@ -1097,7 +1097,7 @@ class TestSpawnIntakeShutdownNoOrphan:
         # _wire_spawn_mocks' _remove_container is a no-op; override to record.
         removed: list[str] = []
 
-        async def _remove(name: str) -> None:
+        async def _remove(name: str, **_kw: Any) -> None:
             removed.append(name)
 
         monkeypatch.setattr(orch, "_remove_container", _remove)

tests/unit/runtime/test_sandbox_env.py

@@ -188,4 +188,8 @@ async def test_spawn_container_stale_clear_spares_fresh_sandbox(
     )
     await orch._spawn_container(_config(info))
 
-    remove.assert_awaited_once_with("roboco-agent-dev-1", teardown_sandbox=False)
+    remove.assert_awaited_once_with(
+        "roboco-agent-dev-1",
+        teardown_sandbox=False,
+        stop_reason="pre_spawn_stale_clear",
+    )

tests/unit/runtime/test_secretary_spawn_shutdown.py

@@ -64,7 +64,7 @@ def _wire_secretary_spawn_mocks(
             orch._running = False
         return "containerid0123456789"
 
-    async def _remove(name: str) -> None:
+    async def _remove(name: str, **_kw: Any) -> None:
         removed.append(name)
 
     monkeypatch.setattr(

tests/unit/runtime/test_stale_claim_reaper.py

@@ -206,7 +206,9 @@ async def test_reaper_kills_and_releases_wedged_grok_container(
 
     await orch._reap_with_service(svc)
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="reaper_wedged_grok"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
     svc.unclaim_for_reaper.assert_awaited_once_with(task_id)  # released
 
@@ -338,7 +340,9 @@ async def test_reaper_kills_stuck_claude_past_stuck_ttl(
 
     await orch._reap_with_service(svc)
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="reaper_stuck_claude"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
     svc.unclaim_for_reaper.assert_awaited_once_with(stuck.id)  # released
 

tests/unit/test_config_properties.py

@@ -2,9 +2,11 @@
 
 from __future__ import annotations
 
+import importlib
+
 import pytest
 from pydantic import ValidationError
-from roboco.config import Settings
+from roboco.config import Settings, resolve_uvicorn_loop_factory
 
 
 def test_internal_api_url_uses_api_url_when_set() -> None:
@@ -126,3 +128,42 @@ def test_local_llm_base_url_public_rejected() -> None:
 def test_local_llm_base_url_missing_host_rejected() -> None:
     with pytest.raises(ValidationError):
         Settings(local_llm_base_url="http://")
+
+
+# ---------------------------------------------------------------------------
+# uvicorn_loop — default asyncio, uvloop opt-in (CI segfault fix)
+# ---------------------------------------------------------------------------
+
+
+def test_uvicorn_loop_defaults_to_asyncio() -> None:
+    assert Settings().uvicorn_loop == "asyncio"
+
+
+def test_uvicorn_loop_honors_constructor_override() -> None:
+    assert Settings(uvicorn_loop="uvloop").uvicorn_loop == "uvloop"
+
+
+def test_uvicorn_loop_honors_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("ROBOCO_UVICORN_LOOP", "uvloop")
+    assert Settings().uvicorn_loop == "uvloop"
+
+
+def test_uvicorn_loop_rejects_unknown_value() -> None:
+    with pytest.raises(ValidationError):
+        Settings(uvicorn_loop="unknown")  # type: ignore[arg-type]
+
+
+def test_resolve_uvicorn_loop_factory_asyncio_is_none() -> None:
+    """The default: no override, so asyncio.run() picks its own stock loop."""
+    assert resolve_uvicorn_loop_factory("asyncio") is None
+
+
+def test_resolve_uvicorn_loop_factory_uvloop_returns_new_event_loop() -> None:
+    factory = resolve_uvicorn_loop_factory("uvloop")
+    assert factory is not None
+    loop = factory()
+    try:
+        uvloop = importlib.import_module("uvloop")
+        assert isinstance(loop, uvloop.Loop)
+    finally:
+        loop.close()