fix(api): default event loop to asyncio + cancellation-safe commit — kills the CI segfault (#340)

* fix(api): default the event loop to asyncio + cancellation-safe commit

The recurring CI e2e segfault traced to uvloop: the harness's
uvicorn.run() auto-selected it while production's serve() path never
consulted Config.loop (stock asyncio, accidentally safe). Every launch
site now resolves ROBOCO_UVICORN_LOOP (default asyncio; uvloop opt-in),
and DbCommitMiddleware's commit-in-send can no longer be interrupted
mid-wire: on cancellation it gets a bounded grace to finish (committed
data survives the 504), else invalidate-and-reraise.

* feat(runtime): expected-stop breadcrumbs attribute container deaths

Two production exit-143s had no attributable source: every orchestrator
kill path now records a short reason breadcrumb, and the exit monitor
consumes it -- an expected stop logs its reason at info, a genuinely
unexpected one logs none_recorded plus docker-inspect diagnostics
(OOMKilled, timestamps) so the next mystery SIGTERM self-identifies.

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>

tests/unit/runtime/test_expected_stop_breadcrumb.py

@@ -0,0 +1,199 @@
+"""Expected-stop breadcrumb registry.
+
+Production containers can exit 143 (SIGTERM) with the orchestrator's exit
+monitor logging only "Agent container stopped unexpectedly" — no line
+identifies who stopped it, and containers are gone by the time anyone looks
+(docker events empty). Every orchestrator-initiated stop/kill path now
+breadcrumbs the agent_id (_record_expected_stop) before it acts; the monitor
+consumes it (_consume_expected_stop) when the container turns up dead and
+downgrades an attributed death to an info "(expected)" line, keeping the
+warning meaningful for genuinely unexplained SIGTERMs/crashes.
+"""
+
+from __future__ import annotations
+
+import time
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from roboco.runtime.orchestrator import AgentOrchestrator, AgentState
+from structlog.testing import capture_logs
+
+
+def _make_orchestrator() -> AgentOrchestrator:
+    with patch.object(AgentOrchestrator, "__init__", return_value=None):
+        orch = AgentOrchestrator.__new__(AgentOrchestrator)
+    orch._instances = {}
+    orch._lock = MagicMock()
+    return orch
+
+
+def _instance() -> MagicMock:
+    inst = MagicMock()
+    inst.state = AgentState.ACTIVE
+    inst.container_id = "deadbeef1234"
+    inst.current_task_id = None
+    inst.error_count = 0
+    inst.config = MagicMock(git_context=None)
+    return inst
+
+
+# ---------------------------------------------------------------------------
+# _record_expected_stop / _consume_expected_stop
+# ---------------------------------------------------------------------------
+
+
+def test_record_then_consume_returns_the_reason() -> None:
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    assert orch._consume_expected_stop("be-dev-1") == "budget_sweep"
+
+
+def test_consume_pops_the_entry() -> None:
+    """A second consume for the same agent finds nothing — one-shot breadcrumb."""
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    orch._consume_expected_stop("be-dev-1")
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_no_breadcrumb_is_none_recorded() -> None:
+    orch = _make_orchestrator()
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_stale_breadcrumb_is_ignored() -> None:
+    """A breadcrumb older than the freshness window can't attribute a later,
+    unrelated exit — treated the same as never having been recorded."""
+    orch = _make_orchestrator()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    reason, _ts = orch._expected_stops["be-dev-1"]
+    orch._expected_stops["be-dev-1"] = (reason, time.monotonic() - 121.0)
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+
+
+def test_registry_defensive_on_bare_new_instance() -> None:
+    """A __new__-constructed instance (many existing test fixtures across the
+    suite bypass __init__ this way) has no _expected_stops attribute until
+    first use — both helpers must self-heal it rather than raise
+    AttributeError."""
+    orch = AgentOrchestrator.__new__(AgentOrchestrator)
+    assert orch._consume_expected_stop("be-dev-1") == "none_recorded"
+    orch._record_expected_stop("be-dev-1", "stop_agent_api")
+    assert orch._consume_expected_stop("be-dev-1") == "stop_agent_api"
+
+
+# ---------------------------------------------------------------------------
+# _check_health / _handle_stopped_container attribution
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_kill_path_breadcrumb_downgrades_the_monitor_log() -> None:
+    """A kill path (e.g. the budget sweep) records a breadcrumb; when the
+    monitor later observes the same container gone, it logs "(expected)" at
+    info with the recorded reason instead of "unexpectedly" at warning."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+    # The exact call a kill path makes (_sweep_budget_exceeded -> stop_agent)
+    # before it issues its own docker stop/kill.
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    expected = [e for e in logs if e["event"] == "Agent container stopped (expected)"]
+    assert expected, logs
+    assert expected[0]["log_level"] == "info"
+    assert expected[0]["expected_stop_reason"] == "budget_sweep"
+    assert not [e for e in logs if e["event"] == "Agent container stopped unexpectedly"]
+
+
+@pytest.mark.asyncio
+async def test_no_breadcrumb_stays_a_warning() -> None:
+    """No breadcrumb recorded: the death is genuinely unattributed, so the
+    line stays a warning carrying expected_stop_reason="none_recorded"."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert unexpected[0]["log_level"] == "warning"
+    assert unexpected[0]["expected_stop_reason"] == "none_recorded"
+
+
+@pytest.mark.asyncio
+async def test_stale_breadcrumb_does_not_suppress_the_warning() -> None:
+    """A breadcrumb from a much older stop must not attribute an unrelated,
+    later exit — the warning line still fires with none_recorded."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+    orch._record_expected_stop("be-dev-1", "budget_sweep")
+    orch._expected_stops["be-dev-1"] = ("budget_sweep", time.monotonic() - 121.0)
+
+    proc = MagicMock()
+    proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert unexpected[0]["expected_stop_reason"] == "none_recorded"
+
+
+@pytest.mark.asyncio
+async def test_inspect_diagnostics_failure_is_tolerated() -> None:
+    """A failed/timed-out extra `docker inspect` for OOMKilled/StartedAt/etc
+    must not break the monitor — the log line still emits, just without
+    those fields (best-effort, never blocks the log)."""
+    orch = _make_orchestrator()
+    orch._instances["be-dev-1"] = _instance()
+
+    async def _create_subprocess_exec(*args: object, **_kw: object) -> MagicMock:
+        if any(isinstance(a, str) and "OOMKilled" in a for a in args):
+            raise RuntimeError("docker daemon unreachable")
+        proc = MagicMock()
+        proc.communicate = AsyncMock(return_value=(b"false 137\n", b""))
+        return proc
+
+    with (
+        patch.object(orch, "spawn_agent", new=AsyncMock()),
+        patch(
+            "asyncio.create_subprocess_exec",
+            AsyncMock(side_effect=_create_subprocess_exec),
+        ),
+        capture_logs() as logs,
+    ):
+        await orch._check_health()
+
+    unexpected = [
+        e for e in logs if e["event"] == "Agent container stopped unexpectedly"
+    ]
+    assert unexpected, logs
+    assert "oom_killed" not in unexpected[0]

tests/unit/runtime/test_gateway_health.py

@@ -121,7 +121,9 @@ async def test_broken_past_grace_is_killed(monkeypatch: pytest.MonkeyPatch) -> N
     monkeypatch.setattr(orch, "_remove_container", remove)
     monkeypatch.setattr(orch, "_probe_gateway_health", AsyncMock(return_value=False))
     assert await orch._maybe_recover_broken_gateway(_task()) is True
-    remove.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="gateway_health_recovery"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
 
 

tests/unit/runtime/test_grok_cost_budget.py

@@ -52,7 +52,9 @@ async def test_cost_over_cap_kills_and_evicts(monkeypatch: pytest.MonkeyPatch) -
 
     await orch._enforce_grok_cost_budget()
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="grok_cost_cap"
+    )
     assert "be-dev-1" not in orch._instances
 
 
@@ -140,7 +142,9 @@ async def test_interactive_kill_closes_the_relay(
 
     await orch._enforce_grok_cost_budget()
 
-    remove_mock.assert_awaited_once_with(f"roboco-agent-{INTAKE_AGENT_ID}")
+    remove_mock.assert_awaited_once_with(
+        f"roboco-agent-{INTAKE_AGENT_ID}", stop_reason="grok_cost_cap"
+    )
     assert INTAKE_AGENT_ID not in orch._instances
     assert len(registry.calls) == 1
     assert registry.calls[0][0] == INTAKE_AGENT_ID

tests/unit/runtime/test_intake_spawn.py

@@ -1040,7 +1040,7 @@ class TestSpawnIntakeShutdownNoOrphan:
         _wire_spawn_mocks(monkeypatch, orch, run_calls)
         removed: list[str] = []
 
-        async def _remove(name: str) -> None:
+        async def _remove(name: str, **_kw: Any) -> None:
             removed.append(name)
 
         async def _run(cmd: list[str]) -> str:
@@ -1097,7 +1097,7 @@ class TestSpawnIntakeShutdownNoOrphan:
         # _wire_spawn_mocks' _remove_container is a no-op; override to record.
         removed: list[str] = []
 
-        async def _remove(name: str) -> None:
+        async def _remove(name: str, **_kw: Any) -> None:
             removed.append(name)
 
         monkeypatch.setattr(orch, "_remove_container", _remove)

tests/unit/runtime/test_sandbox_env.py

@@ -188,4 +188,8 @@ async def test_spawn_container_stale_clear_spares_fresh_sandbox(
     )
     await orch._spawn_container(_config(info))
 
-    remove.assert_awaited_once_with("roboco-agent-dev-1", teardown_sandbox=False)
+    remove.assert_awaited_once_with(
+        "roboco-agent-dev-1",
+        teardown_sandbox=False,
+        stop_reason="pre_spawn_stale_clear",
+    )

tests/unit/runtime/test_secretary_spawn_shutdown.py

@@ -64,7 +64,7 @@ def _wire_secretary_spawn_mocks(
             orch._running = False
         return "containerid0123456789"
 
-    async def _remove(name: str) -> None:
+    async def _remove(name: str, **_kw: Any) -> None:
         removed.append(name)
 
     monkeypatch.setattr(

tests/unit/runtime/test_stale_claim_reaper.py

@@ -206,7 +206,9 @@ async def test_reaper_kills_and_releases_wedged_grok_container(
 
     await orch._reap_with_service(svc)
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="reaper_wedged_grok"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
     svc.unclaim_for_reaper.assert_awaited_once_with(task_id)  # released
 
@@ -338,7 +340,9 @@ async def test_reaper_kills_stuck_claude_past_stuck_ttl(
 
     await orch._reap_with_service(svc)
 
-    remove_mock.assert_awaited_once_with("roboco-agent-be-dev-1")
+    remove_mock.assert_awaited_once_with(
+        "roboco-agent-be-dev-1", stop_reason="reaper_stuck_claude"
+    )
     assert "be-dev-1" not in orch._instances  # evicted
     svc.unclaim_for_reaper.assert_awaited_once_with(stuck.id)  # released