feat(sandbox): on-demand provisioning via request_sandbox verb (#338)

* feat(sandbox): on-demand request_sandbox verb replaces eager provisioning

Sandboxes were provisioned at every agent spawn for opted-in projects,
so every role paid the sidecar spin-up and a provisioning failure
refused the spawn. Provisioning now happens when an agent asks: the
request_sandbox do-verb (dev + QA) reaches the orchestrator through
ContentActionsDeps, ensure_sandbox provisions idempotently with an
in-memory per-agent cache (evicted at teardown and janitor sweep), and
creds return in the envelope payload including ready-to-export
ROBOCO_TEST_* values. Spawn now only injects a marker env naming the
available services plus a briefing line; sandbox failures can no longer
refuse a spawn. Teardown lifecycle unchanged.

* feat(sandbox): harden request_sandbox + Phase 3 wiring proof and docs

Hardening from adversarial review: ensure_sandbox now provisions the
project's full opted-in set on first request (a later superset can
never tear down a live sandbox mid-use), serializes per-agent behind an
asyncio lock (a client timeout-retry no longer races its own in-flight
provision), and verifies container liveness on every cache hit (a dead
sandbox evicts and re-provisions instead of serving dead creds). MCP
client budget 720->1080s for the full-set cold case. Phase 3: e2e smoke
wiring test (manifest grants + guard-chain envelopes over the real
API), sandbox-db/tools/map docs and CLAUDE.md rewritten for on-demand.

* feat(sandbox): release sandboxes when the agent's work ends

CEO directive: sidecars must not dangle once the agent is done. The six
work-ending verbs (i_am_done, unclaim, i_am_idle, pass_review,
fail_review, i_documented) now release the caller's sandbox best-effort
on their success path via release_sandbox (lock + teardown + cache
evict; a no-sandbox agent costs a dict lookup). Container removal and
the janitor remain the backstop; a re-request provisions fresh.

* test(sandbox): monkeypatch the release hook instead of method assignment

mypy method-assign rejected the direct AsyncMock assignments; the prior
static gate ran before this test file landed.

* test(sandbox): guard envelope evidence for mypy in verb tests

* chore(prompts): regenerate verb tables for request_sandbox

* chore: resolve merge with master (breadcrumbs + statement budget)

---------

Co-authored-by: Renn F <rennf93@users.noreply.github.com>

tests/unit/runtime/test_briefing_sandbox_line.py

@@ -0,0 +1,40 @@
+"""`_write_agent_briefing`'s sandbox availability line.
+
+Names the `request_sandbox` verb for an opted-in project (spec's "name it —
+cheap and kills a discovery failure mode" default) and is silent otherwise.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+from roboco.runtime.orchestrator import AgentOrchestrator
+
+
+def _orch() -> AgentOrchestrator:
+    with patch.object(AgentOrchestrator, "__init__", return_value=None):
+        orch = AgentOrchestrator.__new__(AgentOrchestrator)
+    object.__setattr__(orch, "_TOOL_LOAD_CACHE", {})
+    return orch
+
+
+@pytest.mark.asyncio
+async def test_briefing_names_request_sandbox_when_opted_in(tmp_path: object) -> None:
+    orch = _orch()
+    path = await orch._write_agent_briefing(
+        "dev-1", None, str(tmp_path), ["postgres", "redis"]
+    )
+    assert path is not None
+    content = path.read_text()
+    assert "request_sandbox()" in content
+    assert "postgres, redis" in content
+
+
+@pytest.mark.asyncio
+async def test_briefing_omits_sandbox_line_when_not_opted_in(tmp_path: object) -> None:
+    orch = _orch()
+    path = await orch._write_agent_briefing("dev-1", None, str(tmp_path), [])
+    assert path is not None
+    content = path.read_text()
+    assert "request_sandbox()" not in content

tests/unit/runtime/test_sandbox_env.py

@@ -1,8 +1,9 @@
-"""Sandbox env injection: `_append_sandbox_env` + the `_spawn_container` branch.
+"""Sandbox marker env: `_append_sandbox_marker_env` + the `_spawn_container` branch.
 
-A sandbox-active spawn must inject `ROBOCO_TEST_DB_*` / `ROBOCO_TEST_REDIS_*`
-pointed at the sandbox and MUST NOT also run the legacy `_append_gate_env`
-prod-creds injection — sandbox replaces, never coexists with, prod creds.
+An opted-in spawn injects a cheap `ROBOCO_SANDBOX_SERVICES_AVAILABLE` marker
+(never prod creds — actual provisioning is on-demand via `request_sandbox`)
+and MUST NOT also run the legacy `_append_gate_env` prod-creds injection —
+the marker replaces, never coexists with, prod creds.
 """
 
 from __future__ import annotations
@@ -13,93 +14,32 @@ from unittest.mock import AsyncMock
 
 import pytest
 from roboco.models.runtime import OrchestratorAgentConfig
-from roboco.models.sandbox import SandboxConnection, SandboxInfo
 from roboco.runtime.orchestrator import AgentOrchestrator
 
 
-def _config(sandbox_info: SandboxInfo | None = None) -> OrchestratorAgentConfig:
+def _config(
+    sandbox_available_services: list[str] | None = None,
+) -> OrchestratorAgentConfig:
     return OrchestratorAgentConfig(
         agent_id="dev-1",
         blueprint_path=Path(),
         mcp_config_path=Path("/tmp/mcp.json"),
-        sandbox_info=sandbox_info,
+        sandbox_available_services=sandbox_available_services or [],
     )
 
 
-def test_append_sandbox_env_injects_postgres_and_redis() -> None:
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="roboco-sandbox-pg-dev-1",
-                port=5432,
-                password="pgpw",
-                user="sandbox",
-                database="sandbox",
-            ),
-            "redis": SandboxConnection(
-                host="roboco-sandbox-redis-dev-1", port=6379, password="rdpw"
-            ),
-        }
-    )
+def test_append_sandbox_marker_env_lists_services() -> None:
     cmd: list[str] = []
-    AgentOrchestrator._append_sandbox_env(cmd, _config(info))
+    AgentOrchestrator._append_sandbox_marker_env(cmd, ["postgres", "redis"])
 
-    assert "ROBOCO_TEST_DB_HOST=roboco-sandbox-pg-dev-1" in cmd
-    assert "ROBOCO_TEST_DB_PORT=5432" in cmd
-    assert "ROBOCO_TEST_DB_USER=sandbox" in cmd
-    assert "ROBOCO_TEST_DB_PASSWORD=pgpw" in cmd
-    assert "ROBOCO_TEST_DB_ADMIN_DB=sandbox" in cmd
-    assert "ROBOCO_TEST_REDIS_HOST=roboco-sandbox-redis-dev-1" in cmd
-    assert "ROBOCO_TEST_REDIS_PORT=6379" in cmd
-    assert "ROBOCO_TEST_REDIS_PASSWORD=rdpw" in cmd
+    assert "ROBOCO_SANDBOX_SERVICES_AVAILABLE=postgres,redis" in cmd
 
 
-def test_append_sandbox_env_postgres_only_omits_redis_vars() -> None:
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="roboco-sandbox-pg-dev-1",
-                port=5432,
-                password="pgpw",
-                user="sandbox",
-                database="sandbox",
-            )
-        }
-    )
+def test_append_sandbox_marker_env_single_service() -> None:
     cmd: list[str] = []
-    AgentOrchestrator._append_sandbox_env(cmd, _config(info))
+    AgentOrchestrator._append_sandbox_marker_env(cmd, ["mongo"])
 
-    assert "ROBOCO_TEST_DB_HOST=roboco-sandbox-pg-dev-1" in cmd
-    assert not any(v.startswith("ROBOCO_TEST_REDIS_") for v in cmd)
-
-
-def test_append_sandbox_env_injects_mongo() -> None:
-    info = SandboxInfo(
-        services={
-            "mongo": SandboxConnection(
-                host="roboco-sandbox-mongo-dev-1",
-                port=27017,
-                password="mpw",
-                user="sandbox",
-                database="admin",
-            )
-        }
-    )
-    cmd: list[str] = []
-    AgentOrchestrator._append_sandbox_env(cmd, _config(info))
-
-    assert "ROBOCO_TEST_MONGO_HOST=roboco-sandbox-mongo-dev-1" in cmd
-    assert "ROBOCO_TEST_MONGO_PORT=27017" in cmd
-    assert "ROBOCO_TEST_MONGO_USER=sandbox" in cmd
-    assert "ROBOCO_TEST_MONGO_PASSWORD=mpw" in cmd
-    assert "ROBOCO_TEST_MONGO_AUTH_DB=admin" in cmd
-    assert not any(v.startswith("ROBOCO_TEST_DB_") for v in cmd)
-
-
-def test_append_sandbox_env_noop_without_sandbox_info() -> None:
-    cmd: list[str] = []
-    AgentOrchestrator._append_sandbox_env(cmd, _config(None))
-    assert cmd == []
+    assert "ROBOCO_SANDBOX_SERVICES_AVAILABLE=mongo" in cmd
 
 
 def _fake_proc() -> AsyncMock:
@@ -125,7 +65,7 @@ def _stub_spawn_container_collaborators(
     monkeypatch.setattr(orch, "_append_gate_env", lambda *_a: calls.append("gate"))
     monkeypatch.setattr(
         orch,
-        "_append_sandbox_env",
+        "_append_sandbox_marker_env",
         lambda *_a: calls.append("sandbox"),
     )
     monkeypatch.setattr(orch, "_append_image_and_claude_args", lambda *_a: None)
@@ -135,27 +75,20 @@ def _stub_spawn_container_collaborators(
 
 
 @pytest.mark.asyncio
-async def test_spawn_container_uses_sandbox_env_when_sandbox_active(
+async def test_spawn_container_uses_marker_env_when_opted_in(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     orch = AgentOrchestrator.__new__(AgentOrchestrator)
     calls: list[str] = []
     _stub_spawn_container_collaborators(monkeypatch, orch, calls)
 
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="h", port=5432, password="pw", user="sandbox", database="sandbox"
-            )
-        }
-    )
-    await orch._spawn_container(_config(info))
+    await orch._spawn_container(_config(["postgres"]))
 
     assert calls == ["sandbox"]
 
 
 @pytest.mark.asyncio
-async def test_spawn_container_uses_legacy_gate_env_without_sandbox(
+async def test_spawn_container_uses_legacy_gate_env_when_not_opted_in(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     orch = AgentOrchestrator.__new__(AgentOrchestrator)
@@ -168,25 +101,19 @@ async def test_spawn_container_uses_legacy_gate_env_without_sandbox(
 
 
 @pytest.mark.asyncio
-async def test_spawn_container_stale_clear_spares_fresh_sandbox(
+async def test_spawn_container_stale_clear_runs_with_teardown_sandbox_false(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    """The pre-spawn stale-clear must not tear down the sandbox that was
-    just provisioned for this very spawn (teardown_sandbox=False)."""
+    """The pre-spawn stale-clear is vestigial now (nothing is provisioned
+    before spawn) but still passes teardown_sandbox=False — it must not
+    tear down a sandbox the agent requested moments ago via the verb."""
     orch = AgentOrchestrator.__new__(AgentOrchestrator)
     calls: list[str] = []
     _stub_spawn_container_collaborators(monkeypatch, orch, calls)
     remove = AsyncMock(return_value=None)
     monkeypatch.setattr(orch, "_remove_container", remove)
 
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="h", port=5432, password="pw", user="sandbox", database="sandbox"
-            )
-        }
-    )
-    await orch._spawn_container(_config(info))
+    await orch._spawn_container(_config(["postgres"]))
 
     remove.assert_awaited_once_with(
         "roboco-agent-dev-1",

tests/unit/runtime/test_sandbox_provision_spawn.py

@@ -1,13 +1,18 @@
-"""`AgentOrchestrator._maybe_provision_sandbox` — the spawn-time decision gate.
+"""`AgentOrchestrator._sandbox_available_services` — the spawn-time availability
+probe — and `ensure_sandbox` — the on-demand provision/cache path used by the
+`request_sandbox` do-verb.
 
-Off (flag or project) => None, byte-for-byte identical to legacy behavior. A
-project lookup hiccup degrades to "no sandbox" (best-effort, matching the
-ambient-conventions-resolution convention); an actual provisioning failure
-IS fail-loud — an agent whose gate can't run must never spawn.
+Off (flag or project) => [], byte-for-byte identical to legacy behavior. A
+project lookup hiccup degrades to "no sandbox available" (best-effort,
+matching the ambient-conventions-resolution convention). Provisioning itself
+no longer happens at spawn time — a spawn never fails on sandbox
+infrastructure; `ensure_sandbox` is the only path that calls
+`SandboxProvisioner.provision`, and it is idempotent via an in-memory cache.
 """
 
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -15,15 +20,19 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 from roboco.config import settings
 from roboco.models.sandbox import SandboxConnection, SandboxInfo
-from roboco.runtime.orchestrator import AgentOrchestrator, AgentReadinessError
+from roboco.runtime.orchestrator import AgentOrchestrator
 
 
 def _make_orchestrator() -> tuple[AgentOrchestrator, MagicMock]:
     orch = AgentOrchestrator.__new__(AgentOrchestrator)
     orch._bg_tasks = set()
     orch._running = True
+    orch._sandbox_info = {}
     sandbox = MagicMock()
     sandbox.provision = AsyncMock()
+    # Live by default so cache-hit tests that don't care about liveness pass
+    # through; tests exercising DEFECT 3 (dead-container eviction) override.
+    sandbox.is_live = AsyncMock(return_value=True)
     orch._sandbox = sandbox
     return orch, sandbox
 
@@ -33,23 +42,28 @@ async def _fake_db_ctx(db: Any) -> Any:
     yield db
 
 
+# ---------------------------------------------------------------------------
+# _sandbox_available_services (spawn-time probe, no provisioning)
+# ---------------------------------------------------------------------------
+
+
 @pytest.mark.asyncio
-async def test_flag_off_returns_none_without_project_lookup(
+async def test_flag_off_returns_empty_without_project_lookup(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setattr(settings, "sandbox_db_enabled", False)
     orch, sandbox = _make_orchestrator()
 
     with patch("roboco.services.project.get_project_service") as get_svc:
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")
 
-    assert result is None
+    assert result == []
     get_svc.assert_not_called()
     sandbox.provision.assert_not_called()
 
 
 @pytest.mark.asyncio
-async def test_project_without_sandbox_services_returns_none(
+async def test_project_without_sandbox_services_returns_empty(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setattr(settings, "sandbox_db_enabled", True)
@@ -65,14 +79,14 @@ async def test_project_without_sandbox_services_returns_none(
             return_value=project_service,
         ),
     ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")
 
-    assert result is None
+    assert result == []
     sandbox.provision.assert_not_called()
 
 
 @pytest.mark.asyncio
-async def test_missing_project_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_missing_project_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(settings, "sandbox_db_enabled", True)
     orch, _sandbox = _make_orchestrator()
     project_service = MagicMock()
@@ -85,13 +99,13 @@ async def test_missing_project_returns_none(monkeypatch: pytest.MonkeyPatch) ->
             return_value=project_service,
         ),
     ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")
 
-    assert result is None
+    assert result == []
 
 
 @pytest.mark.asyncio
-async def test_opted_in_project_provisions_sandbox(
+async def test_opted_in_project_returns_services_without_provisioning(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setattr(settings, "sandbox_db_enabled", True)
@@ -99,14 +113,6 @@ async def test_opted_in_project_provisions_sandbox(
     project = MagicMock(sandbox_services=["postgres"])
     project_service = MagicMock()
     project_service.get_by_slug = AsyncMock(return_value=project)
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="h", port=5432, password="pw", user="sandbox", database="sandbox"
-            )
-        }
-    )
-    sandbox.provision.return_value = info
 
     with (
         patch("roboco.db.base.get_db_context", return_value=_fake_db_ctx(MagicMock())),
@@ -115,32 +121,10 @@ async def test_opted_in_project_provisions_sandbox(
             return_value=project_service,
         ),
     ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")
 
-    assert result is info
-    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres"])
-
-
-@pytest.mark.asyncio
-async def test_provisioning_failure_raises_readiness_error(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
-    orch, sandbox = _make_orchestrator()
-    project = MagicMock(sandbox_services=["postgres", "redis"])
-    project_service = MagicMock()
-    project_service.get_by_slug = AsyncMock(return_value=project)
-    sandbox.provision.side_effect = RuntimeError("boom")
-
-    with (
-        patch("roboco.db.base.get_db_context", return_value=_fake_db_ctx(MagicMock())),
-        patch(
-            "roboco.services.project.get_project_service",
-            return_value=project_service,
-        ),
-        pytest.raises(AgentReadinessError, match="sandbox provisioning failed"),
-    ):
-        await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+    assert result == ["postgres"]
+    sandbox.provision.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -151,7 +135,138 @@ async def test_project_lookup_failure_degrades_to_no_sandbox(
     orch, sandbox = _make_orchestrator()
 
     with patch("roboco.db.base.get_db_context", side_effect=RuntimeError("db down")):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")
 
-    assert result is None
+    assert result == []
     sandbox.provision.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# ensure_sandbox (on-demand provision + cache, called by request_sandbox)
+# ---------------------------------------------------------------------------
+
+
+def _info(services: dict[str, SandboxConnection]) -> SandboxInfo:
+    return SandboxInfo(services=services)
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_miss_provisions_and_caches() -> None:
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    sandbox.provision.return_value = info
+
+    result = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    assert result is info
+    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres"])
+    assert orch._sandbox_info["dev-1"] is info
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_hit_skips_second_provision() -> None:
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    sandbox.provision.return_value = info
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    second = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    assert first is second is info
+    sandbox.provision.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_first_subset_request_provisions_full_opted_set() -> None:
+    """DEFECT 1 fix: a first request for a subset of the project's opted-in
+    set provisions the FULL opted set — not just what this call named — so a
+    later call for the rest of that set is a guaranteed cache hit and never
+    falls through to a fresh provision() (whose pre-clear teardown() would
+    otherwise kill the live container the agent is already using)."""
+    orch, sandbox = _make_orchestrator()
+    info = _info(
+        {
+            "postgres": SandboxConnection(host="h", port=5432, password="pw"),
+            "redis": SandboxConnection(host="h", port=6379, password="rw"),
+        }
+    )
+    sandbox.provision.return_value = info
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres", "redis"])
+    second = await orch.ensure_sandbox(
+        "dev-1", ["postgres", "redis"], ["postgres", "redis"]
+    )
+
+    assert first is second is info
+    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres", "redis"])
+    assert orch._sandbox_info["dev-1"] is info
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_is_per_agent_slug() -> None:
+    """Caller A's cache entry never leaks to caller B (cross-agent isolation)."""
+    orch, sandbox = _make_orchestrator()
+    info_a = _info({"postgres": SandboxConnection(host="a", port=5432, password="pa")})
+    info_b = _info({"postgres": SandboxConnection(host="b", port=5432, password="pb")})
+    sandbox.provision.side_effect = [info_a, info_b]
+
+    result_a = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    result_b = await orch.ensure_sandbox("dev-2", ["postgres"], ["postgres"])
+
+    assert result_a is info_a
+    assert result_b is info_b
+    assert orch._sandbox_info["dev-1"] is info_a
+    assert orch._sandbox_info["dev-2"] is info_b
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_concurrent_calls_serialize_on_agent_lock() -> None:
+    """DEFECT 2 fix: two concurrent ensure_sandbox calls for the same agent
+    (e.g. a client timeout + retry) must serialize behind the per-agent lock
+    so only one provision() ever runs — never a race between provision() and
+    a concurrent teardown()."""
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    calls = 0
+
+    async def _slow_provision(_agent_id: str, _services: list[str]) -> SandboxInfo:
+        nonlocal calls
+        calls += 1
+        await asyncio.sleep(0.05)
+        return info
+
+    sandbox.provision.side_effect = _slow_provision
+
+    results = await asyncio.gather(
+        orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"]),
+        orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"]),
+    )
+
+    assert results[0] is results[1] is info
+    assert calls == 1
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_hit_with_dead_container_reprovisions() -> None:
+    """DEFECT 3 fix: a cache hit whose container is no longer live (OOM-killed,
+    manually removed) is evicted and re-provisioned with fresh creds, rather
+    than handing back creds for a container that no longer exists."""
+    orch, sandbox = _make_orchestrator()
+    stale_info = _info(
+        {"postgres": SandboxConnection(host="h", port=5432, password="pw-old")}
+    )
+    fresh_info = _info(
+        {"postgres": SandboxConnection(host="h", port=5432, password="pw-new")}
+    )
+    sandbox.provision.side_effect = [stale_info, fresh_info]
+    sandbox.is_live.return_value = False
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    second = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    expected_provision_calls = 2
+    assert first is stale_info
+    assert second is fresh_info
+    assert sandbox.provision.await_count == expected_provision_calls
+    assert orch._sandbox_info["dev-1"] is fresh_info
+    sandbox.is_live.assert_awaited_once_with("dev-1", ["postgres"])

tests/unit/runtime/test_sandbox_provisioner.py

@@ -40,6 +40,10 @@ class _FakeRunner:
         # path skips the pull). Tests exercising the pull path override these.
         self.image_present: bool = True
         self.pull_rc: int = 0
+        # is_live()'s `docker inspect --format={{.State.Running}}` fake —
+        # set post-construction, mirroring image_present/pull_rc above.
+        self.inspect_rc: int = 0
+        self.inspect_running: bool = True
         self._ps_call_count = 0
 
     async def __call__(
@@ -48,26 +52,33 @@ class _FakeRunner:
         self.calls.append(args)
         verb = args[0]
         if verb == "run":
-            return self.run_rc, b"container-id\n", b""
-        if verb == "exec":
-            return self.exec_rc, b"", b""
-        if verb in ("stop", "kill", "rm"):
-            return self.teardown_rc, b"", b""
-        if verb == "image":
+            rc, out, err = self.run_rc, b"container-id\n", b""
+        elif verb == "exec":
+            rc, out, err = self.exec_rc, b"", b""
+        elif verb in ("stop", "kill", "rm"):
+            rc, out, err = self.teardown_rc, b"", b""
+        elif verb == "image":
             # `image inspect <img>` — rc 0 means present (skip pull).
             if args[1] != "inspect":
                 raise AssertionError(f"unexpected image subverb: {args[1]}")
-            return (0 if self.image_present else 1), b"", b""
-        if verb == "pull":
-            return self.pull_rc, b"", b"" if self.pull_rc == 0 else b"pull failed\n"
-        if verb == "ps":
+            rc, out, err = (0 if self.image_present else 1), b"", b""
+        elif verb == "pull":
+            rc = self.pull_rc
+            out, err = b"", (b"" if rc == 0 else b"pull failed\n")
+        elif verb == "ps":
             self._ps_call_count += 1
             # First ps call = the sandbox-labeled listing; second = live agents.
             listing = (
                 self.ps_output if self._ps_call_count == 1 else self.ps_live_output
             )
-            return 0, listing, b""
-        raise AssertionError(f"unexpected docker verb: {verb}")
+            rc, out, err = 0, listing, b""
+        elif verb == "inspect":
+            rc = self.inspect_rc
+            out = b"true\n" if self.inspect_running else b"false\n"
+            err = b""
+        else:
+            raise AssertionError(f"unexpected docker verb: {verb}")
+        return rc, out, err
 
 
 @pytest.fixture(autouse=True)
@@ -296,3 +307,48 @@ async def test_provision_pull_failure_raises() -> None:
         await provisioner.provision("dev-9", ["postgres"])
     # `docker run` never reached — pull failed first.
     assert not any(c[0] == "run" for c in runner.calls)
+
+
+@pytest.mark.asyncio
+async def test_is_live_true_when_container_running() -> None:
+    runner = _FakeRunner()
+
+    provisioner = SandboxProvisioner(network=_NETWORK, runner=runner)
+
+    assert await provisioner.is_live("dev-10", ["postgres", "redis"]) is True
+    expected_inspect_calls = 2
+    inspects = [c for c in runner.calls if c[0] == "inspect"]
+    assert len(inspects) == expected_inspect_calls
+
+
+@pytest.mark.asyncio
+async def test_is_live_false_when_container_stopped() -> None:
+    """rc 0 but State.Running == false — container exists but isn't running."""
+    runner = _FakeRunner()
+    runner.inspect_running = False
+    provisioner = SandboxProvisioner(network=_NETWORK, runner=runner)
+
+    assert await provisioner.is_live("dev-11", ["postgres"]) is False
+
+
+@pytest.mark.asyncio
+async def test_is_live_false_when_container_missing() -> None:
+    """Nonzero rc — `docker inspect` fails outright on a removed container."""
+    runner = _FakeRunner()
+    runner.inspect_rc = 1
+    provisioner = SandboxProvisioner(network=_NETWORK, runner=runner)
+
+    assert await provisioner.is_live("dev-12", ["postgres"]) is False
+
+
+@pytest.mark.asyncio
+async def test_is_live_short_circuits_on_first_dead_service() -> None:
+    """A dead first service skips checking the rest — no need to inspect
+    every container once one is already known dead."""
+    runner = _FakeRunner()
+    runner.inspect_rc = 1
+    provisioner = SandboxProvisioner(network=_NETWORK, runner=runner)
+
+    assert await provisioner.is_live("dev-13", ["postgres", "redis"]) is False
+    inspects = [c for c in runner.calls if c[0] == "inspect"]
+    assert len(inspects) == 1

tests/unit/runtime/test_sandbox_teardown_wiring.py

@@ -118,3 +118,98 @@ async def test_sandbox_janitor_sweep_swallows_errors(
     sandbox.janitor_sweep.side_effect = RuntimeError("boom")
 
     await orch._sandbox_janitor_sweep()  # must not raise
+
+
+# ---------------------------------------------------------------------------
+# ensure_sandbox cache eviction (request_sandbox on-demand provisioning)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_remove_container_evicts_ensure_sandbox_cache(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
+    monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_create_subprocess_exec)
+    orch, _sandbox = _make_orchestrator()
+    orch._sandbox_info = {"dev-1": MagicMock(), "dev-2": MagicMock()}
+
+    await orch._remove_container("roboco-agent-dev-1")
+
+    assert "dev-1" not in orch._sandbox_info
+    assert "dev-2" in orch._sandbox_info
+
+
+@pytest.mark.asyncio
+async def test_remove_container_teardown_false_spares_cache(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
+    monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_create_subprocess_exec)
+    orch, _sandbox = _make_orchestrator()
+    orch._sandbox_info = {"dev-1": MagicMock()}
+
+    await orch._remove_container("roboco-agent-dev-1", teardown_sandbox=False)
+
+    assert "dev-1" in orch._sandbox_info
+
+
+@pytest.mark.asyncio
+async def test_janitor_sweep_evicts_cache_for_reaped_agents(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
+    orch, _sandbox = _make_orchestrator()
+    orch._sandbox_info = {"dev-1": MagicMock(), "dev-2": MagicMock()}
+    orch._instances = {"dev-2": MagicMock()}  # dev-1's agent instance is gone
+
+    await orch._sandbox_janitor_sweep()
+
+    assert "dev-1" not in orch._sandbox_info
+    assert "dev-2" in orch._sandbox_info
+
+
+# ---------------------------------------------------------------------------
+# release_sandbox (end-of-engagement teardown, called by the Choreographer's
+# post-verb hook — i_am_done / unclaim / i_am_idle / pass_review / fail_review
+# / i_documented — instead of only at container removal)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_release_sandbox_no_cache_entry_is_fast_noop() -> None:
+    """The overwhelmingly common case: no sandbox for this agent. Must not
+    take the per-agent lock or call docker — the cache dict check alone
+    decides, before any lock is even allocated."""
+    orch, sandbox = _make_orchestrator()
+    orch._sandbox_info = {}
+
+    await orch.release_sandbox("dev-1")
+
+    sandbox.teardown.assert_not_called()
+    assert not hasattr(orch, "_sandbox_locks")
+
+
+@pytest.mark.asyncio
+async def test_release_sandbox_tears_down_and_evicts_cache() -> None:
+    orch, sandbox = _make_orchestrator()
+    orch._sandbox_info = {"dev-1": MagicMock(), "dev-2": MagicMock()}
+
+    await orch.release_sandbox("dev-1")
+
+    sandbox.teardown.assert_awaited_once_with("dev-1")
+    assert "dev-1" not in orch._sandbox_info
+    assert "dev-2" in orch._sandbox_info
+
+
+@pytest.mark.asyncio
+async def test_release_sandbox_is_idempotent() -> None:
+    """A second release for the same slug (e.g. unclaim right after
+    i_am_idle) is a no-op — the first call already evicted the cache."""
+    orch, sandbox = _make_orchestrator()
+    orch._sandbox_info = {"dev-1": MagicMock()}
+
+    await orch.release_sandbox("dev-1")
+    await orch.release_sandbox("dev-1")
+
+    sandbox.teardown.assert_awaited_once_with("dev-1")