feat(sandbox): on-demand provisioning via request_sandbox verb (#338)

* feat(sandbox): on-demand request_sandbox verb replaces eager provisioning Sandboxes were provisioned at every agent spawn for opted-in projects, so every role paid the sidecar spin-up and a provisioning failure refused the spawn. Provisioning now happens when an agent asks: the request_sandbox do-verb (dev + QA) reaches the orchestrator through ContentActionsDeps, ensure_sandbox provisions idempotently with an in-memory per-agent cache (evicted at teardown and janitor sweep), and creds return in the envelope payload including ready-to-export ROBOCO_TEST_* values. Spawn now only injects a marker env naming the available services plus a briefing line; sandbox failures can no longer refuse a spawn. Teardown lifecycle unchanged. * feat(sandbox): harden request_sandbox + Phase 3 wiring proof and docs Hardening from adversarial review: ensure_sandbox now provisions the project's full opted-in set on first request (a later superset can never tear down a live sandbox mid-use), serializes per-agent behind an asyncio lock (a client timeout-retry no longer races its own in-flight provision), and verifies container liveness on every cache hit (a dead sandbox evicts and re-provisions instead of serving dead creds). MCP client budget 720->1080s for the full-set cold case. Phase 3: e2e smoke wiring test (manifest grants + guard-chain envelopes over the real API), sandbox-db/tools/map docs and CLAUDE.md rewritten for on-demand. * feat(sandbox): release sandboxes when the agent's work ends CEO directive: sidecars must not dangle once the agent is done. The six work-ending verbs (i_am_done, unclaim, i_am_idle, pass_review, fail_review, i_documented) now release the caller's sandbox best-effort on their success path via release_sandbox (lock + teardown + cache evict; a no-sandbox agent costs a dict lookup). Container removal and the janitor remain the backstop; a re-request provisions fresh. * test(sandbox): monkeypatch the release hook instead of method assignment mypy method-assign rejected the direct AsyncMock assignments; the prior static gate ran before this test file landed. * test(sandbox): guard envelope evidence for mypy in verb tests * chore(prompts): regenerate verb tables for request_sandbox * chore: resolve merge with master (breadcrumbs + statement budget) --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
2026-08-03 07:23:24 +02:00 · 2026-07-08 16:40:02 +02:00
parent 9e4025b822
commit 47d78f50ee
30 changed files with 1824 additions and 246 deletions
@@ -1,13 +1,18 @@
-"""`AgentOrchestrator._maybe_provision_sandbox` — the spawn-time decision gate.
+"""`AgentOrchestrator._sandbox_available_services` — the spawn-time availability
+probe — and `ensure_sandbox` — the on-demand provision/cache path used by the
+`request_sandbox` do-verb.

-Off (flag or project) => None, byte-for-byte identical to legacy behavior. A
-project lookup hiccup degrades to "no sandbox" (best-effort, matching the
-ambient-conventions-resolution convention); an actual provisioning failure
-IS fail-loud — an agent whose gate can't run must never spawn.
+Off (flag or project) => [], byte-for-byte identical to legacy behavior. A
+project lookup hiccup degrades to "no sandbox available" (best-effort,
+matching the ambient-conventions-resolution convention). Provisioning itself
+no longer happens at spawn time — a spawn never fails on sandbox
+infrastructure; `ensure_sandbox` is the only path that calls
+`SandboxProvisioner.provision`, and it is idempotent via an in-memory cache.
 """

 from __future__ import annotations

+import asyncio
 from contextlib import asynccontextmanager
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -15,15 +20,19 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 from roboco.config import settings
 from roboco.models.sandbox import SandboxConnection, SandboxInfo
-from roboco.runtime.orchestrator import AgentOrchestrator, AgentReadinessError
+from roboco.runtime.orchestrator import AgentOrchestrator


 def _make_orchestrator() -> tuple[AgentOrchestrator, MagicMock]:
    orch = AgentOrchestrator.__new__(AgentOrchestrator)
    orch._bg_tasks = set()
    orch._running = True
+    orch._sandbox_info = {}
    sandbox = MagicMock()
    sandbox.provision = AsyncMock()
+    # Live by default so cache-hit tests that don't care about liveness pass
+    # through; tests exercising DEFECT 3 (dead-container eviction) override.
+    sandbox.is_live = AsyncMock(return_value=True)
    orch._sandbox = sandbox
    return orch, sandbox

@@ -33,23 +42,28 @@ async def _fake_db_ctx(db: Any) -> Any:
    yield db


+# ---------------------------------------------------------------------------
+# _sandbox_available_services (spawn-time probe, no provisioning)
+# ---------------------------------------------------------------------------
+
+
@pytest.mark.asyncio
-async def test_flag_off_returns_none_without_project_lookup(
+async def test_flag_off_returns_empty_without_project_lookup(
    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
    monkeypatch.setattr(settings, "sandbox_db_enabled", False)
    orch, sandbox = _make_orchestrator()

    with patch("roboco.services.project.get_project_service") as get_svc:
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")

-    assert result is None
+    assert result == []
    get_svc.assert_not_called()
    sandbox.provision.assert_not_called()


@pytest.mark.asyncio
-async def test_project_without_sandbox_services_returns_none(
+async def test_project_without_sandbox_services_returns_empty(
    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
@@ -65,14 +79,14 @@ async def test_project_without_sandbox_services_returns_none(
            return_value=project_service,
        ),
    ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")

-    assert result is None
+    assert result == []
    sandbox.provision.assert_not_called()


@pytest.mark.asyncio
-async def test_missing_project_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_missing_project_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
    orch, _sandbox = _make_orchestrator()
    project_service = MagicMock()
@@ -85,13 +99,13 @@ async def test_missing_project_returns_none(monkeypatch: pytest.MonkeyPatch) ->
            return_value=project_service,
        ),
    ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")

-    assert result is None
+    assert result == []


@pytest.mark.asyncio
-async def test_opted_in_project_provisions_sandbox(
+async def test_opted_in_project_returns_services_without_provisioning(
    monkeypatch: pytest.MonkeyPatch,
 ) -> None:
    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
@@ -99,14 +113,6 @@ async def test_opted_in_project_provisions_sandbox(
    project = MagicMock(sandbox_services=["postgres"])
    project_service = MagicMock()
    project_service.get_by_slug = AsyncMock(return_value=project)
-    info = SandboxInfo(
-        services={
-            "postgres": SandboxConnection(
-                host="h", port=5432, password="pw", user="sandbox", database="sandbox"
-            )
-        }
-    )
-    sandbox.provision.return_value = info

    with (
        patch("roboco.db.base.get_db_context", return_value=_fake_db_ctx(MagicMock())),
@@ -115,32 +121,10 @@ async def test_opted_in_project_provisions_sandbox(
            return_value=project_service,
        ),
    ):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")

-    assert result is info
-    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres"])
-
-
-@pytest.mark.asyncio
-async def test_provisioning_failure_raises_readiness_error(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    monkeypatch.setattr(settings, "sandbox_db_enabled", True)
-    orch, sandbox = _make_orchestrator()
-    project = MagicMock(sandbox_services=["postgres", "redis"])
-    project_service = MagicMock()
-    project_service.get_by_slug = AsyncMock(return_value=project)
-    sandbox.provision.side_effect = RuntimeError("boom")
-
-    with (
-        patch("roboco.db.base.get_db_context", return_value=_fake_db_ctx(MagicMock())),
-        patch(
-            "roboco.services.project.get_project_service",
-            return_value=project_service,
-        ),
-        pytest.raises(AgentReadinessError, match="sandbox provisioning failed"),
-    ):
-        await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+    assert result == ["postgres"]
+    sandbox.provision.assert_not_called()


@pytest.mark.asyncio
@@ -151,7 +135,138 @@ async def test_project_lookup_failure_degrades_to_no_sandbox(
    orch, sandbox = _make_orchestrator()

    with patch("roboco.db.base.get_db_context", side_effect=RuntimeError("db down")):
-        result = await orch._maybe_provision_sandbox("dev-1", "roboco-api", "task-1")
+        result = await orch._sandbox_available_services("roboco-api")

-    assert result is None
+    assert result == []
    sandbox.provision.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# ensure_sandbox (on-demand provision + cache, called by request_sandbox)
+# ---------------------------------------------------------------------------
+
+
+def _info(services: dict[str, SandboxConnection]) -> SandboxInfo:
+    return SandboxInfo(services=services)
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_miss_provisions_and_caches() -> None:
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    sandbox.provision.return_value = info
+
+    result = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    assert result is info
+    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres"])
+    assert orch._sandbox_info["dev-1"] is info
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_hit_skips_second_provision() -> None:
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    sandbox.provision.return_value = info
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    second = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    assert first is second is info
+    sandbox.provision.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_first_subset_request_provisions_full_opted_set() -> None:
+    """DEFECT 1 fix: a first request for a subset of the project's opted-in
+    set provisions the FULL opted set — not just what this call named — so a
+    later call for the rest of that set is a guaranteed cache hit and never
+    falls through to a fresh provision() (whose pre-clear teardown() would
+    otherwise kill the live container the agent is already using)."""
+    orch, sandbox = _make_orchestrator()
+    info = _info(
+        {
+            "postgres": SandboxConnection(host="h", port=5432, password="pw"),
+            "redis": SandboxConnection(host="h", port=6379, password="rw"),
+        }
+    )
+    sandbox.provision.return_value = info
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres", "redis"])
+    second = await orch.ensure_sandbox(
+        "dev-1", ["postgres", "redis"], ["postgres", "redis"]
+    )
+
+    assert first is second is info
+    sandbox.provision.assert_awaited_once_with("dev-1", ["postgres", "redis"])
+    assert orch._sandbox_info["dev-1"] is info
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_is_per_agent_slug() -> None:
+    """Caller A's cache entry never leaks to caller B (cross-agent isolation)."""
+    orch, sandbox = _make_orchestrator()
+    info_a = _info({"postgres": SandboxConnection(host="a", port=5432, password="pa")})
+    info_b = _info({"postgres": SandboxConnection(host="b", port=5432, password="pb")})
+    sandbox.provision.side_effect = [info_a, info_b]
+
+    result_a = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    result_b = await orch.ensure_sandbox("dev-2", ["postgres"], ["postgres"])
+
+    assert result_a is info_a
+    assert result_b is info_b
+    assert orch._sandbox_info["dev-1"] is info_a
+    assert orch._sandbox_info["dev-2"] is info_b
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_concurrent_calls_serialize_on_agent_lock() -> None:
+    """DEFECT 2 fix: two concurrent ensure_sandbox calls for the same agent
+    (e.g. a client timeout + retry) must serialize behind the per-agent lock
+    so only one provision() ever runs — never a race between provision() and
+    a concurrent teardown()."""
+    orch, sandbox = _make_orchestrator()
+    info = _info({"postgres": SandboxConnection(host="h", port=5432, password="pw")})
+    calls = 0
+
+    async def _slow_provision(_agent_id: str, _services: list[str]) -> SandboxInfo:
+        nonlocal calls
+        calls += 1
+        await asyncio.sleep(0.05)
+        return info
+
+    sandbox.provision.side_effect = _slow_provision
+
+    results = await asyncio.gather(
+        orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"]),
+        orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"]),
+    )
+
+    assert results[0] is results[1] is info
+    assert calls == 1
+
+
+@pytest.mark.asyncio
+async def test_ensure_sandbox_cache_hit_with_dead_container_reprovisions() -> None:
+    """DEFECT 3 fix: a cache hit whose container is no longer live (OOM-killed,
+    manually removed) is evicted and re-provisioned with fresh creds, rather
+    than handing back creds for a container that no longer exists."""
+    orch, sandbox = _make_orchestrator()
+    stale_info = _info(
+        {"postgres": SandboxConnection(host="h", port=5432, password="pw-old")}
+    )
+    fresh_info = _info(
+        {"postgres": SandboxConnection(host="h", port=5432, password="pw-new")}
+    )
+    sandbox.provision.side_effect = [stale_info, fresh_info]
+    sandbox.is_live.return_value = False
+
+    first = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+    second = await orch.ensure_sandbox("dev-1", ["postgres"], ["postgres"])
+
+    expected_provision_calls = 2
+    assert first is stale_info
+    assert second is fresh_info
+    assert sandbox.provision.await_count == expected_provision_calls
+    assert orch._sandbox_info["dev-1"] is fresh_info
+    sandbox.is_live.assert_awaited_once_with("dev-1", ["postgres"])