"""SandboxProvisioner: throwaway per-spawn Postgres/Redis sibling containers. All docker calls are mocked — no real docker in unit tests. Readiness deadlines are monkeypatched down so the timeout path runs in milliseconds. """ from __future__ import annotations import time import pytest from roboco.runtime import sandbox as sandbox_module from roboco.runtime.sandbox import SandboxProvisioner, SandboxProvisionError _NETWORK = "roboco_default" _PG_PORT = 5432 _REDIS_PORT = 6379 _MONGO_PORT = 27017 class _FakeRunner: """Records every docker invocation; behavior configured per test.""" def __init__( self, *, run_rc: int = 0, exec_rc: int = 0, teardown_rc: int = 0, ps_output: bytes = b"", ps_live_output: bytes = b"", ) -> None: self.calls: list[list[str]] = [] self.run_rc = run_rc self.exec_rc = exec_rc self.teardown_rc = teardown_rc self.ps_output = ps_output self.ps_live_output = ps_live_output # Image state — defaults assume the image is already present (the happy # path skips the pull). Tests exercising the pull path override these. self.image_present: bool = True self.pull_rc: int = 0 # is_live()'s `docker inspect --format={{.State.Running}}` fake — # set post-construction, mirroring image_present/pull_rc above. self.inspect_rc: int = 0 self.inspect_running: bool = True self._ps_call_count = 0 async def __call__( self, args: list[str], _timeout: float ) -> tuple[int, bytes, bytes]: self.calls.append(args) verb = args[0] if verb == "run": rc, out, err = self.run_rc, b"container-id\n", b"" elif verb == "exec": rc, out, err = self.exec_rc, b"", b"" elif verb in ("stop", "kill", "rm"): rc, out, err = self.teardown_rc, b"", b"" elif verb == "image": # `image inspect ` — rc 0 means present (skip pull). if args[1] != "inspect": raise AssertionError(f"unexpected image subverb: {args[1]}") rc, out, err = (0 if self.image_present else 1), b"", b"" elif verb == "pull": rc = self.pull_rc out, err = b"", (b"" if rc == 0 else b"pull failed\n") elif verb == "ps": self._ps_call_count += 1 # First ps call = the sandbox-labeled listing; second = live agents. listing = ( self.ps_output if self._ps_call_count == 1 else self.ps_live_output ) rc, out, err = 0, listing, b"" elif verb == "inspect": rc = self.inspect_rc out = b"true\n" if self.inspect_running else b"false\n" err = b"" else: raise AssertionError(f"unexpected docker verb: {verb}") return rc, out, err @pytest.fixture(autouse=True) def _fast_readiness_deadlines(monkeypatch: pytest.MonkeyPatch) -> None: """Shrink the polling cadence + every engine's deadline so the timeout path is fast in tests. Deadlines live on the engine instances now (not module constants), so monkeypatch them on the registry.""" monkeypatch.setattr(sandbox_module, "_READY_POLL_INTERVAL_SECONDS", 0.01) for engine in sandbox_module.SANDBOX_ENGINES.values(): monkeypatch.setattr(engine, "ready_deadline", 0.05) @pytest.mark.asyncio async def test_provision_both_services_happy_path() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-1", ["postgres", "redis"]) pg = info.services["postgres"] assert pg.host == "roboco-sandbox-pg-dev-1" assert pg.port == _PG_PORT assert pg.user == "sandbox" assert pg.database == "sandbox" rd = info.services["redis"] assert rd.host == "roboco-sandbox-redis-dev-1" assert rd.port == _REDIS_PORT # Passwords are per-sandbox random tokens, not equal to each other. assert pg.password != rd.password @pytest.mark.asyncio async def test_provision_labels_are_correct() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-2", ["postgres"]) run_call = next(c for c in runner.calls if c[0] == "run") assert "--network" in run_call assert run_call[run_call.index("--network") + 1] == _NETWORK label_indices = [i for i, a in enumerate(run_call) if a == "--label"] labels = [run_call[i + 1] for i in label_indices] assert sandbox_module.SANDBOX_LABEL in labels assert "roboco.sandbox.owner=roboco-agent-dev-2" in labels @pytest.mark.asyncio async def test_provision_mongo_engine() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-mongo", ["mongo"]) mongo = info.services["mongo"] assert mongo.host == "roboco-sandbox-mongo-dev-mongo" assert mongo.port == _MONGO_PORT assert mongo.user == "sandbox" assert mongo.database == "admin" run_call = next(c for c in runner.calls if c[0] == "run") assert "mongo:8" in run_call # MONGO_INITDB_ROOT_PASSWORD env is baked into the run. assert any(a.startswith("MONGO_INITDB_ROOT_PASSWORD=") for a in run_call) # /data/db tmpfs mount for the engine. assert "--tmpfs" in run_call assert run_call[run_call.index("--tmpfs") + 1] == "/data/db" @pytest.mark.asyncio async def test_provision_readiness_timeout_tears_down_and_raises() -> None: runner = _FakeRunner(run_rc=0, exec_rc=1) # container starts, never ready provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-3", ["postgres"]) # Teardown attempted for the container that failed readiness. teardown_verbs = {c[0] for c in runner.calls if c[0] in ("stop", "kill", "rm")} assert "stop" in teardown_verbs or "rm" in teardown_verbs rm_calls = [c for c in runner.calls if c[0] == "rm"] assert any("roboco-sandbox-pg-dev-3" in c for c in rm_calls) @pytest.mark.asyncio async def test_provision_run_failure_tears_down_and_raises() -> None: runner = _FakeRunner(run_rc=1) # docker run itself fails provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-4", ["redis"]) @pytest.mark.asyncio async def test_provision_rejects_unknown_service() -> None: runner = _FakeRunner() provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-5", ["mysql"]) # Nothing was ever run for an unknown service. assert runner.calls == [] @pytest.mark.asyncio async def test_teardown_idempotent_on_missing_container() -> None: runner = _FakeRunner(teardown_rc=1) # "no such container" for every verb provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) # Must not raise even though every teardown call reports failure. await provisioner.teardown("never-provisioned") verbs = [c[0] for c in runner.calls] assert "rm" in verbs @pytest.mark.asyncio async def test_janitor_removes_orphaned_sandbox_only() -> None: # Two sandboxes on the host: one owned by a still-live agent, one orphaned. ps_output = ( b"roboco-sandbox-pg-alive\troboco-agent-alive\n" b"roboco-sandbox-pg-orphan\troboco-agent-orphan\n" ) live_output = b"roboco-agent-alive\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=live_output) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.janitor_sweep() rm_calls = [c for c in runner.calls if c[0] == "rm"] torn_down = {c[-1] for c in rm_calls} assert "roboco-sandbox-pg-orphan" in torn_down assert "roboco-sandbox-pg-alive" not in torn_down @pytest.mark.asyncio async def test_janitor_noop_when_no_sandboxes() -> None: runner = _FakeRunner(ps_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.janitor_sweep() assert all(c[0] != "rm" for c in runner.calls) @pytest.mark.asyncio async def test_provision_preclears_stale_sandboxes_before_run() -> None: """A crash-missed teardown leaves same-named containers; provision must clear them first or `docker run` fails on the name conflict.""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-6", ["postgres"]) first_run = next(i for i, c in enumerate(runner.calls) if c[0] == "run") preclear_rms = [ c for c in runner.calls[:first_run] if c[0] == "rm" and c[-1].endswith("dev-6") ] assert any("roboco-sandbox-pg-dev-6" in c for c in preclear_rms) assert any("roboco-sandbox-redis-dev-6" in c for c in preclear_rms) @pytest.mark.asyncio async def test_janitor_grace_skips_freshly_provisioned_owner() -> None: """A sandbox is provisioned before its agent container exists — a sweep racing that mid-flight spawn must not reap the fresh sandbox.""" ps_output = b"roboco-sandbox-pg-fresh\troboco-agent-fresh\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) provisioner._provisioned_at = {"roboco-agent-fresh": time.monotonic()} await provisioner.janitor_sweep() assert all(c[0] != "rm" for c in runner.calls) @pytest.mark.asyncio async def test_janitor_reaps_after_grace_expiry() -> None: ps_output = b"roboco-sandbox-pg-old\troboco-agent-old\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) provisioner._provisioned_at = { "roboco-agent-old": time.monotonic() - 10 * sandbox_module._JANITOR_GRACE_SECONDS } await provisioner.janitor_sweep() rm_calls = [c for c in runner.calls if c[0] == "rm"] assert any("roboco-sandbox-pg-old" in c for c in rm_calls) assert provisioner._provisioned_at == {} @pytest.mark.asyncio async def test_provision_skips_pull_when_image_present() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-7", ["postgres"]) assert any(c[0] == "image" and c[1] == "inspect" for c in runner.calls) assert not any(c[0] == "pull" for c in runner.calls) @pytest.mark.asyncio async def test_provision_pulls_when_image_absent() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.image_present = False provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-8", ["postgres"]) inspect = [c for c in runner.calls if c[0] == "image" and c[1] == "inspect"] pulls = [c for c in runner.calls if c[0] == "pull"] assert inspect and pulls assert pulls[0][-1] == "postgres:16-alpine" @pytest.mark.asyncio async def test_provision_pull_failure_raises() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.image_present = False runner.pull_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError, match="image pull failed"): await provisioner.provision("dev-9", ["postgres"]) # `docker run` never reached — pull failed first. assert not any(c[0] == "run" for c in runner.calls) @pytest.mark.asyncio async def test_is_live_true_when_container_running() -> None: runner = _FakeRunner() provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-10", ["postgres", "redis"]) is True expected_inspect_calls = 2 inspects = [c for c in runner.calls if c[0] == "inspect"] assert len(inspects) == expected_inspect_calls @pytest.mark.asyncio async def test_is_live_false_when_container_stopped() -> None: """rc 0 but State.Running == false — container exists but isn't running.""" runner = _FakeRunner() runner.inspect_running = False provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-11", ["postgres"]) is False @pytest.mark.asyncio async def test_is_live_false_when_container_missing() -> None: """Nonzero rc — `docker inspect` fails outright on a removed container.""" runner = _FakeRunner() runner.inspect_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-12", ["postgres"]) is False @pytest.mark.asyncio async def test_is_live_short_circuits_on_first_dead_service() -> None: """A dead first service skips checking the rest — no need to inspect every container once one is already known dead.""" runner = _FakeRunner() runner.inspect_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-13", ["postgres", "redis"]) is False inspects = [c for c in runner.calls if c[0] == "inspect"] assert len(inspects) == 1