"""SandboxProvisioner: throwaway per-spawn Postgres/Redis sibling containers. All docker calls are mocked — no real docker in unit tests. Readiness deadlines are monkeypatched down so the timeout path runs in milliseconds. """ from __future__ import annotations import time import pytest from roboco.runtime import sandbox as sandbox_module from roboco.runtime.sandbox import SandboxProvisioner, SandboxProvisionError _NETWORK = "roboco_default" _PG_PORT = 5432 _REDIS_PORT = 6379 _MONGO_PORT = 27017 class _FakeRunner: """Records every docker invocation; behavior configured per test.""" def __init__( self, *, run_rc: int = 0, exec_rc: int = 0, teardown_rc: int = 0, ps_output: bytes = b"", ps_live_output: bytes = b"", ) -> None: self.calls: list[list[str]] = [] self.run_rc = run_rc self.exec_rc = exec_rc # exec stdout — set post-construction (mirrors inspect_rc/pull_rc) by # tests that exercise the verify step's stdout interpretation. self.exec_out: bytes = b"" self.teardown_rc = teardown_rc self.ps_output = ps_output self.ps_live_output = ps_live_output # Image state — defaults assume the image is already present (the happy # path skips the pull). Tests exercising the pull path override these. self.image_present: bool = True self.pull_rc: int = 0 # is_live()'s `docker inspect --format={{.State.Running}}` fake — # set post-construction, mirroring image_present/pull_rc above. self.inspect_rc: int = 0 self.inspect_running: bool = True self._ps_call_count = 0 async def __call__( self, args: list[str], _timeout: float ) -> tuple[int, bytes, bytes]: self.calls.append(args) verb = args[0] if verb == "run": rc, out, err = self.run_rc, b"container-id\n", b"" elif verb == "exec": rc, out, err = self.exec_rc, self.exec_out, b"" elif verb in ("stop", "kill", "rm"): rc, out, err = self.teardown_rc, b"", b"" elif verb == "image": # `image inspect ` — rc 0 means present (skip pull). if args[1] != "inspect": raise AssertionError(f"unexpected image subverb: {args[1]}") rc, out, err = (0 if self.image_present else 1), b"", b"" elif verb == "pull": rc = self.pull_rc out, err = b"", (b"" if rc == 0 else b"pull failed\n") elif verb == "ps": self._ps_call_count += 1 # First ps call = the sandbox-labeled listing; second = live agents. listing = ( self.ps_output if self._ps_call_count == 1 else self.ps_live_output ) rc, out, err = 0, listing, b"" elif verb == "inspect": rc = self.inspect_rc out = b"true\n" if self.inspect_running else b"false\n" err = b"" else: raise AssertionError(f"unexpected docker verb: {verb}") return rc, out, err @pytest.fixture(autouse=True) def _fast_readiness_deadlines(monkeypatch: pytest.MonkeyPatch) -> None: """Shrink the polling cadence + every engine's deadline so the timeout path is fast in tests. Deadlines live on the engine instances now (not module constants), so monkeypatch them on the registry.""" monkeypatch.setattr(sandbox_module, "_READY_POLL_INTERVAL_SECONDS", 0.01) for engine in sandbox_module.SANDBOX_ENGINES.values(): monkeypatch.setattr(engine, "ready_deadline", 0.05) @pytest.mark.asyncio async def test_provision_both_services_happy_path() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-1", ["postgres", "redis"]) pg = info.services["postgres"] assert pg.host == "roboco-sandbox-pg-dev-1" assert pg.port == _PG_PORT assert pg.user == "sandbox" assert pg.database == "sandbox" rd = info.services["redis"] assert rd.host == "roboco-sandbox-redis-dev-1" assert rd.port == _REDIS_PORT # Passwords are per-sandbox random tokens, not equal to each other. assert pg.password != rd.password @pytest.mark.asyncio async def test_provision_labels_are_correct() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-2", ["postgres"]) run_call = next(c for c in runner.calls if c[0] == "run") assert "--network" in run_call assert run_call[run_call.index("--network") + 1] == _NETWORK label_indices = [i for i, a in enumerate(run_call) if a == "--label"] labels = [run_call[i + 1] for i in label_indices] assert sandbox_module.SANDBOX_LABEL in labels assert "roboco.sandbox.owner=roboco-agent-dev-2" in labels @pytest.mark.asyncio async def test_provision_mongo_engine() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-mongo", ["mongo"]) mongo = info.services["mongo"] assert mongo.host == "roboco-sandbox-mongo-dev-mongo" assert mongo.port == _MONGO_PORT assert mongo.user == "sandbox" assert mongo.database == "admin" run_call = next(c for c in runner.calls if c[0] == "run") assert "mongo:8" in run_call # MONGO_INITDB_ROOT_PASSWORD env is baked into the run. assert any(a.startswith("MONGO_INITDB_ROOT_PASSWORD=") for a in run_call) # /data/db tmpfs mount for the engine. assert "--tmpfs" in run_call assert run_call[run_call.index("--tmpfs") + 1] == "/data/db" @pytest.mark.asyncio async def test_provision_readiness_timeout_tears_down_and_raises() -> None: runner = _FakeRunner(run_rc=0, exec_rc=1) # container starts, never ready provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-3", ["postgres"]) # Teardown attempted for the container that failed readiness. teardown_verbs = {c[0] for c in runner.calls if c[0] in ("stop", "kill", "rm")} assert "stop" in teardown_verbs or "rm" in teardown_verbs rm_calls = [c for c in runner.calls if c[0] == "rm"] assert any("roboco-sandbox-pg-dev-3" in c for c in rm_calls) @pytest.mark.asyncio async def test_provision_run_failure_tears_down_and_raises() -> None: runner = _FakeRunner(run_rc=1) # docker run itself fails provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-4", ["redis"]) @pytest.mark.asyncio async def test_provision_rejects_unknown_service() -> None: runner = _FakeRunner() provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision("dev-5", ["mysql"]) # Nothing was ever run for an unknown service. assert runner.calls == [] @pytest.mark.asyncio async def test_teardown_idempotent_on_missing_container() -> None: runner = _FakeRunner(teardown_rc=1) # "no such container" for every verb provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) # Must not raise even though every teardown call reports failure. await provisioner.teardown("never-provisioned") verbs = [c[0] for c in runner.calls] assert "rm" in verbs @pytest.mark.asyncio async def test_janitor_removes_orphaned_sandbox_only() -> None: # Two sandboxes on the host: one owned by a still-live agent, one orphaned. ps_output = ( b"roboco-sandbox-pg-alive\troboco-agent-alive\n" b"roboco-sandbox-pg-orphan\troboco-agent-orphan\n" ) live_output = b"roboco-agent-alive\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=live_output) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.janitor_sweep() rm_calls = [c for c in runner.calls if c[0] == "rm"] torn_down = {c[-1] for c in rm_calls} assert "roboco-sandbox-pg-orphan" in torn_down assert "roboco-sandbox-pg-alive" not in torn_down @pytest.mark.asyncio async def test_janitor_noop_when_no_sandboxes() -> None: runner = _FakeRunner(ps_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.janitor_sweep() assert all(c[0] != "rm" for c in runner.calls) @pytest.mark.asyncio async def test_provision_preclears_stale_sandboxes_before_run() -> None: """A crash-missed teardown leaves same-named containers; provision must clear them first or `docker run` fails on the name conflict.""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-6", ["postgres"]) first_run = next(i for i, c in enumerate(runner.calls) if c[0] == "run") preclear_rms = [ c for c in runner.calls[:first_run] if c[0] == "rm" and c[-1].endswith("dev-6") ] assert any("roboco-sandbox-pg-dev-6" in c for c in preclear_rms) assert any("roboco-sandbox-redis-dev-6" in c for c in preclear_rms) @pytest.mark.asyncio async def test_janitor_grace_skips_freshly_provisioned_owner() -> None: """A sandbox is provisioned before its agent container exists — a sweep racing that mid-flight spawn must not reap the fresh sandbox.""" ps_output = b"roboco-sandbox-pg-fresh\troboco-agent-fresh\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) provisioner._provisioned_at = {"roboco-agent-fresh": time.monotonic()} await provisioner.janitor_sweep() assert all(c[0] != "rm" for c in runner.calls) @pytest.mark.asyncio async def test_janitor_reaps_after_grace_expiry() -> None: ps_output = b"roboco-sandbox-pg-old\troboco-agent-old\n" runner = _FakeRunner(ps_output=ps_output, ps_live_output=b"") provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) provisioner._provisioned_at = { "roboco-agent-old": time.monotonic() - 10 * sandbox_module._JANITOR_GRACE_SECONDS } await provisioner.janitor_sweep() rm_calls = [c for c in runner.calls if c[0] == "rm"] assert any("roboco-sandbox-pg-old" in c for c in rm_calls) assert provisioner._provisioned_at == {} @pytest.mark.asyncio async def test_provision_skips_pull_when_image_present() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-7", ["postgres"]) assert any(c[0] == "image" and c[1] == "inspect" for c in runner.calls) assert not any(c[0] == "pull" for c in runner.calls) @pytest.mark.asyncio async def test_provision_pulls_when_image_absent() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.image_present = False provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-8", ["postgres"]) inspect = [c for c in runner.calls if c[0] == "image" and c[1] == "inspect"] pulls = [c for c in runner.calls if c[0] == "pull"] assert inspect and pulls assert pulls[0][-1] == "postgres:16-alpine" @pytest.mark.asyncio async def test_provision_pull_failure_raises() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.image_present = False runner.pull_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError, match="image pull failed"): await provisioner.provision("dev-9", ["postgres"]) # `docker run` never reached — pull failed first. assert not any(c[0] == "run" for c in runner.calls) @pytest.mark.asyncio async def test_is_live_true_when_container_running() -> None: runner = _FakeRunner() provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-10", ["postgres", "redis"]) is True expected_inspect_calls = 2 inspects = [c for c in runner.calls if c[0] == "inspect"] assert len(inspects) == expected_inspect_calls @pytest.mark.asyncio async def test_is_live_false_when_container_stopped() -> None: """rc 0 but State.Running == false — container exists but isn't running.""" runner = _FakeRunner() runner.inspect_running = False provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-11", ["postgres"]) is False @pytest.mark.asyncio async def test_is_live_false_when_container_missing() -> None: """Nonzero rc — `docker inspect` fails outright on a removed container.""" runner = _FakeRunner() runner.inspect_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-12", ["postgres"]) is False @pytest.mark.asyncio async def test_is_live_short_circuits_on_first_dead_service() -> None: """A dead first service skips checking the rest — no need to inspect every container once one is already known dead.""" runner = _FakeRunner() runner.inspect_rc = 1 provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) assert await provisioner.is_live("dev-13", ["postgres", "redis"]) is False inspects = [c for c in runner.calls if c[0] == "inspect"] assert len(inspects) == 1 # --------------------------------------------------------------------------- # Post-ready feature activation: enable_step + verify_step + allowlist guard. # All docker calls mocked; the enable/verify loop is exercised end-to-end. # --------------------------------------------------------------------------- def _exec_calls(runner: _FakeRunner, containish: str | None = None) -> list[list[str]]: calls = [c for c in runner.calls if c[0] == "exec"] if containish is None: return calls return [c for c in calls if containish in " ".join(c)] @pytest.mark.asyncio async def test_provision_pg_features_runs_enable_then_verify() -> None: # exec_out = "2\n" satisfies the pg verify (count == len(features)). runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"2\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision( "dev-feat", ["postgres"], features={"postgres": ["vector", "postgis"]} ) pg = info.services["postgres"] assert pg.features == ("vector", "postgis") # The enable exec creates both extensions; the verify exec counts them — # assert each by content rather than a magic count. execs = _exec_calls(runner, "psql") enable = next(c for c in execs if "CREATE EXTENSION" in " ".join(c)) verify = next(c for c in execs if "pg_extension" in " ".join(c)) assert "CREATE EXTENSION IF NOT EXISTS vector" in " ".join(enable) assert "CREATE EXTENSION IF NOT EXISTS postgis" in " ".join(enable) assert "ON_ERROR_STOP=1" in enable assert "vector" in " ".join(verify) assert "postgis" in " ".join(verify) @pytest.mark.asyncio async def test_provision_no_features_skips_enable_and_verify() -> None: """Bare provision (no features) is byte-for-byte unchanged: only the base readiness probe exec runs, no enable/verify.""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-bare", ["postgres"]) assert info.services["postgres"].features == () pg_execs = _exec_calls(runner, "pg_isready") assert len(pg_execs) == 1 # readiness only assert _exec_calls(runner, "psql") == [] @pytest.mark.asyncio async def test_provision_redis_features_loads_each_module_then_lists() -> None: # MODULE LIST raw output carrying both module names. runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"search\n99999\nReJSON\n99999\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision( "dev-redis", ["redis"], features={"redis": ["search", "json"]} ) assert info.services["redis"].features == ("search", "json") loads = [ c for c in _exec_calls(runner) if "MODULE" in " ".join(c) and "LOAD" in " ".join(c) ] # one MODULE LOAD per module — assert each .so is loaded, not a magic count. joined = " ".join(" ".join(c) for c in loads) assert "/opt/redis-stack/lib/redisearch.so" in joined assert "/opt/redis-stack/lib/rejson.so" in joined # verify is a single MODULE LIST lists = [ c for c in _exec_calls(runner) if "MODULE" in " ".join(c) and "LIST" in " ".join(c) ] assert len(lists) == 1 @pytest.mark.asyncio async def test_provision_mongo_features_are_a_noop() -> None: """Mongo is batteries-included; features for it are accepted (allowlist is empty so only [] passes) and no enable/verify exec runs.""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-mongo-f", ["mongo"], features={"mongo": []}) assert info.services["mongo"].features == () # Only the mongosh readiness probe — no enable/verify. assert all( "MODULE" not in " ".join(c) and "pg_extension" not in " ".join(c) for c in _exec_calls(runner) ) # --------------------------------------------------------------------------- # Feature-aware image selection: kitchen-sink image only when features are # requested, bare provisions stay on the light upstream image (no heavier pull). # --------------------------------------------------------------------------- def _run_call(runner: _FakeRunner) -> list[str]: runs = [c for c in runner.calls if c[0] == "run"] assert runs, "no docker run call recorded" return runs[0] @pytest.mark.asyncio async def test_provision_pg_features_uses_kitchen_sink_image() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"1\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision( "dev-pgimg", ["postgres"], features={"postgres": ["vector"]} ) joined = " ".join(_run_call(runner)) assert "roboco-sandbox-pg:latest" in joined assert "postgres:16-alpine" not in joined @pytest.mark.asyncio async def test_provision_pg_bare_uses_light_image() -> None: """Bare pg (no extensions) stays on the light upstream image — no heavier pull.""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-pgbare", ["postgres"]) joined = " ".join(_run_call(runner)) assert "postgres:16-alpine" in joined # The container is named roboco-sandbox-pg- regardless of image, so # check the full image ref (with tag) — the name carries no :latest. assert "roboco-sandbox-pg:latest" not in joined @pytest.mark.asyncio async def test_provision_redis_features_uses_redis_stack_server() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"search\n99999\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision( "dev-redisimg", ["redis"], features={"redis": ["search"]} ) joined = " ".join(_run_call(runner)) assert "redis/redis-stack-server:latest" in joined assert "redis:8-alpine" not in joined @pytest.mark.asyncio async def test_provision_redis_bare_uses_light_image() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) await provisioner.provision("dev-redisbare", ["redis"]) joined = " ".join(_run_call(runner)) assert "redis:8-alpine" in joined assert "redis/redis-stack-server:latest" not in joined def test_engine_image_for_selects_kitchen_sink_iff_features() -> None: """Pure unit check on the registry: bare -> light image, features -> kitchen-sink; mongo (no activatable features) ignores features and returns its base image.""" pg = sandbox_module.SANDBOX_ENGINES["postgres"] assert pg.image_for([]) == "postgres:16-alpine" assert pg.image_for(["vector"]) == "roboco-sandbox-pg:latest" redis = sandbox_module.SANDBOX_ENGINES["redis"] assert redis.image_for([]) == "redis:8-alpine" assert redis.image_for(["search"]) == "redis/redis-stack-server:latest" assert sandbox_module.SANDBOX_ENGINES["mongo"].image_for(["anything"]) == "mongo:8" @pytest.mark.asyncio async def test_provision_rejects_unallowed_pg_extension() -> None: """plpython3u is a superuser-RCE vector — the allowlist rejects it before any container runs (no docker calls at all).""" runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError, match="plpython3u"): await provisioner.provision( "dev-bad", ["postgres"], features={"postgres": ["vector", "plpython3u"]} ) assert not any(c[0] == "run" for c in runner.calls) @pytest.mark.asyncio async def test_provision_rejects_feature_for_unknown_service() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError, match="unknown service"): await provisioner.provision( "dev-x", ["postgres"], features={"mysql": ["vector"]} ) @pytest.mark.asyncio async def test_provision_failed_enable_tears_down_and_raises() -> None: """A failed enable exec (e.g. typo'd module path) is fatal — the agent never receives creds for a db missing what it asked for.""" runner = _FakeRunner(run_rc=0, exec_rc=1) # every exec fails (incl. ready probe) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError): await provisioner.provision( "dev-fail", ["postgres"], features={"postgres": ["vector"]} ) # Teardown attempted on the failed container. assert any(c[0] in ("stop", "kill", "rm") for c in runner.calls) @pytest.mark.asyncio async def test_provision_failed_verify_raises_with_image_hint() -> None: """A successful enable but a verify count that's short (the image is missing the extension files) is fatal with the 'image may be missing' hint.""" # 2 features requested but verify reports only 1 present. runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"1\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) with pytest.raises(SandboxProvisionError, match="missing the extension"): await provisioner.provision( "dev-verify", ["postgres"], features={"postgres": ["vector", "postgis"]} ) @pytest.mark.asyncio async def test_as_payload_surfaces_available_extensions() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) runner.exec_out = b"1\n" provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision( "dev-payload", ["postgres"], features={"postgres": ["vector"]} ) payload = info.as_payload() assert payload["postgres"]["available_extensions"] == ["vector"] assert "available_modules" not in payload["postgres"] @pytest.mark.asyncio async def test_as_payload_no_extensions_key_when_bare() -> None: runner = _FakeRunner(run_rc=0, exec_rc=0) provisioner = SandboxProvisioner(network=_NETWORK, runner=runner) info = await provisioner.provision("dev-bare2", ["postgres"]) payload = info.as_payload() assert "available_extensions" not in payload["postgres"]