"""do_server wires content-tool rejections into the SDK per-verb circuit breaker. Smoke-6 (2026-05-14): be-pm's main-pm hit `note(scope='decision')` with `context: null` 8 times in a row, each returning `incomplete_input`, and the agent kept retrying. The breaker existed on flow_server but not on do_server — content tools had no safety net. These tests mirror test_flow_server_circuit_breaker.py: stub httpx.Client so the orchestrator URL returns the rejection envelope and the SDK URL returns the breaker state. The do_server's _post must call /verb/attempted on rejection kinds (tracing_gap, invalid_state, not_authorized, incomplete_input) and substitute circuit_open when the SDK reports open. """ from __future__ import annotations import importlib import json from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock, patch import pytest if TYPE_CHECKING: import types from pathlib import Path _FULL_MANIFEST = { "agent_id": "00000000-0000-0000-0000-000000000099", "role": "developer", "team": "backend", "workspace_path": "/tmp/test", "flow_tools": [], "do_tools": [ "commit", "note", "dm", "evidence", "progress", "notify", "open_session", "link_session", "notify_list", "notify_get", "notify_ack", "channels", "pr_update", ], "read_tools": ["Read", "Glob", "Grep"], "write_tools": ["Edit", "Write"], "bash_allowed": True, "subagent_allowed": False, "subagent_model": None, "env": {}, } @pytest.fixture() def do_module(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> types.ModuleType: """Import do_server with a tmp manifest + known orchestrator/SDK URLs.""" manifest_path = tmp_path / "tool-manifest.json" manifest_path.write_text(json.dumps(_FULL_MANIFEST)) monkeypatch.setenv("ROBOCO_AGENT_ID", "00000000-0000-0000-0000-000000000099") monkeypatch.setenv("ROBOCO_AGENT_ROLE", "developer") monkeypatch.setenv("ROBOCO_ORCHESTRATOR_URL", "http://test-orchestrator:8000") monkeypatch.setenv("ROBOCO_SDK_URL", "http://test-sdk:9000") monkeypatch.setenv("ROBOCO_TOOL_MANIFEST_PATH", str(manifest_path)) import roboco.mcp.do_server as srv importlib.reload(srv) return srv def _make_client( orchestrator_response: dict[str, Any], sdk_response: dict[str, Any] | None ) -> Any: """Build an httpx.Client mock that dispatches by destination URL.""" captured: list[tuple[str, dict[str, Any] | None]] = [] def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock: client = MagicMock() client.__enter__ = MagicMock(return_value=client) client.__exit__ = MagicMock(return_value=False) def _post(url: str, **kwargs: Any) -> MagicMock: captured.append((url, kwargs.get("json"))) resp = MagicMock() if "test-sdk" in url: if sdk_response is None: raise AssertionError("SDK called unexpectedly") resp.json.return_value = sdk_response else: resp.json.return_value = orchestrator_response return resp client.post.side_effect = _post return client return _client_factory, captured def test_ok_envelope_does_not_touch_sdk(do_module: types.ModuleType) -> None: """An envelope with error=None never POSTs to /verb/attempted.""" factory, captured = _make_client( orchestrator_response={ "status": "noted", "task_id": None, "next": "continue", "error": None, }, sdk_response=None, ) with patch("httpx.Client", side_effect=factory): result = do_module.note(text="hi", scope="note") assert result["error"] is None assert all("test-sdk" not in url for url, _ in captured) def test_incomplete_input_forwards_to_sdk(do_module: types.ModuleType) -> None: """A note rejection (incomplete_input) triggers POST /verb/attempted.""" factory, captured = _make_client( orchestrator_response={ "error": "incomplete_input", "missing": ["context", "chosen", "rationale"], "remediate": "re-issue note(scope='decision', ...)", }, sdk_response={ "verb": "note", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): result = do_module.note(text="...", scope="decision") # Original rejection survives — breaker is not yet open. assert result["error"] == "incomplete_input" sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 sdk_url, sdk_body = sdk_calls[0] assert sdk_url.endswith("/verb/attempted") assert sdk_body == { "verb": "note", "task_id": None, "rejection_kind": "incomplete_input", } @pytest.mark.parametrize( "rejection_kind", ["tracing_gap", "invalid_state", "not_authorized", "incomplete_input"], ) def test_all_counted_rejection_kinds_forwarded( do_module: types.ModuleType, rejection_kind: str ) -> None: """All four counted error kinds forward to the SDK from do_server.""" factory, captured = _make_client( orchestrator_response={ "error": rejection_kind, "message": "no", "remediate": "fix", }, sdk_response={ "verb": "note", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): do_module.note(text="x") sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == rejection_kind def test_breaker_open_substitutes_circuit_open(do_module: types.ModuleType) -> None: """When SDK reports open=true, return the circuit_envelope to the agent. The original fixable rejection is preserved nested as ``inner`` so the agent still sees why the verb failed (#60) — the circuit_open envelope only says the breaker tripped, not the underlying kind/remediate. """ circuit_env = { "error": "circuit_open", "message": "verb 'note' rejected too often (3 in 60s)", "remediate": "fix the missing fields once, then retry", "missing": [], } original = { "error": "incomplete_input", "missing": ["context"], "remediate": "fill context", } factory, _ = _make_client( orchestrator_response=original, sdk_response={ "verb": "note", "task_id": None, "attempts": 3, "limit": 3, "window_seconds": 60, "open": True, "circuit_envelope": circuit_env, }, ) with patch("httpx.Client", side_effect=factory): result = do_module.note(text="x", scope="decision") assert result["error"] == "circuit_open" assert result["remediate"] == "fix the missing fields once, then retry" # Original fixable rejection preserved, not erased. assert result["inner"] == original # SDK envelope not mutated in place. assert "inner" not in circuit_env def test_sdk_unreachable_falls_open(do_module: types.ModuleType) -> None: """If SDK loopback dies, return the original rejection — never break the path.""" import httpx def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock: client = MagicMock() client.__enter__ = MagicMock(return_value=client) client.__exit__ = MagicMock(return_value=False) def _post(url: str, **_kw: Any) -> MagicMock: if "test-sdk" in url: raise httpx.ConnectError("connection refused") resp = MagicMock() resp.json.return_value = { "error": "incomplete_input", "missing": ["context"], "remediate": "fill context", } return resp client.post.side_effect = _post return client with patch("httpx.Client", side_effect=_client_factory): result = do_module.note(text="x", scope="decision") # Original rejection — never circuit_open when the SDK is unreachable. assert result["error"] == "incomplete_input" def test_verb_extracted_from_path(do_module: types.ModuleType) -> None: """commit() reports verb='commit', say() reports verb='say' etc.""" factory, captured = _make_client( orchestrator_response={"error": "invalid_state", "message": "no commits"}, sdk_response={ "verb": "commit", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): do_module.commit(message="[abc12345] test commit message under 20 chars") sdk_body = next(body for url, body in captured if "test-sdk" in url) assert sdk_body["verb"] == "commit" def test_dict_shaped_error_normalized_to_envelope(do_module: types.ModuleType) -> None: """A RobocoError.to_dict()-shaped response must not TypeError the breaker. A dict-shaped `error` is a retry-storm-worthy rejection (the orchestrator's exception handlers surface this shape on 4xx/5xx), so the breaker must count it via the classifier. The dict body is normalized to an Envelope wire format (#232): the agent receives a string `error` kind (not the dict, which violates the Envelope contract) with the message lifted and a remediate. The breaker still counts it (the synthesized string kind is in the counted set). """ factory, captured = _make_client( orchestrator_response={ "error": { "code": "A2A_ACCESS_DENIED", "message": "be-qa cannot A2A with qa-all", "details": {}, } }, sdk_response={ "verb": "dm", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) # No TypeError; the dict-shaped rejection is normalized + forwarded to the SDK. with patch("httpx.Client", side_effect=factory): result = do_module.dm(recipient="qa-all", text="x") # Normalized to a string-kind Envelope — the dict `error` never reaches the # agent (#232). A2A_ACCESS_DENIED maps to not_authorized (exact-code, #161). assert result["error"] == "not_authorized" assert result["message"] == "be-qa cannot A2A with qa-all" assert isinstance(result["remediate"], str) and result["remediate"] assert result["missing"] == [] # SDK breaker MUST still be called so a storm of these counts. sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 # ACCESS_DENIED maps to the not_authorized counted kind. assert sdk_calls[0][1]["rejection_kind"] == "not_authorized" def test_422_validation_failure_counts_as_incomplete_input( do_module: types.ModuleType, ) -> None: """A 422 validation-failure body (`{"detail": [...], "body": ...}`, no `error` field) must count toward the breaker — a storm of 422s is retry-storm-worthy. Mapped to `incomplete_input`. """ factory, captured = _make_client( orchestrator_response={ "detail": [ { "loc": ["body", "text"], "msg": "field required", "type": "value_error.missing", } ], "body": None, }, sdk_response={ "verb": "note", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): do_module.note(text="") sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input" def test_dict_shaped_internal_error_counts_as_invalid_state( do_module: types.ModuleType, ) -> None: """A 500 INTERNAL_ERROR dict-shaped response (generic_exception_handler) must count toward the breaker as `invalid_state` — a storm of 500s is retry-storm-worthy. """ factory, captured = _make_client( orchestrator_response={ "error": { "code": "INTERNAL_ERROR", "message": "An internal error occurred", "details": {"correlation_id": "abc"}, } }, sdk_response={ "verb": "commit", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): do_module.commit(message="[abc12345] a valid commit message here") sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == "invalid_state" def test_dict_shaped_invalid_input_counts_as_incomplete_input( do_module: types.ModuleType, ) -> None: """A dict-shaped INVALID_INPUT (mapped from 422 by http_exception_handler) counts as `incomplete_input` — semantically the agent's input was invalid. """ factory, captured = _make_client( orchestrator_response={ "error": {"code": "INVALID_INPUT", "message": "bad payload"} }, sdk_response={ "verb": "dm", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): do_module.dm(recipient="be-qa", text="x") sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input" # --------------------------------------------------------------------------- # A manifest-registered content tool whose route is missing must return an # envelope rejection (not a raw 404 body) so the breaker counts it. # --------------------------------------------------------------------------- def test_missing_route_404_returns_envelope_and_counts( do_module: types.ModuleType, ) -> None: """A 404 from the orchestrator (manifest-registered tool with no route) must surface as a proper `invalid_state` Envelope rejection — not FastAPI's raw ``{"detail": "Not Found"}`` body — and the breaker must count it. Mirrors flow_server's 404 handling. """ captured: list[tuple[str, dict[str, Any] | None]] = [] def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock: client = MagicMock() client.__enter__ = MagicMock(return_value=client) client.__exit__ = MagicMock(return_value=False) def _post(url: str, **kwargs: Any) -> MagicMock: captured.append((url, kwargs.get("json"))) resp = MagicMock() if "test-sdk" in url: resp.json.return_value = { "verb": "evidence", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, } else: # FastAPI's default 404 for a missing route. resp.status_code = 404 resp.json.return_value = {"detail": "Not Found"} return resp client.post.side_effect = _post return client with patch("httpx.Client", side_effect=_client_factory): result = do_module.evidence(task_id="some-task") # Envelope rejection, not the raw 404 body. assert result["error"] == "invalid_state" assert "remediate" in result assert "detail" not in result # the raw 404 body was not passed through # Breaker was notified so a storm of these trips it. sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 _, sdk_body = sdk_calls[0] assert sdk_body is not None assert sdk_body["rejection_kind"] == "invalid_state" # --------------------------------------------------------------------------- # #232: a non-404 JSON exception-handler body (dict `error` / 422 `detail`) # must be normalized to an Envelope wire format before reaching the agent. # Mirrors flow_server. # --------------------------------------------------------------------------- def test_422_detail_normalized_to_incomplete_input_envelope( do_module: types.ModuleType, ) -> None: """A 422 body (`{"detail": [...]}`, no `error`) is normalized to an Envelope with `error='incomplete_input'`, a non-empty `remediate`, `missing=[]`, and the raw validation `detail` preserved. The breaker still counts it. Mirrors flow_server (#232).""" detail_body = [ {"loc": ["body", "text"], "msg": "field required", "type": "missing"} ] factory, captured = _make_client( orchestrator_response={"detail": detail_body, "body": None}, sdk_response={ "verb": "note", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): result = do_module.note(text="") assert result["error"] == "incomplete_input" assert isinstance(result["remediate"], str) and result["remediate"] assert result["missing"] == [] assert result["detail"] == detail_body sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input" def test_dict_internal_error_normalized_to_invalid_state_envelope( do_module: types.ModuleType, ) -> None: """A dict-shaped INTERNAL_ERROR (generic_exception_handler) is normalized to `error='invalid_state'` with the message lifted + a remediate. Mirrors flow_server (#232).""" factory, captured = _make_client( orchestrator_response={ "error": { "code": "INTERNAL_ERROR", "message": "An internal error occurred", "details": {"correlation_id": "abc"}, } }, sdk_response={ "verb": "commit", "task_id": None, "attempts": 1, "limit": 3, "window_seconds": 60, "open": False, "circuit_envelope": None, }, ) with patch("httpx.Client", side_effect=factory): result = do_module.commit(message="[abc12345] a valid commit message here") assert result["error"] == "invalid_state" assert result["message"] == "An internal error occurred" assert isinstance(result["remediate"], str) and result["remediate"] assert result["missing"] == [] sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url] assert len(sdk_calls) == 1 assert sdk_calls[0][1]["rejection_kind"] == "invalid_state" # --------------------------------------------------------------------------- # #359: the circuit_open substitution lifts task_id/correlation_id from the # original rejection to the top-level envelope. Mirrors flow_server. # --------------------------------------------------------------------------- def test_circuit_open_lifts_task_id_and_correlation_id( do_module: types.ModuleType, ) -> None: """The SDK's circuit_envelope omits task_id/correlation_id; the substitution lifts them from the original rejection to the top level so the agent's envelope contract and ops audit-join still work. Mirrors flow_server (#359).""" circuit_env = { "error": "circuit_open", "message": "verb 'note' rejected too often (3 in 60s)", "remediate": "fix the missing fields once, then retry", "missing": [], } factory, _ = _make_client( orchestrator_response={ "error": "incomplete_input", "missing": ["context"], "remediate": "fill context", "task_id": "T7", "correlation_id": "C7", }, sdk_response={ "verb": "note", "task_id": "T7", "attempts": 3, "limit": 3, "window_seconds": 60, "open": True, "circuit_envelope": circuit_env, }, ) with patch("httpx.Client", side_effect=factory): result = do_module.note(text="x", scope="decision") assert result["error"] == "circuit_open" assert result["task_id"] == "T7" assert result["correlation_id"] == "C7" assert result["inner"]["error"] == "incomplete_input" assert result["inner"]["task_id"] == "T7"