Files
roboco/tests/unit/mcp_servers/test_do_server_circuit_breaker.py
T
Renn F 7cb00611e1 chore(comms): finalize #306 teardown — changelog + purge dangling refs
#306 removed the channels/sessions/messages subsystem but left dangling
references. Agents were still told to call removed verbs at spawn, and a
maintenance script referenced a dropped table.

- prompts (roles/identities/teams/base): drop say/open_session/link_session/
  channels() and the dead Channels sections; comms rows now teach A2A
  (dm + read_a2a); renumber the PM workflow steps the removed open_session
  step left behind
- scripts/reset_runtime_state.sql: drop the dropped chunks_conversations table
- models/events.py: mark the retired SESSION_*/MESSAGE_SENT enum members inert
- mcp/{do,flow}_server.py: drop the dead SESSION_CLOSED error-map key
- panel: drop channels_read/write from AgentPermissions; remove dead channel:
  KB source branch
- pyproject.toml: refresh a ruff-exemption example off the removed verb kwargs
- CHANGELOG: record #306 under [Unreleased]
2026-07-04 03:42:04 +02:00

584 lines
21 KiB
Python

"""do_server wires content-tool rejections into the SDK per-verb circuit breaker.
Smoke-6 (2026-05-14): be-pm's main-pm hit `note(scope='decision')` with
`context: null` 8 times in a row, each returning `incomplete_input`, and
the agent kept retrying. The breaker existed on flow_server but not on
do_server — content tools had no safety net.
These tests mirror test_flow_server_circuit_breaker.py: stub httpx.Client
so the orchestrator URL returns the rejection envelope and the SDK URL
returns the breaker state. The do_server's _post must call /verb/attempted
on rejection kinds (tracing_gap, invalid_state, not_authorized,
incomplete_input) and substitute circuit_open when the SDK reports open.
"""
from __future__ import annotations
import importlib
import json
from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, patch
import pytest
if TYPE_CHECKING:
import types
from pathlib import Path
_FULL_MANIFEST = {
"agent_id": "00000000-0000-0000-0000-000000000099",
"role": "developer",
"team": "backend",
"workspace_path": "/tmp/test",
"flow_tools": [],
"do_tools": [
"commit",
"note",
"dm",
"evidence",
"progress",
"notify",
"notify_list",
"notify_get",
"notify_ack",
"pr_update",
],
"read_tools": ["Read", "Glob", "Grep"],
"write_tools": ["Edit", "Write"],
"bash_allowed": True,
"subagent_allowed": False,
"subagent_model": None,
"env": {},
}
@pytest.fixture()
def do_module(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> types.ModuleType:
"""Import do_server with a tmp manifest + known orchestrator/SDK URLs."""
manifest_path = tmp_path / "tool-manifest.json"
manifest_path.write_text(json.dumps(_FULL_MANIFEST))
monkeypatch.setenv("ROBOCO_AGENT_ID", "00000000-0000-0000-0000-000000000099")
monkeypatch.setenv("ROBOCO_AGENT_ROLE", "developer")
monkeypatch.setenv("ROBOCO_ORCHESTRATOR_URL", "http://test-orchestrator:8000")
monkeypatch.setenv("ROBOCO_SDK_URL", "http://test-sdk:9000")
monkeypatch.setenv("ROBOCO_TOOL_MANIFEST_PATH", str(manifest_path))
import roboco.mcp.do_server as srv
importlib.reload(srv)
return srv
def _make_client(
orchestrator_response: dict[str, Any], sdk_response: dict[str, Any] | None
) -> Any:
"""Build an httpx.Client mock that dispatches by destination URL."""
captured: list[tuple[str, dict[str, Any] | None]] = []
def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
client = MagicMock()
client.__enter__ = MagicMock(return_value=client)
client.__exit__ = MagicMock(return_value=False)
def _post(url: str, **kwargs: Any) -> MagicMock:
captured.append((url, kwargs.get("json")))
resp = MagicMock()
if "test-sdk" in url:
if sdk_response is None:
raise AssertionError("SDK called unexpectedly")
resp.json.return_value = sdk_response
else:
resp.json.return_value = orchestrator_response
return resp
client.post.side_effect = _post
return client
return _client_factory, captured
def test_ok_envelope_does_not_touch_sdk(do_module: types.ModuleType) -> None:
"""An envelope with error=None never POSTs to /verb/attempted."""
factory, captured = _make_client(
orchestrator_response={
"status": "noted",
"task_id": None,
"next": "continue",
"error": None,
},
sdk_response=None,
)
with patch("httpx.Client", side_effect=factory):
result = do_module.note(text="hi", scope="note")
assert result["error"] is None
assert all("test-sdk" not in url for url, _ in captured)
def test_incomplete_input_forwards_to_sdk(do_module: types.ModuleType) -> None:
"""A note rejection (incomplete_input) triggers POST /verb/attempted."""
factory, captured = _make_client(
orchestrator_response={
"error": "incomplete_input",
"missing": ["context", "chosen", "rationale"],
"remediate": "re-issue note(scope='decision', ...)",
},
sdk_response={
"verb": "note",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
result = do_module.note(text="...", scope="decision")
# Original rejection survives — breaker is not yet open.
assert result["error"] == "incomplete_input"
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
sdk_url, sdk_body = sdk_calls[0]
assert sdk_url.endswith("/verb/attempted")
assert sdk_body == {
"verb": "note",
"task_id": None,
"rejection_kind": "incomplete_input",
}
@pytest.mark.parametrize(
"rejection_kind",
["tracing_gap", "invalid_state", "not_authorized", "incomplete_input"],
)
def test_all_counted_rejection_kinds_forwarded(
do_module: types.ModuleType, rejection_kind: str
) -> None:
"""All four counted error kinds forward to the SDK from do_server."""
factory, captured = _make_client(
orchestrator_response={
"error": rejection_kind,
"message": "no",
"remediate": "fix",
},
sdk_response={
"verb": "note",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
do_module.note(text="x")
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == rejection_kind
def test_breaker_open_substitutes_circuit_open(do_module: types.ModuleType) -> None:
"""When SDK reports open=true, return the circuit_envelope to the agent.
The original fixable rejection is preserved nested as ``inner`` so the
agent still sees why the verb failed (#60) — the circuit_open envelope
only says the breaker tripped, not the underlying kind/remediate.
"""
circuit_env = {
"error": "circuit_open",
"message": "verb 'note' rejected too often (3 in 60s)",
"remediate": "fix the missing fields once, then retry",
"missing": [],
}
original = {
"error": "incomplete_input",
"missing": ["context"],
"remediate": "fill context",
}
factory, _ = _make_client(
orchestrator_response=original,
sdk_response={
"verb": "note",
"task_id": None,
"attempts": 3,
"limit": 3,
"window_seconds": 60,
"open": True,
"circuit_envelope": circuit_env,
},
)
with patch("httpx.Client", side_effect=factory):
result = do_module.note(text="x", scope="decision")
assert result["error"] == "circuit_open"
assert result["remediate"] == "fix the missing fields once, then retry"
# Original fixable rejection preserved, not erased.
assert result["inner"] == original
# SDK envelope not mutated in place.
assert "inner" not in circuit_env
def test_sdk_unreachable_falls_open(do_module: types.ModuleType) -> None:
"""If SDK loopback dies, return the original rejection — never break the path."""
import httpx
def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
client = MagicMock()
client.__enter__ = MagicMock(return_value=client)
client.__exit__ = MagicMock(return_value=False)
def _post(url: str, **_kw: Any) -> MagicMock:
if "test-sdk" in url:
raise httpx.ConnectError("connection refused")
resp = MagicMock()
resp.json.return_value = {
"error": "incomplete_input",
"missing": ["context"],
"remediate": "fill context",
}
return resp
client.post.side_effect = _post
return client
with patch("httpx.Client", side_effect=_client_factory):
result = do_module.note(text="x", scope="decision")
# Original rejection — never circuit_open when the SDK is unreachable.
assert result["error"] == "incomplete_input"
def test_verb_extracted_from_path(do_module: types.ModuleType) -> None:
"""commit() reports verb='commit', say() reports verb='say' etc."""
factory, captured = _make_client(
orchestrator_response={"error": "invalid_state", "message": "no commits"},
sdk_response={
"verb": "commit",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
do_module.commit(message="[abc12345] test commit message under 20 chars")
sdk_body = next(body for url, body in captured if "test-sdk" in url)
assert sdk_body["verb"] == "commit"
def test_dict_shaped_error_normalized_to_envelope(do_module: types.ModuleType) -> None:
"""A RobocoError.to_dict()-shaped response must not TypeError the breaker.
A dict-shaped `error` is a retry-storm-worthy rejection (the orchestrator's
exception handlers surface this shape on 4xx/5xx), so the breaker must count
it via the classifier. The dict body is normalized to an Envelope wire format
(#232): the agent receives a string `error` kind (not the dict, which violates
the Envelope contract) with the message lifted and a remediate. The breaker
still counts it (the synthesized string kind is in the counted set).
"""
factory, captured = _make_client(
orchestrator_response={
"error": {
"code": "A2A_ACCESS_DENIED",
"message": "be-qa cannot A2A with qa-all",
"details": {},
}
},
sdk_response={
"verb": "dm",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
# No TypeError; the dict-shaped rejection is normalized + forwarded to the SDK.
with patch("httpx.Client", side_effect=factory):
result = do_module.dm(recipient="qa-all", text="x")
# Normalized to a string-kind Envelope — the dict `error` never reaches the
# agent (#232). A2A_ACCESS_DENIED maps to not_authorized (exact-code, #161).
assert result["error"] == "not_authorized"
assert result["message"] == "be-qa cannot A2A with qa-all"
assert isinstance(result["remediate"], str) and result["remediate"]
assert result["missing"] == []
# SDK breaker MUST still be called so a storm of these counts.
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
# ACCESS_DENIED maps to the not_authorized counted kind.
assert sdk_calls[0][1]["rejection_kind"] == "not_authorized"
def test_422_validation_failure_counts_as_incomplete_input(
do_module: types.ModuleType,
) -> None:
"""A 422 validation-failure body (`{"detail": [...], "body": ...}`, no `error`
field) must count toward the breaker — a storm of 422s is retry-storm-worthy.
Mapped to `incomplete_input`.
"""
factory, captured = _make_client(
orchestrator_response={
"detail": [
{
"loc": ["body", "text"],
"msg": "field required",
"type": "value_error.missing",
}
],
"body": None,
},
sdk_response={
"verb": "note",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
do_module.note(text="")
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"
def test_dict_shaped_internal_error_counts_as_invalid_state(
do_module: types.ModuleType,
) -> None:
"""A 500 INTERNAL_ERROR dict-shaped response (generic_exception_handler) must
count toward the breaker as `invalid_state` — a storm of 500s is retry-storm-worthy.
"""
factory, captured = _make_client(
orchestrator_response={
"error": {
"code": "INTERNAL_ERROR",
"message": "An internal error occurred",
"details": {"correlation_id": "abc"},
}
},
sdk_response={
"verb": "commit",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
do_module.commit(message="[abc12345] a valid commit message here")
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == "invalid_state"
def test_dict_shaped_invalid_input_counts_as_incomplete_input(
do_module: types.ModuleType,
) -> None:
"""A dict-shaped INVALID_INPUT (mapped from 422 by http_exception_handler)
counts as `incomplete_input` — semantically the agent's input was invalid.
"""
factory, captured = _make_client(
orchestrator_response={
"error": {"code": "INVALID_INPUT", "message": "bad payload"}
},
sdk_response={
"verb": "dm",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
do_module.dm(recipient="be-qa", text="x")
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"
# ---------------------------------------------------------------------------
# A manifest-registered content tool whose route is missing must return an
# envelope rejection (not a raw 404 body) so the breaker counts it.
# ---------------------------------------------------------------------------
def test_missing_route_404_returns_envelope_and_counts(
do_module: types.ModuleType,
) -> None:
"""A 404 from the orchestrator (manifest-registered tool with no route) must
surface as a proper `invalid_state` Envelope rejection — not FastAPI's raw
``{"detail": "Not Found"}`` body — and the breaker must count it. Mirrors
flow_server's 404 handling.
"""
captured: list[tuple[str, dict[str, Any] | None]] = []
def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
client = MagicMock()
client.__enter__ = MagicMock(return_value=client)
client.__exit__ = MagicMock(return_value=False)
def _post(url: str, **kwargs: Any) -> MagicMock:
captured.append((url, kwargs.get("json")))
resp = MagicMock()
if "test-sdk" in url:
resp.json.return_value = {
"verb": "evidence",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
}
else:
# FastAPI's default 404 for a missing route.
resp.status_code = 404
resp.json.return_value = {"detail": "Not Found"}
return resp
client.post.side_effect = _post
return client
with patch("httpx.Client", side_effect=_client_factory):
result = do_module.evidence(task_id="some-task")
# Envelope rejection, not the raw 404 body.
assert result["error"] == "invalid_state"
assert "remediate" in result
assert "detail" not in result # the raw 404 body was not passed through
# Breaker was notified so a storm of these trips it.
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
_, sdk_body = sdk_calls[0]
assert sdk_body is not None
assert sdk_body["rejection_kind"] == "invalid_state"
# ---------------------------------------------------------------------------
# #232: a non-404 JSON exception-handler body (dict `error` / 422 `detail`)
# must be normalized to an Envelope wire format before reaching the agent.
# Mirrors flow_server.
# ---------------------------------------------------------------------------
def test_422_detail_normalized_to_incomplete_input_envelope(
do_module: types.ModuleType,
) -> None:
"""A 422 body (`{"detail": [...]}`, no `error`) is normalized to an Envelope
with `error='incomplete_input'`, a non-empty `remediate`, `missing=[]`, and
the raw validation `detail` preserved. The breaker still counts it. Mirrors
flow_server (#232)."""
detail_body = [
{"loc": ["body", "text"], "msg": "field required", "type": "missing"}
]
factory, captured = _make_client(
orchestrator_response={"detail": detail_body, "body": None},
sdk_response={
"verb": "note",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
result = do_module.note(text="")
assert result["error"] == "incomplete_input"
assert isinstance(result["remediate"], str) and result["remediate"]
assert result["missing"] == []
assert result["detail"] == detail_body
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"
def test_dict_internal_error_normalized_to_invalid_state_envelope(
do_module: types.ModuleType,
) -> None:
"""A dict-shaped INTERNAL_ERROR (generic_exception_handler) is normalized to
`error='invalid_state'` with the message lifted + a remediate. Mirrors
flow_server (#232)."""
factory, captured = _make_client(
orchestrator_response={
"error": {
"code": "INTERNAL_ERROR",
"message": "An internal error occurred",
"details": {"correlation_id": "abc"},
}
},
sdk_response={
"verb": "commit",
"task_id": None,
"attempts": 1,
"limit": 3,
"window_seconds": 60,
"open": False,
"circuit_envelope": None,
},
)
with patch("httpx.Client", side_effect=factory):
result = do_module.commit(message="[abc12345] a valid commit message here")
assert result["error"] == "invalid_state"
assert result["message"] == "An internal error occurred"
assert isinstance(result["remediate"], str) and result["remediate"]
assert result["missing"] == []
sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
assert len(sdk_calls) == 1
assert sdk_calls[0][1]["rejection_kind"] == "invalid_state"
# ---------------------------------------------------------------------------
# #359: the circuit_open substitution lifts task_id/correlation_id from the
# original rejection to the top-level envelope. Mirrors flow_server.
# ---------------------------------------------------------------------------
def test_circuit_open_lifts_task_id_and_correlation_id(
do_module: types.ModuleType,
) -> None:
"""The SDK's circuit_envelope omits task_id/correlation_id; the substitution
lifts them from the original rejection to the top level so the agent's
envelope contract and ops audit-join still work. Mirrors flow_server (#359)."""
circuit_env = {
"error": "circuit_open",
"message": "verb 'note' rejected too often (3 in 60s)",
"remediate": "fix the missing fields once, then retry",
"missing": [],
}
factory, _ = _make_client(
orchestrator_response={
"error": "incomplete_input",
"missing": ["context"],
"remediate": "fill context",
"task_id": "T7",
"correlation_id": "C7",
},
sdk_response={
"verb": "note",
"task_id": "T7",
"attempts": 3,
"limit": 3,
"window_seconds": 60,
"open": True,
"circuit_envelope": circuit_env,
},
)
with patch("httpx.Client", side_effect=factory):
result = do_module.note(text="x", scope="decision")
assert result["error"] == "circuit_open"
assert result["task_id"] == "T7"
assert result["correlation_id"] == "C7"
assert result["inner"]["error"] == "incomplete_input"
assert result["inner"]["task_id"] == "T7"