roboco/tests/unit/mcp_servers/test_do_server_circuit_breaker.py

"""do_server wires content-tool rejections into the SDK per-verb circuit breaker.

Smoke-6 (2026-05-14): be-pm's main-pm hit `note(scope='decision')` with
`context: null` 8 times in a row, each returning `incomplete_input`, and
the agent kept retrying. The breaker existed on flow_server but not on
do_server — content tools had no safety net.

These tests mirror test_flow_server_circuit_breaker.py: stub httpx.Client
so the orchestrator URL returns the rejection envelope and the SDK URL
returns the breaker state. The do_server's _post must call /verb/attempted
on rejection kinds (tracing_gap, invalid_state, not_authorized,
incomplete_input) and substitute circuit_open when the SDK reports open.
"""

from __future__ import annotations

import importlib
import json
from typing import TYPE_CHECKING, Any
from unittest.mock import MagicMock, patch

import pytest

if TYPE_CHECKING:
    import types
    from pathlib import Path


_FULL_MANIFEST = {
    "agent_id": "00000000-0000-0000-0000-000000000099",
    "role": "developer",
    "team": "backend",
    "workspace_path": "/tmp/test",
    "flow_tools": [],
    "do_tools": [
        "commit",
        "note",
        "dm",
        "evidence",
        "progress",
        "notify",
        "open_session",
        "link_session",
        "notify_list",
        "notify_get",
        "notify_ack",
        "channels",
        "pr_update",
    ],
    "read_tools": ["Read", "Glob", "Grep"],
    "write_tools": ["Edit", "Write"],
    "bash_allowed": True,
    "subagent_allowed": False,
    "subagent_model": None,
    "env": {},
}


@pytest.fixture()
def do_module(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> types.ModuleType:
    """Import do_server with a tmp manifest + known orchestrator/SDK URLs."""
    manifest_path = tmp_path / "tool-manifest.json"
    manifest_path.write_text(json.dumps(_FULL_MANIFEST))

    monkeypatch.setenv("ROBOCO_AGENT_ID", "00000000-0000-0000-0000-000000000099")
    monkeypatch.setenv("ROBOCO_AGENT_ROLE", "developer")
    monkeypatch.setenv("ROBOCO_ORCHESTRATOR_URL", "http://test-orchestrator:8000")
    monkeypatch.setenv("ROBOCO_SDK_URL", "http://test-sdk:9000")
    monkeypatch.setenv("ROBOCO_TOOL_MANIFEST_PATH", str(manifest_path))

    import roboco.mcp.do_server as srv

    importlib.reload(srv)
    return srv


def _make_client(
    orchestrator_response: dict[str, Any], sdk_response: dict[str, Any] | None
) -> Any:
    """Build an httpx.Client mock that dispatches by destination URL."""
    captured: list[tuple[str, dict[str, Any] | None]] = []

    def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
        client = MagicMock()
        client.__enter__ = MagicMock(return_value=client)
        client.__exit__ = MagicMock(return_value=False)

        def _post(url: str, **kwargs: Any) -> MagicMock:
            captured.append((url, kwargs.get("json")))
            resp = MagicMock()
            if "test-sdk" in url:
                if sdk_response is None:
                    raise AssertionError("SDK called unexpectedly")
                resp.json.return_value = sdk_response
            else:
                resp.json.return_value = orchestrator_response
            return resp

        client.post.side_effect = _post
        return client

    return _client_factory, captured


def test_ok_envelope_does_not_touch_sdk(do_module: types.ModuleType) -> None:
    """An envelope with error=None never POSTs to /verb/attempted."""
    factory, captured = _make_client(
        orchestrator_response={
            "status": "noted",
            "task_id": None,
            "next": "continue",
            "error": None,
        },
        sdk_response=None,
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.note(text="hi", scope="note")
    assert result["error"] is None
    assert all("test-sdk" not in url for url, _ in captured)


def test_incomplete_input_forwards_to_sdk(do_module: types.ModuleType) -> None:
    """A note rejection (incomplete_input) triggers POST /verb/attempted."""
    factory, captured = _make_client(
        orchestrator_response={
            "error": "incomplete_input",
            "missing": ["context", "chosen", "rationale"],
            "remediate": "re-issue note(scope='decision', ...)",
        },
        sdk_response={
            "verb": "note",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.note(text="...", scope="decision")
    # Original rejection survives — breaker is not yet open.
    assert result["error"] == "incomplete_input"
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    sdk_url, sdk_body = sdk_calls[0]
    assert sdk_url.endswith("/verb/attempted")
    assert sdk_body == {
        "verb": "note",
        "task_id": None,
        "rejection_kind": "incomplete_input",
    }


@pytest.mark.parametrize(
    "rejection_kind",
    ["tracing_gap", "invalid_state", "not_authorized", "incomplete_input"],
)
def test_all_counted_rejection_kinds_forwarded(
    do_module: types.ModuleType, rejection_kind: str
) -> None:
    """All four counted error kinds forward to the SDK from do_server."""
    factory, captured = _make_client(
        orchestrator_response={
            "error": rejection_kind,
            "message": "no",
            "remediate": "fix",
        },
        sdk_response={
            "verb": "note",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        do_module.note(text="x")
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == rejection_kind


def test_breaker_open_substitutes_circuit_open(do_module: types.ModuleType) -> None:
    """When SDK reports open=true, return the circuit_envelope to the agent.

    The original fixable rejection is preserved nested as ``inner`` so the
    agent still sees why the verb failed (#60) — the circuit_open envelope
    only says the breaker tripped, not the underlying kind/remediate.
    """
    circuit_env = {
        "error": "circuit_open",
        "message": "verb 'note' rejected too often (3 in 60s)",
        "remediate": "fix the missing fields once, then retry",
        "missing": [],
    }
    original = {
        "error": "incomplete_input",
        "missing": ["context"],
        "remediate": "fill context",
    }
    factory, _ = _make_client(
        orchestrator_response=original,
        sdk_response={
            "verb": "note",
            "task_id": None,
            "attempts": 3,
            "limit": 3,
            "window_seconds": 60,
            "open": True,
            "circuit_envelope": circuit_env,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.note(text="x", scope="decision")
    assert result["error"] == "circuit_open"
    assert result["remediate"] == "fix the missing fields once, then retry"
    # Original fixable rejection preserved, not erased.
    assert result["inner"] == original
    # SDK envelope not mutated in place.
    assert "inner" not in circuit_env


def test_sdk_unreachable_falls_open(do_module: types.ModuleType) -> None:
    """If SDK loopback dies, return the original rejection — never break the path."""
    import httpx

    def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
        client = MagicMock()
        client.__enter__ = MagicMock(return_value=client)
        client.__exit__ = MagicMock(return_value=False)

        def _post(url: str, **_kw: Any) -> MagicMock:
            if "test-sdk" in url:
                raise httpx.ConnectError("connection refused")
            resp = MagicMock()
            resp.json.return_value = {
                "error": "incomplete_input",
                "missing": ["context"],
                "remediate": "fill context",
            }
            return resp

        client.post.side_effect = _post
        return client

    with patch("httpx.Client", side_effect=_client_factory):
        result = do_module.note(text="x", scope="decision")
    # Original rejection — never circuit_open when the SDK is unreachable.
    assert result["error"] == "incomplete_input"


def test_verb_extracted_from_path(do_module: types.ModuleType) -> None:
    """commit() reports verb='commit', say() reports verb='say' etc."""
    factory, captured = _make_client(
        orchestrator_response={"error": "invalid_state", "message": "no commits"},
        sdk_response={
            "verb": "commit",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        do_module.commit(message="[abc12345] test commit message under 20 chars")
    sdk_body = next(body for url, body in captured if "test-sdk" in url)
    assert sdk_body["verb"] == "commit"


def test_dict_shaped_error_normalized_to_envelope(do_module: types.ModuleType) -> None:
    """A RobocoError.to_dict()-shaped response must not TypeError the breaker.

    A dict-shaped `error` is a retry-storm-worthy rejection (the orchestrator's
    exception handlers surface this shape on 4xx/5xx), so the breaker must count
    it via the classifier. The dict body is normalized to an Envelope wire format
    (#232): the agent receives a string `error` kind (not the dict, which violates
    the Envelope contract) with the message lifted and a remediate. The breaker
    still counts it (the synthesized string kind is in the counted set).
    """
    factory, captured = _make_client(
        orchestrator_response={
            "error": {
                "code": "A2A_ACCESS_DENIED",
                "message": "be-qa cannot A2A with qa-all",
                "details": {},
            }
        },
        sdk_response={
            "verb": "dm",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    # No TypeError; the dict-shaped rejection is normalized + forwarded to the SDK.
    with patch("httpx.Client", side_effect=factory):
        result = do_module.dm(recipient="qa-all", text="x")
    # Normalized to a string-kind Envelope — the dict `error` never reaches the
    # agent (#232). A2A_ACCESS_DENIED maps to not_authorized (exact-code, #161).
    assert result["error"] == "not_authorized"
    assert result["message"] == "be-qa cannot A2A with qa-all"
    assert isinstance(result["remediate"], str) and result["remediate"]
    assert result["missing"] == []
    # SDK breaker MUST still be called so a storm of these counts.
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    # ACCESS_DENIED maps to the not_authorized counted kind.
    assert sdk_calls[0][1]["rejection_kind"] == "not_authorized"


def test_422_validation_failure_counts_as_incomplete_input(
    do_module: types.ModuleType,
) -> None:
    """A 422 validation-failure body (`{"detail": [...], "body": ...}`, no `error`
    field) must count toward the breaker — a storm of 422s is retry-storm-worthy.
    Mapped to `incomplete_input`.
    """
    factory, captured = _make_client(
        orchestrator_response={
            "detail": [
                {
                    "loc": ["body", "text"],
                    "msg": "field required",
                    "type": "value_error.missing",
                }
            ],
            "body": None,
        },
        sdk_response={
            "verb": "note",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        do_module.note(text="")
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"


def test_dict_shaped_internal_error_counts_as_invalid_state(
    do_module: types.ModuleType,
) -> None:
    """A 500 INTERNAL_ERROR dict-shaped response (generic_exception_handler) must
    count toward the breaker as `invalid_state` — a storm of 500s is retry-storm-worthy.
    """
    factory, captured = _make_client(
        orchestrator_response={
            "error": {
                "code": "INTERNAL_ERROR",
                "message": "An internal error occurred",
                "details": {"correlation_id": "abc"},
            }
        },
        sdk_response={
            "verb": "commit",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        do_module.commit(message="[abc12345] a valid commit message here")
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == "invalid_state"


def test_dict_shaped_invalid_input_counts_as_incomplete_input(
    do_module: types.ModuleType,
) -> None:
    """A dict-shaped INVALID_INPUT (mapped from 422 by http_exception_handler)
    counts as `incomplete_input` — semantically the agent's input was invalid.
    """
    factory, captured = _make_client(
        orchestrator_response={
            "error": {"code": "INVALID_INPUT", "message": "bad payload"}
        },
        sdk_response={
            "verb": "dm",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        do_module.dm(recipient="be-qa", text="x")
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"


# ---------------------------------------------------------------------------
# A manifest-registered content tool whose route is missing must return an
# envelope rejection (not a raw 404 body) so the breaker counts it.
# ---------------------------------------------------------------------------


def test_missing_route_404_returns_envelope_and_counts(
    do_module: types.ModuleType,
) -> None:
    """A 404 from the orchestrator (manifest-registered tool with no route) must
    surface as a proper `invalid_state` Envelope rejection — not FastAPI's raw
    ``{"detail": "Not Found"}`` body — and the breaker must count it. Mirrors
    flow_server's 404 handling.
    """
    captured: list[tuple[str, dict[str, Any] | None]] = []

    def _client_factory(*_args: Any, **_kwargs: Any) -> MagicMock:
        client = MagicMock()
        client.__enter__ = MagicMock(return_value=client)
        client.__exit__ = MagicMock(return_value=False)

        def _post(url: str, **kwargs: Any) -> MagicMock:
            captured.append((url, kwargs.get("json")))
            resp = MagicMock()
            if "test-sdk" in url:
                resp.json.return_value = {
                    "verb": "evidence",
                    "task_id": None,
                    "attempts": 1,
                    "limit": 3,
                    "window_seconds": 60,
                    "open": False,
                    "circuit_envelope": None,
                }
            else:
                # FastAPI's default 404 for a missing route.
                resp.status_code = 404
                resp.json.return_value = {"detail": "Not Found"}
            return resp

        client.post.side_effect = _post
        return client

    with patch("httpx.Client", side_effect=_client_factory):
        result = do_module.evidence(task_id="some-task")
    # Envelope rejection, not the raw 404 body.
    assert result["error"] == "invalid_state"
    assert "remediate" in result
    assert "detail" not in result  # the raw 404 body was not passed through
    # Breaker was notified so a storm of these trips it.
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    _, sdk_body = sdk_calls[0]
    assert sdk_body is not None
    assert sdk_body["rejection_kind"] == "invalid_state"


# ---------------------------------------------------------------------------
# #232: a non-404 JSON exception-handler body (dict `error` / 422 `detail`)
# must be normalized to an Envelope wire format before reaching the agent.
# Mirrors flow_server.
# ---------------------------------------------------------------------------


def test_422_detail_normalized_to_incomplete_input_envelope(
    do_module: types.ModuleType,
) -> None:
    """A 422 body (`{"detail": [...]}`, no `error`) is normalized to an Envelope
    with `error='incomplete_input'`, a non-empty `remediate`, `missing=[]`, and
    the raw validation `detail` preserved. The breaker still counts it. Mirrors
    flow_server (#232)."""
    detail_body = [
        {"loc": ["body", "text"], "msg": "field required", "type": "missing"}
    ]
    factory, captured = _make_client(
        orchestrator_response={"detail": detail_body, "body": None},
        sdk_response={
            "verb": "note",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.note(text="")
    assert result["error"] == "incomplete_input"
    assert isinstance(result["remediate"], str) and result["remediate"]
    assert result["missing"] == []
    assert result["detail"] == detail_body
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == "incomplete_input"


def test_dict_internal_error_normalized_to_invalid_state_envelope(
    do_module: types.ModuleType,
) -> None:
    """A dict-shaped INTERNAL_ERROR (generic_exception_handler) is normalized to
    `error='invalid_state'` with the message lifted + a remediate. Mirrors
    flow_server (#232)."""
    factory, captured = _make_client(
        orchestrator_response={
            "error": {
                "code": "INTERNAL_ERROR",
                "message": "An internal error occurred",
                "details": {"correlation_id": "abc"},
            }
        },
        sdk_response={
            "verb": "commit",
            "task_id": None,
            "attempts": 1,
            "limit": 3,
            "window_seconds": 60,
            "open": False,
            "circuit_envelope": None,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.commit(message="[abc12345] a valid commit message here")
    assert result["error"] == "invalid_state"
    assert result["message"] == "An internal error occurred"
    assert isinstance(result["remediate"], str) and result["remediate"]
    assert result["missing"] == []
    sdk_calls = [(url, body) for url, body in captured if "test-sdk" in url]
    assert len(sdk_calls) == 1
    assert sdk_calls[0][1]["rejection_kind"] == "invalid_state"


# ---------------------------------------------------------------------------
# #359: the circuit_open substitution lifts task_id/correlation_id from the
# original rejection to the top-level envelope. Mirrors flow_server.
# ---------------------------------------------------------------------------


def test_circuit_open_lifts_task_id_and_correlation_id(
    do_module: types.ModuleType,
) -> None:
    """The SDK's circuit_envelope omits task_id/correlation_id; the substitution
    lifts them from the original rejection to the top level so the agent's
    envelope contract and ops audit-join still work. Mirrors flow_server (#359)."""
    circuit_env = {
        "error": "circuit_open",
        "message": "verb 'note' rejected too often (3 in 60s)",
        "remediate": "fix the missing fields once, then retry",
        "missing": [],
    }
    factory, _ = _make_client(
        orchestrator_response={
            "error": "incomplete_input",
            "missing": ["context"],
            "remediate": "fill context",
            "task_id": "T7",
            "correlation_id": "C7",
        },
        sdk_response={
            "verb": "note",
            "task_id": "T7",
            "attempts": 3,
            "limit": 3,
            "window_seconds": 60,
            "open": True,
            "circuit_envelope": circuit_env,
        },
    )
    with patch("httpx.Client", side_effect=factory):
        result = do_module.note(text="x", scope="decision")
    assert result["error"] == "circuit_open"
    assert result["task_id"] == "T7"
    assert result["correlation_id"] == "C7"
    assert result["inner"]["error"] == "incomplete_input"
    assert result["inner"]["task_id"] == "T7"