"""Per-verb circuit breaker — agent_sdk denies after N retries in 60s. Phase 3 Task 14. The 2026-05-10 smoke run showed `i_am_done` retried 5+ times in 2 minutes (each rejection was a `tracing_gap`; the agent kept calling). This module verifies the runtime tracker added to `agent_sdk.server` enforces the per-verb cap from `foundation.policy.agent_loop.retry_limit_for`. The tracker is keyed on `(verb, task_id)` and operates over a 60s sliding window. Tests poke time.monotonic via patch to drive window decay deterministically — wallclock sleeps would slow the suite. """ from __future__ import annotations from collections import deque from typing import TYPE_CHECKING from unittest.mock import patch import pytest import roboco.agent_sdk.server as srv from fastapi.testclient import TestClient from roboco.foundation.policy.agent_loop import retry_limit_for from roboco.services.gateway.envelope import Envelope if TYPE_CHECKING: from collections.abc import Iterator # Named constants — ruff PLR2004 forbids magic comparisons. Mirror the # foundation cap for i_am_done so the tests remain readable. _OK = 200 _I_AM_DONE_CAP = 3 # foundation.VERB_RETRY_LIMITS["i_am_done"] @pytest.fixture(autouse=True) def _reset_state() -> Iterator[None]: """Wipe the SDK session state before every test. Helpers share a module-level `_state` singleton, so leakage between tests would silently corrupt counts. """ srv._state.reset() yield srv._state.reset() # --------------------------------------------------------------------------- # Envelope.circuit_open # --------------------------------------------------------------------------- def test_envelope_circuit_open_kind() -> None: """Envelope.circuit_open is a distinct error kind.""" env = Envelope.circuit_open( verb="i_am_done", attempts=4, window_seconds=60, remediate="call i_am_blocked", ) body = env.as_dict() assert body["error"] == "circuit_open" assert body["remediate"] == "call i_am_blocked" assert body["message"] is not None assert "i_am_done" in body["message"] assert "60" in body["message"] assert "4" in body["message"] def test_envelope_circuit_open_carries_briefing() -> None: """Optional context_briefing is preserved when caller supplies it.""" env = Envelope.circuit_open( verb="i_am_done", attempts=4, window_seconds=60, remediate="x", context_briefing={"hint": "blocked"}, ) body = env.as_dict() assert body["context_briefing"] == {"hint": "blocked"} def test_envelope_circuit_open_default_briefing_empty() -> None: """Omitted context_briefing defaults to an empty dict (not None).""" env = Envelope.circuit_open( verb="i_am_done", attempts=3, window_seconds=60, remediate="x" ) body = env.as_dict() assert body["context_briefing"] == {} # --------------------------------------------------------------------------- # Tracker — record + count # --------------------------------------------------------------------------- def test_record_attempt_increments_count_for_same_key() -> None: """Three records for the same (verb, task_id) yield count == 3.""" expected = 3 for _ in range(expected): srv._record_verb_attempt("i_am_done", "task-A") assert srv._verb_attempt_count("i_am_done", "task-A") == expected def test_tracker_keys_per_verb_task_pair() -> None: """The tracker keys on (verb, task_id), not just verb. Records for task-A are independent of records for task-B even when the verb is identical. Without this guarantee, an agent juggling two tasks would trip the breaker on the wrong one. """ a_count = 3 b_count = 1 for _ in range(a_count): srv._record_verb_attempt("i_am_done", "task-A") for _ in range(b_count): srv._record_verb_attempt("i_am_done", "task-B") assert srv._verb_attempt_count("i_am_done", "task-A") == a_count assert srv._verb_attempt_count("i_am_done", "task-B") == b_count def test_tracker_keys_per_verb_independent_of_task() -> None: """Different verbs on the same task are tracked independently.""" done_count = 2 submit_count = 1 for _ in range(done_count): srv._record_verb_attempt("i_am_done", "task-A") for _ in range(submit_count): srv._record_verb_attempt("submit_up", "task-A") assert srv._verb_attempt_count("i_am_done", "task-A") == done_count assert srv._verb_attempt_count("submit_up", "task-A") == submit_count def test_tracker_handles_none_task_id() -> None: """Verbs without a task_id collapse to (verb, None) — still tracked.""" expected = 2 for _ in range(expected): srv._record_verb_attempt("triage", None) assert srv._verb_attempt_count("triage", None) == expected def test_count_is_zero_for_unknown_key() -> None: """Querying a never-recorded key returns 0 (no KeyError).""" assert srv._verb_attempt_count("never_called", "task-X") == 0 # --------------------------------------------------------------------------- # Tracker — window decay # --------------------------------------------------------------------------- def test_window_drops_entries_older_than_60s() -> None: """Attempts older than 60s are pruned from the window on next access. Drives time.monotonic via patch so the test runs in <1ms instead of waiting on wall-clock. """ base = 1000.0 initial = 3 after_jump = 1 with patch("roboco.agent_sdk.server.time.monotonic") as mock_time: mock_time.return_value = base for _ in range(initial): srv._record_verb_attempt("i_am_done", "task-A") assert srv._verb_attempt_count("i_am_done", "task-A") == initial # Jump past the 60s window — old attempts must be dropped. mock_time.return_value = base + 61.0 srv._record_verb_attempt("i_am_done", "task-A") # The 3 old entries are pruned; only the new one remains. assert srv._verb_attempt_count("i_am_done", "task-A") == after_jump def test_window_keeps_entries_within_60s() -> None: """Attempts within the window survive — only ones past 60s drop.""" base = 1000.0 expected = 2 with patch("roboco.agent_sdk.server.time.monotonic") as mock_time: mock_time.return_value = base srv._record_verb_attempt("i_am_done", "task-A") mock_time.return_value = base + 30.0 srv._record_verb_attempt("i_am_done", "task-A") mock_time.return_value = base + 59.0 # Both entries within the 60s window starting at base. assert srv._verb_attempt_count("i_am_done", "task-A") == expected def test_count_prunes_on_read_without_recording() -> None: """_verb_attempt_count prunes the window even when not recording. A read-only check should still see a fresh count after time advances — callers (e.g. _check_verb_circuit) rely on this. """ base = 1000.0 with patch("roboco.agent_sdk.server.time.monotonic") as mock_time: mock_time.return_value = base srv._record_verb_attempt("i_am_done", "task-A") srv._record_verb_attempt("i_am_done", "task-A") mock_time.return_value = base + 61.0 # Count must drop without us recording anything new. assert srv._verb_attempt_count("i_am_done", "task-A") == 0 # --------------------------------------------------------------------------- # _check_verb_circuit — gating logic # --------------------------------------------------------------------------- def test_check_returns_none_when_under_limit() -> None: """Below the cap, _check_verb_circuit returns None — call may proceed.""" srv._record_verb_attempt("i_am_done", "task-A") srv._record_verb_attempt("i_am_done", "task-A") # i_am_done cap is 3; we recorded 2. assert srv._check_verb_circuit("i_am_done", "task-A") is None def test_check_returns_envelope_at_or_above_limit() -> None: """At the cap, _check_verb_circuit returns a circuit_open envelope dict.""" limit = retry_limit_for("i_am_done") assert limit is not None for _ in range(limit): srv._record_verb_attempt("i_am_done", "task-A") result = srv._check_verb_circuit("i_am_done", "task-A") assert result is not None assert result["error"] == "circuit_open" assert "i_am_done" in result["message"] assert result["remediate"] is not None assert "i_am_blocked" in result["remediate"] def test_check_returns_none_for_unlimited_retry_verbs() -> None: """give_me_work / triage / evidence never trip the breaker.""" assert retry_limit_for("give_me_work") is None # Even after many recorded attempts, no envelope is returned. for _ in range(20): srv._record_verb_attempt("give_me_work", None) assert srv._check_verb_circuit("give_me_work", None) is None def test_check_uses_default_cap_for_unknown_verb() -> None: """Unknown verbs fall back to BudgetPolicy.verb_retry_max_per_minute.""" limit = retry_limit_for("not_a_real_verb") assert limit is not None # default cap applies for _ in range(limit): srv._record_verb_attempt("not_a_real_verb", "task-A") assert srv._check_verb_circuit("not_a_real_verb", "task-A") is not None # --------------------------------------------------------------------------- # /verb/attempted endpoint # --------------------------------------------------------------------------- def test_verb_attempted_endpoint_records_rejection() -> None: """POST /verb/attempted with a counted rejection_kind increments the window.""" client = TestClient(srv.app) resp = client.post( "/verb/attempted", json={ "verb": "i_am_done", "task_id": "task-A", "rejection_kind": "tracing_gap", }, ) assert resp.status_code == _OK body = resp.json() assert body["verb"] == "i_am_done" assert body["task_id"] == "task-A" assert body["attempts"] == 1 assert body["limit"] == retry_limit_for("i_am_done") assert body["open"] is False assert body["circuit_envelope"] is None def test_verb_attempted_endpoint_ignores_uncounted_kinds() -> None: """A non-rejection (e.g. ok-related) kind doesn't move the counter.""" client = TestClient(srv.app) resp = client.post( "/verb/attempted", json={ "verb": "i_am_done", "task_id": "task-A", "rejection_kind": "ok", # not in the counted set }, ) assert resp.status_code == _OK body = resp.json() assert body["attempts"] == 0 assert body["open"] is False def test_verb_attempted_endpoint_opens_circuit_at_threshold() -> None: """After `limit` rejections in 60s the endpoint reports open=True with envelope.""" client = TestClient(srv.app) limit = retry_limit_for("i_am_done") assert limit is not None # Hit the cap. last_body: dict[str, object] | None = None for _ in range(limit): resp = client.post( "/verb/attempted", json={ "verb": "i_am_done", "task_id": "task-A", "rejection_kind": "tracing_gap", }, ) assert resp.status_code == _OK last_body = resp.json() assert last_body is not None assert last_body["attempts"] == limit assert last_body["open"] is True env = last_body["circuit_envelope"] assert isinstance(env, dict) assert env["error"] == "circuit_open" assert "i_am_done" in env["message"] def test_verb_attempted_endpoint_unlimited_verb_never_opens() -> None: """give_me_work bypasses the breaker even after many rejections.""" client = TestClient(srv.app) for _ in range(10): resp = client.post( "/verb/attempted", json={ "verb": "give_me_work", "task_id": None, "rejection_kind": "tracing_gap", }, ) assert resp.status_code == _OK body = resp.json() assert body["limit"] is None assert body["open"] is False assert body["circuit_envelope"] is None def test_verb_circuit_status_endpoint_does_not_record() -> None: """GET /verb/circuit_status reads state without incrementing.""" client = TestClient(srv.app) # Record one rejection via the POST endpoint. client.post( "/verb/attempted", json={ "verb": "i_am_done", "task_id": "task-A", "rejection_kind": "tracing_gap", }, ) # Now poll status repeatedly — count must stay at 1. for _ in range(5): resp = client.get( "/verb/circuit_status", params={"verb": "i_am_done", "task_id": "task-A"}, ) assert resp.status_code == _OK assert resp.json()["attempts"] == 1 # --------------------------------------------------------------------------- # Reset semantics # --------------------------------------------------------------------------- def test_state_reset_clears_verb_attempts() -> None: """_state.reset() wipes the verb tracker (orchestrator calls this on spawn).""" expected = 2 for _ in range(expected): srv._record_verb_attempt("i_am_done", "task-A") assert srv._verb_attempt_count("i_am_done", "task-A") == expected srv._state.reset() assert srv._verb_attempt_count("i_am_done", "task-A") == 0 def test_state_reset_via_endpoint_clears_verb_attempts() -> None: """POST /budget/reset (called by orchestrator on spawn) clears the tracker too.""" client = TestClient(srv.app) client.post( "/verb/attempted", json={ "verb": "i_am_done", "task_id": "task-A", "rejection_kind": "tracing_gap", }, ) assert srv._verb_attempt_count("i_am_done", "task-A") == 1 resp = client.post("/budget/reset") assert resp.status_code == _OK assert srv._verb_attempt_count("i_am_done", "task-A") == 0 def test_verb_attempts_default_is_empty_deque() -> None: """defaultdict yields an empty deque for unseen keys — sanity check.""" fresh = srv._SessionState() assert isinstance(fresh.verb_attempts[("never_seen", None)], deque) assert len(fresh.verb_attempts[("never_seen", None)]) == 0 def test_i_am_done_cap_matches_foundation() -> None: """The local _I_AM_DONE_CAP constant tracks foundation. If foundation changes the cap, this test fails so the test constants are updated alongside. """ assert retry_limit_for("i_am_done") == _I_AM_DONE_CAP