mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
f48d088c08
A live task burned 5+ hours because every exit was locked. unclaim now works from verifying and needs_revision (service guard + lifecycle edge); the circuit breaker and the i_am_done push-failure remediate name the working chain ending in unclaim(); sync_branch(stash=true) clears the DIRTY_WORKSPACE dead-end (pop-conflict preserves the stash); blocking a task QA already owns now says to idle instead of listing states; the orchestrator auto-block logs real errors and skips states where blocking is meaningless instead of force-blocking them. declare_coverage (cell/main PM) retroactively stamps parent-AC refs on a child that implements them -- closing the roll-up deadlock where the declaring child was cancelled and its re-delegated replacement completed the work uncredited. Cancelling a ref-declaring child now warns and surfaces the orphaned criteria. Co-authored-by: Renn F <rennf93@users.noreply.github.com>
602 lines
22 KiB
Python
602 lines
22 KiB
Python
"""Per-verb circuit breaker — agent_sdk denies after N retries in 60s.
|
|
|
|
Phase 3 Task 14. The 2026-05-10 smoke run showed `i_am_done` retried 5+
|
|
times in 2 minutes (each rejection was a `tracing_gap`; the agent kept
|
|
calling). This module verifies the runtime tracker added to
|
|
`agent_sdk.server` enforces the per-verb cap from
|
|
`foundation.policy.agent_loop.retry_limit_for`.
|
|
|
|
The tracker is keyed on `(verb, task_id)` and operates over a 60s sliding
|
|
window. Tests poke time.monotonic via patch to drive window decay
|
|
deterministically — wallclock sleeps would slow the suite.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from typing import TYPE_CHECKING
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import roboco.agent_sdk.server as srv
|
|
from fastapi.testclient import TestClient
|
|
from roboco.foundation.policy.agent_loop import (
|
|
VERB_RETRY_LIMITS,
|
|
absolute_retry_limit_for,
|
|
retry_limit_for,
|
|
)
|
|
from roboco.services.gateway.envelope import Envelope
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Iterator
|
|
|
|
|
|
# Named constants — ruff PLR2004 forbids magic comparisons. Mirror the
|
|
# foundation cap for i_am_done so the tests remain readable.
|
|
_OK = 200
|
|
_I_AM_DONE_CAP = 3 # foundation.VERB_RETRY_LIMITS["i_am_done"]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_state() -> Iterator[None]:
|
|
"""Wipe the SDK session state before every test.
|
|
|
|
Helpers share a module-level `_state` singleton, so leakage between
|
|
tests would silently corrupt counts.
|
|
"""
|
|
srv._state.reset()
|
|
yield
|
|
srv._state.reset()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Envelope.circuit_open
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_envelope_circuit_open_kind() -> None:
|
|
"""Envelope.circuit_open is a distinct error kind."""
|
|
env = Envelope.circuit_open(
|
|
verb="i_am_done",
|
|
attempts=4,
|
|
window_seconds=60,
|
|
remediate="call i_am_blocked",
|
|
)
|
|
body = env.as_dict()
|
|
assert body["error"] == "circuit_open"
|
|
assert body["remediate"] == "call i_am_blocked"
|
|
assert body["message"] is not None
|
|
assert "i_am_done" in body["message"]
|
|
assert "60" in body["message"]
|
|
assert "4" in body["message"]
|
|
|
|
|
|
def test_envelope_circuit_open_carries_briefing() -> None:
|
|
"""Optional context_briefing is preserved when caller supplies it."""
|
|
env = Envelope.circuit_open(
|
|
verb="i_am_done",
|
|
attempts=4,
|
|
window_seconds=60,
|
|
remediate="x",
|
|
context_briefing={"hint": "blocked"},
|
|
)
|
|
body = env.as_dict()
|
|
assert body["context_briefing"] == {"hint": "blocked"}
|
|
|
|
|
|
def test_envelope_circuit_open_default_briefing_empty() -> None:
|
|
"""Omitted context_briefing defaults to an empty dict (not None)."""
|
|
env = Envelope.circuit_open(
|
|
verb="i_am_done", attempts=3, window_seconds=60, remediate="x"
|
|
)
|
|
body = env.as_dict()
|
|
assert body["context_briefing"] == {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tracker — record + count
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_record_attempt_increments_count_for_same_key() -> None:
|
|
"""Three records for the same (verb, task_id) yield count == 3."""
|
|
expected = 3
|
|
for _ in range(expected):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == expected
|
|
|
|
|
|
def test_tracker_keys_per_verb_task_pair() -> None:
|
|
"""The tracker keys on (verb, task_id), not just verb.
|
|
|
|
Records for task-A are independent of records for task-B even when the
|
|
verb is identical. Without this guarantee, an agent juggling two tasks
|
|
would trip the breaker on the wrong one.
|
|
"""
|
|
a_count = 3
|
|
b_count = 1
|
|
for _ in range(a_count):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
for _ in range(b_count):
|
|
srv._record_verb_attempt("i_am_done", "task-B")
|
|
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == a_count
|
|
assert srv._verb_attempt_count("i_am_done", "task-B") == b_count
|
|
|
|
|
|
def test_tracker_keys_per_verb_independent_of_task() -> None:
|
|
"""Different verbs on the same task are tracked independently."""
|
|
done_count = 2
|
|
submit_count = 1
|
|
for _ in range(done_count):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
for _ in range(submit_count):
|
|
srv._record_verb_attempt("submit_up", "task-A")
|
|
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == done_count
|
|
assert srv._verb_attempt_count("submit_up", "task-A") == submit_count
|
|
|
|
|
|
def test_tracker_handles_none_task_id() -> None:
|
|
"""Verbs without a task_id collapse to (verb, None) — still tracked."""
|
|
expected = 2
|
|
for _ in range(expected):
|
|
srv._record_verb_attempt("triage", None)
|
|
assert srv._verb_attempt_count("triage", None) == expected
|
|
|
|
|
|
def test_count_is_zero_for_unknown_key() -> None:
|
|
"""Querying a never-recorded key returns 0 (no KeyError)."""
|
|
assert srv._verb_attempt_count("never_called", "task-X") == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tracker — window decay
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_window_drops_entries_older_than_60s() -> None:
|
|
"""Attempts older than 60s are pruned from the window on next access.
|
|
|
|
Drives time.monotonic via patch so the test runs in <1ms instead of
|
|
waiting on wall-clock.
|
|
"""
|
|
base = 1000.0
|
|
initial = 3
|
|
after_jump = 1
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
mock_time.return_value = base
|
|
for _ in range(initial):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == initial
|
|
|
|
# Jump past the 60s window — old attempts must be dropped.
|
|
mock_time.return_value = base + 61.0
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
# The 3 old entries are pruned; only the new one remains.
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == after_jump
|
|
|
|
|
|
def test_window_keeps_entries_within_60s() -> None:
|
|
"""Attempts within the window survive — only ones past 60s drop."""
|
|
base = 1000.0
|
|
expected = 2
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
mock_time.return_value = base
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
|
|
mock_time.return_value = base + 30.0
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
|
|
mock_time.return_value = base + 59.0
|
|
# Both entries within the 60s window starting at base.
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == expected
|
|
|
|
|
|
def test_count_prunes_on_read_without_recording() -> None:
|
|
"""_verb_attempt_count prunes the window even when not recording.
|
|
|
|
A read-only check should still see a fresh count after time advances —
|
|
callers (e.g. _check_verb_circuit) rely on this.
|
|
"""
|
|
base = 1000.0
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
mock_time.return_value = base
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
|
|
mock_time.return_value = base + 61.0
|
|
# Count must drop without us recording anything new.
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _check_verb_circuit — gating logic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_check_returns_none_when_under_limit() -> None:
|
|
"""Below the cap, _check_verb_circuit returns None — call may proceed."""
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
# i_am_done cap is 3; we recorded 2.
|
|
assert srv._check_verb_circuit("i_am_done", "task-A") is None
|
|
|
|
|
|
def test_check_returns_envelope_at_or_above_limit() -> None:
|
|
"""At the cap, _check_verb_circuit returns a circuit_open envelope dict."""
|
|
limit = retry_limit_for("i_am_done")
|
|
assert limit is not None
|
|
for _ in range(limit):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
|
|
result = srv._check_verb_circuit("i_am_done", "task-A")
|
|
assert result is not None
|
|
assert result["error"] == "circuit_open"
|
|
assert "i_am_done" in result["message"]
|
|
assert result["remediate"] is not None
|
|
assert "i_am_blocked" in result["remediate"]
|
|
assert "unclaim" in result["remediate"]
|
|
|
|
|
|
def test_check_returns_none_for_unlimited_retry_verbs() -> None:
|
|
"""give_me_work / triage / evidence never trip the breaker."""
|
|
assert retry_limit_for("give_me_work") is None
|
|
# Even after many recorded attempts, no envelope is returned.
|
|
for _ in range(20):
|
|
srv._record_verb_attempt("give_me_work", None)
|
|
assert srv._check_verb_circuit("give_me_work", None) is None
|
|
|
|
|
|
def test_check_uses_default_cap_for_unknown_verb() -> None:
|
|
"""Unknown verbs fall back to BudgetPolicy.verb_retry_max_per_minute."""
|
|
limit = retry_limit_for("not_a_real_verb")
|
|
assert limit is not None # default cap applies
|
|
for _ in range(limit):
|
|
srv._record_verb_attempt("not_a_real_verb", "task-A")
|
|
assert srv._check_verb_circuit("not_a_real_verb", "task-A") is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# /verb/attempted endpoint
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_verb_attempted_endpoint_records_rejection() -> None:
|
|
"""POST /verb/attempted with a counted rejection_kind increments the window."""
|
|
client = TestClient(srv.app)
|
|
resp = client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
assert resp.status_code == _OK
|
|
body = resp.json()
|
|
assert body["verb"] == "i_am_done"
|
|
assert body["task_id"] == "task-A"
|
|
assert body["attempts"] == 1
|
|
assert body["limit"] == retry_limit_for("i_am_done")
|
|
assert body["open"] is False
|
|
assert body["circuit_envelope"] is None
|
|
|
|
|
|
def test_verb_attempted_endpoint_ignores_uncounted_kinds() -> None:
|
|
"""A non-rejection (e.g. ok-related) kind doesn't move the counter."""
|
|
client = TestClient(srv.app)
|
|
resp = client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "ok", # not in the counted set
|
|
},
|
|
)
|
|
assert resp.status_code == _OK
|
|
body = resp.json()
|
|
assert body["attempts"] == 0
|
|
assert body["open"] is False
|
|
|
|
|
|
def test_verb_attempted_endpoint_opens_circuit_at_threshold() -> None:
|
|
"""After `limit` rejections in 60s the endpoint reports open=True with envelope."""
|
|
client = TestClient(srv.app)
|
|
limit = retry_limit_for("i_am_done")
|
|
assert limit is not None
|
|
|
|
# Hit the cap.
|
|
last_body: dict[str, object] | None = None
|
|
for _ in range(limit):
|
|
resp = client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
assert resp.status_code == _OK
|
|
last_body = resp.json()
|
|
|
|
assert last_body is not None
|
|
assert last_body["attempts"] == limit
|
|
assert last_body["open"] is True
|
|
env = last_body["circuit_envelope"]
|
|
assert isinstance(env, dict)
|
|
assert env["error"] == "circuit_open"
|
|
assert "i_am_done" in env["message"]
|
|
|
|
|
|
def test_verb_attempted_endpoint_unlimited_verb_never_opens() -> None:
|
|
"""give_me_work bypasses the breaker even after many rejections."""
|
|
client = TestClient(srv.app)
|
|
for _ in range(10):
|
|
resp = client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "give_me_work",
|
|
"task_id": None,
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
assert resp.status_code == _OK
|
|
|
|
body = resp.json()
|
|
assert body["limit"] is None
|
|
assert body["open"] is False
|
|
assert body["circuit_envelope"] is None
|
|
|
|
|
|
def test_verb_circuit_status_endpoint_does_not_record() -> None:
|
|
"""GET /verb/circuit_status reads state without incrementing."""
|
|
client = TestClient(srv.app)
|
|
# Record one rejection via the POST endpoint.
|
|
client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
# Now poll status repeatedly — count must stay at 1.
|
|
for _ in range(5):
|
|
resp = client.get(
|
|
"/verb/circuit_status",
|
|
params={"verb": "i_am_done", "task_id": "task-A"},
|
|
)
|
|
assert resp.status_code == _OK
|
|
assert resp.json()["attempts"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Reset semantics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_state_reset_clears_verb_attempts() -> None:
|
|
"""_state.reset() wipes the verb tracker (orchestrator calls this on spawn)."""
|
|
expected = 2
|
|
for _ in range(expected):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == expected
|
|
|
|
srv._state.reset()
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == 0
|
|
|
|
|
|
def test_state_reset_via_endpoint_clears_verb_attempts() -> None:
|
|
"""POST /budget/reset (called by orchestrator on spawn) clears the tracker too."""
|
|
client = TestClient(srv.app)
|
|
client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == 1
|
|
|
|
resp = client.post("/budget/reset")
|
|
assert resp.status_code == _OK
|
|
assert srv._verb_attempt_count("i_am_done", "task-A") == 0
|
|
|
|
|
|
def test_verb_attempts_default_is_empty_deque() -> None:
|
|
"""defaultdict yields an empty deque for unseen keys — sanity check."""
|
|
fresh = srv._SessionState()
|
|
assert isinstance(fresh.verb_attempts[("never_seen", None)], deque)
|
|
assert len(fresh.verb_attempts[("never_seen", None)]) == 0
|
|
|
|
|
|
def test_i_am_done_cap_matches_foundation() -> None:
|
|
"""The local _I_AM_DONE_CAP constant tracks foundation.
|
|
|
|
If foundation changes the cap, this test fails so the test constants
|
|
are updated alongside.
|
|
"""
|
|
assert retry_limit_for("i_am_done") == _I_AM_DONE_CAP
|
|
|
|
|
|
def test_qa_handoff_retry_keys_match_mcp_verb_names() -> None:
|
|
"""#150: the per-verb retry caps are keyed by the MCP-exposed verb names
|
|
(``pass`` / ``fail`` — what the SDK receives via /verb/attempted, derived
|
|
from the flow URL path), NOT the IntentSpec-internal ``pass_review`` /
|
|
``fail_review``. A rename on one side without the other would silently drop
|
|
the explicit cap for QA handoffs. Pin that the MCP names are explicit keys
|
|
so the mapping can't drift uncaught.
|
|
"""
|
|
assert "pass" in VERB_RETRY_LIMITS
|
|
assert "fail" in VERB_RETRY_LIMITS
|
|
assert retry_limit_for("pass") == VERB_RETRY_LIMITS["pass"]
|
|
assert retry_limit_for("fail") == VERB_RETRY_LIMITS["fail"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ABSOLUTE (session-scoped, never-pruned) breaker — the slow-drip fix
|
|
#
|
|
# Production, 2026-07-08: an agent's i_am_done was rejected once every 3-4
|
|
# minutes for 30+ minutes. Each rejection arrived alone in an empty 60s
|
|
# window, so `_check_verb_circuit` never saw more than 1 attempt at a time
|
|
# and never tripped. `_verb_absolute_attempts` counts cumulatively across
|
|
# the whole container session (never pruned) so pacing can't defeat it.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_I_AM_DONE_ABSOLUTE_CAP = 9 # VERB_RETRY_LIMITS["i_am_done"] * multiplier(3)
|
|
|
|
|
|
def test_i_am_done_absolute_cap_matches_foundation() -> None:
|
|
"""Pin the effective absolute cap so a foundation change is caught here too."""
|
|
assert absolute_retry_limit_for("i_am_done") == _I_AM_DONE_ABSOLUTE_CAP
|
|
|
|
|
|
def test_absolute_tracker_keys_per_verb_task_pair() -> None:
|
|
"""Different tasks accumulate independent absolute counts."""
|
|
a_count = 5
|
|
b_count = 2
|
|
for _ in range(a_count):
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
for _ in range(b_count):
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-B")
|
|
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-A") == a_count
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-B") == b_count
|
|
|
|
|
|
def test_absolute_tracker_keys_per_verb_independent_of_task() -> None:
|
|
"""Different verbs on the same task don't share the cumulative counter."""
|
|
done_count = 4
|
|
submit_count = 1
|
|
for _ in range(done_count):
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
for _ in range(submit_count):
|
|
srv._record_verb_attempt_absolute("submit_up", "task-A")
|
|
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-A") == done_count
|
|
assert srv._verb_absolute_attempt_count("submit_up", "task-A") == submit_count
|
|
|
|
|
|
def test_absolute_counter_never_prunes_with_time() -> None:
|
|
"""Unlike the windowed deque, a huge time jump does not reset the count."""
|
|
base = 1000.0
|
|
expected = 2
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
mock_time.return_value = base
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
mock_time.return_value = base + 10_000.0 # far past any sliding window
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-A") == expected
|
|
|
|
|
|
def test_slow_drip_never_trips_window_but_trips_absolute_cap() -> None:
|
|
"""Rejections spaced > 60s apart never trip `_check_verb_circuit`, but
|
|
the absolute cap still trips once the cumulative count reaches it —
|
|
the exact production scenario this breaker was added to close.
|
|
"""
|
|
cap = absolute_retry_limit_for("i_am_done")
|
|
assert cap is not None
|
|
base = 1000.0
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
for i in range(cap):
|
|
mock_time.return_value = base + i * 200.0 # always > 60s apart
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
# The sliding window is always empty when this attempt lands.
|
|
assert srv._check_verb_circuit("i_am_done", "task-A") is None
|
|
|
|
result = srv._check_verb_absolute_circuit("i_am_done", "task-A")
|
|
assert result is not None
|
|
assert result["error"] == "circuit_open"
|
|
assert "absolute cap" in result["message"]
|
|
assert "i_am_blocked" in result["remediate"]
|
|
assert "unclaim" in result["remediate"]
|
|
|
|
|
|
def test_absolute_check_returns_none_below_cap() -> None:
|
|
"""One rejection short of the cap, the absolute breaker stays closed."""
|
|
cap = absolute_retry_limit_for("i_am_done")
|
|
assert cap is not None
|
|
for _ in range(cap - 1):
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
assert srv._check_verb_absolute_circuit("i_am_done", "task-A") is None
|
|
|
|
|
|
def test_absolute_check_returns_none_for_unlimited_retry_verbs() -> None:
|
|
"""give_me_work stays exempt from the absolute cap too."""
|
|
assert absolute_retry_limit_for("give_me_work") is None
|
|
for _ in range(50):
|
|
srv._record_verb_attempt_absolute("give_me_work", None)
|
|
assert srv._check_verb_absolute_circuit("give_me_work", None) is None
|
|
|
|
|
|
def test_combined_check_still_trips_fast_storm_via_window() -> None:
|
|
"""Windowed behavior is unchanged: 3 fast rejections (the existing
|
|
i_am_done cap) still trip via the combined check, well below the
|
|
absolute cap of 9 — and the message reads as a windowed trip, not a
|
|
session one.
|
|
"""
|
|
limit = retry_limit_for("i_am_done")
|
|
assert limit is not None
|
|
for _ in range(limit):
|
|
srv._record_verb_attempt("i_am_done", "task-A")
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
|
|
result = srv._check_any_verb_circuit("i_am_done", "task-A")
|
|
assert result is not None
|
|
assert result["error"] == "circuit_open"
|
|
assert "this session" not in result["message"]
|
|
assert "in last" in result["message"]
|
|
|
|
|
|
def test_verb_attempted_endpoint_trips_absolute_cap_on_slow_drip() -> None:
|
|
"""End-to-end repro through the real /verb/attempted endpoint: rejections
|
|
paced far past 60s apart never open the windowed breaker but do open the
|
|
absolute one once the cumulative count reaches the cap.
|
|
"""
|
|
client = TestClient(srv.app)
|
|
cap = absolute_retry_limit_for("i_am_done")
|
|
assert cap is not None
|
|
last_body: dict[str, object] | None = None
|
|
with patch("roboco.agent_sdk.server.time.monotonic") as mock_time:
|
|
for i in range(cap):
|
|
mock_time.return_value = 1000.0 + i * 200.0
|
|
resp = client.post(
|
|
"/verb/attempted",
|
|
json={
|
|
"verb": "i_am_done",
|
|
"task_id": "task-A",
|
|
"rejection_kind": "tracing_gap",
|
|
},
|
|
)
|
|
assert resp.status_code == _OK
|
|
last_body = resp.json()
|
|
if i < cap - 1:
|
|
assert last_body["open"] is False
|
|
|
|
assert last_body is not None
|
|
assert last_body["open"] is True
|
|
env = last_body["circuit_envelope"]
|
|
assert isinstance(env, dict)
|
|
assert env["error"] == "circuit_open"
|
|
assert "absolute cap" in env["message"]
|
|
|
|
|
|
def test_state_reset_clears_verb_absolute_attempts() -> None:
|
|
"""_state.reset() (a fresh container spawn) wipes the absolute tracker too."""
|
|
expected = 3
|
|
for _ in range(expected):
|
|
srv._record_verb_attempt_absolute("i_am_done", "task-A")
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-A") == expected
|
|
|
|
srv._state.reset()
|
|
assert srv._verb_absolute_attempt_count("i_am_done", "task-A") == 0
|
|
|
|
|
|
def test_verb_absolute_attempts_default_is_zero() -> None:
|
|
"""defaultdict yields 0 for unseen keys — sanity check."""
|
|
fresh = srv._SessionState()
|
|
assert fresh.verb_absolute_attempts[("never_seen", None)] == 0
|