fix(gateway): bounded fail-open evidence assembly + PM decision transient-failure bypass

Evidence-assembly git legs (diff, changed-files, branch fetch, advisory
conventions run) ran unbounded inside claim_review / claim_doc_task /
claim_gate_review / evidence() / i_am_done's envelope build, so a slow
clone turned the whole verb into a silent 120s FlowVerbTimeout 504.
Each leg now runs through run_bounded_leg under a shared LegBudget
(evidence_assembly_timeout_seconds, 45s total): a timed-out leg — both
asyncio TimeoutError and git's own GitTimeoutError — degrades into an
evidence_gaps note on the envelope instead of hanging the verb, while
non-timeout git errors still propagate. The advisory conventions run
gets an inner-only timeout (conventions_validator_advisory_timeout_
seconds, 30s) threaded down to the subprocess so it is never orphaned
by an outer cancel; the fail-closed i_am_done/pr_pass conventions
gates keep their hardcoded 120s.

_ensure_pm_decision now reports a PmDecisionOutcome: a transient DB
failure recording the PM's decision journal (e.g. lock timeout under
load) no longer launders into a journal:decision gate rejection that
escalates and BLOCKS the task — the verb's own rationale satisfies the
gate with a structured warning, across all seven PM verbs.

Also: repo-wide ruff realignment to the lockfile-pinned ruff (8
format-only diffs, 14 UP038 isinstance conversions) that a transiently
newer venv ruff had masked.

Gate: 15474 passed, 459 skipped; ruff/mypy/xenon/vulture/bandit/
pip-audit/deptry/import-linter/foundation-check all green.
This commit is contained in:
Renn F
2026-07-31 13:56:19 +02:00
parent ef8849b0cd
commit ce4d02b3c2
41 changed files with 2160 additions and 138 deletions
@@ -84,7 +84,6 @@ async def test_system_stream_disconnects_on_cancelled_error() -> None:
async def test_notification_stream_disconnects_on_non_disconnect_exception(
monkeypatch: pytest.MonkeyPatch,
) -> None:
agent_id = uuid4()
mgr = ConnectionManager()
ws = _mock_ws_for_receive(RuntimeError("transport reset"))
@@ -109,7 +108,6 @@ async def test_notification_stream_disconnects_on_non_disconnect_exception(
async def test_agent_stream_disconnects_on_non_disconnect_exception(
monkeypatch: pytest.MonkeyPatch,
) -> None:
target_id = uuid4()
viewer_id = uuid4()
mgr = ConnectionManager()
@@ -89,7 +89,6 @@ async def test_system_stream_reaps_silent_socket_after_idle_timeout() -> None:
async def test_notification_stream_reaps_silent_socket_after_idle_timeout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
agent_id = uuid4()
mgr = ConnectionManager()
hang_future: asyncio.Future[str] = asyncio.Future()
@@ -109,7 +108,6 @@ async def test_notification_stream_reaps_silent_socket_after_idle_timeout(
async def test_agent_stream_reaps_silent_socket_after_idle_timeout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
target_id = uuid4()
viewer_id = uuid4()
mgr = ConnectionManager()
+17 -8
View File
@@ -978,18 +978,27 @@ async def test_escalate_up_survives_journal_write_lock_timeout() -> None:
with no rollback/savepoint, poisoning the session so the very next
attribute touch (``_escalate_up_preflight`` reading ``t.id``) raised an
unhandled ``PendingRollbackError``. The write is now savepoint-guarded
(``begin_nested()``): the failure is contained, the verb falls through
cleanly to the normal tracing_gap rejection (no decision was actually
persisted), and the task stays fully readable — no unhandled exception
escapes ``escalate_up``."""
(``begin_nested()``): the failure is contained and no unhandled
exception escapes ``escalate_up``.
Round-2 fix (transient-failure gate bypass): a lock-timeout with a
non-empty rationale already in hand (``reason``) no longer falls through
to a tracing_gap rejection either — the PM answered the gate's actual
question (why); the write was only ever a convenience. Laundering
transient DB congestion into a rejected/blocked PM verb is exactly the
bug this closes — see test_pm_decision_transient_failure_* below for the
gate-helper-level unit coverage.
"""
pm_id = uuid4()
task_id = uuid4()
t = MagicMock(id=task_id, status="blocked", assigned_to=pm_id, team="backend")
after = MagicMock(**{**t.__dict__, "assigned_to": uuid4()})
task_svc = AsyncMock()
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(
role="cell_pm", escalation_target="main-pm"
)
task_svc.escalate.return_value = after
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = False
journal_svc.latest_decision_at.return_value = None
@@ -1006,11 +1015,11 @@ async def test_escalate_up_survives_journal_write_lock_timeout() -> None:
# The savepoint was actually engaged — proves the fix is wired in, not
# merely that AsyncMock happened to swallow the raise on its own.
task_svc.session.begin_nested.assert_called()
# No unhandled exception escaped escalate_up: the gate falls through to
# its normal clean rejection since the decision write never landed.
# No unhandled exception escaped escalate_up, AND the gate is satisfied
# by the verb's own rationale instead of rejecting a DB hiccup.
body = env.as_dict()
assert body["error"] == "tracing_gap"
assert "journal:decision" in body["missing"]
assert body["error"] is None, body
task_svc.escalate.assert_awaited_once()
# The task is still fully readable afterward — this is exactly where
# the production trace crashed with PendingRollbackError on t.id.
assert t.id == task_id
@@ -109,3 +109,18 @@ async def test_gate_records_findings_even_when_blocking(
env = await c._conventions_gate(_ctx())
assert env is not None # still blocks
assert recorded and recorded[0] is _BLOCK_RESULT
@pytest.mark.asyncio
async def test_gate_never_overrides_the_fail_closed_timeout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""i_am_done's conventions gate is fail-closed and must keep the
validator's hardcoded 120s cap — unlike claim_review's advisory path, it
must never pass a ``timeout`` override down to ``conventions_check_for_task``."""
monkeypatch.setattr(settings, "conventions_enabled", True)
c = _make_choreographer(check_result={"findings": [], "could_not_run": False})
await c._conventions_gate(_ctx())
check = c.git.conventions_check_for_task
check.assert_awaited_once()
assert "timeout" not in check.await_args.kwargs
@@ -125,3 +125,18 @@ async def test_pr_pass_guard_inert_when_flag_off(
monkeypatch.setattr(settings, "conventions_enabled", False)
c = _make_choreographer(check_result=_BLOCK_RESULT)
assert await c._conventions_guard(uuid4(), MagicMock(), {}) is None
@pytest.mark.asyncio
async def test_pr_pass_guard_never_overrides_the_fail_closed_timeout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The pr_pass gate is fail-closed and must keep the validator's
hardcoded 120s cap unlike claim_review's advisory path, it must
never pass a ``timeout`` override down to ``conventions_check_for_task``."""
monkeypatch.setattr(settings, "conventions_enabled", True)
c = _make_choreographer(check_result={"findings": [], "could_not_run": False})
await c._conventions_guard(uuid4(), MagicMock(), {})
check = c.git.conventions_check_for_task
check.assert_awaited_once()
assert "timeout" not in check.await_args.kwargs
@@ -31,7 +31,12 @@ async def test_findings_surfaced_when_flag_on(monkeypatch: pytest.MonkeyPatch) -
monkeypatch.setattr(settings, "conventions_enabled", True)
findings = [{"file": "x.py", "line": 1, "level": "warn", "fix_hint": "h"}]
c = _make_choreographer(check_result={"findings": findings, "could_not_run": False})
assert await c._qa_convention_findings(uuid4(), MagicMock()) == findings
gaps: list[str] = []
assert (
await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
== findings
)
assert gaps == []
@pytest.mark.asyncio
@@ -40,21 +45,32 @@ async def test_empty_when_flag_off(monkeypatch: pytest.MonkeyPatch) -> None:
c = _make_choreographer(
check_result={"findings": [{"file": "x"}], "could_not_run": False}
)
assert await c._qa_convention_findings(uuid4(), MagicMock()) == []
gaps: list[str] = []
assert (
await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
== []
)
assert gaps == []
@pytest.mark.asyncio
async def test_could_not_run_surfaced_as_single_entry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A non-timeout could_not_run reason ("boom") stays fail-open in
convention_findings but must NOT also spam evidence_gaps only a
detected timeout does (see test_claim_review_conventions_timeout_
degrades_with_gap in test_evidence_assembly_bounded_legs.py)."""
monkeypatch.setattr(settings, "conventions_enabled", True)
c = _make_choreographer(
check_result={"findings": [], "could_not_run": True, "reason": "boom"}
)
out = await c._qa_convention_findings(uuid4(), MagicMock())
gaps: list[str] = []
out = await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
assert len(out) == 1
assert out[0]["could_not_run"] is True
assert out[0]["reason"] == "boom"
assert gaps == []
def _stub_task() -> MagicMock:
@@ -0,0 +1,763 @@
"""Bounded advisory-evidence legs on claim_review / claim_doc_task /
claim_gate_review / evidence() / i_am_done's success envelope.
Live bug: claim-evidence assembly's slow legs (branch-fetch-backed diff,
list_changed_files, the conventions-validator subprocess) had no per-leg
budget, so a hung leg silently ate the whole ``flow_verb_timeout_seconds``
(120s) and died as a FlowVerbTimeout 504 holding every row the request
touched for the duration. Fix: each slow leg runs bounded via
``run_bounded_leg`` (``asyncio.wait_for``) against a SHARED ``LegBudget`` per
build; a timeout skips that piece, records a human-readable note in the
evidence's ``evidence_gaps``, and lets the claim verb succeed with partial
evidence instead of hanging.
Adversarial-review follow-up (round 2) covers four confirmed gaps:
1. ``run_bounded_leg`` must catch ``GitTimeoutError`` too (``_run_git``'s own
internal subprocess bound NOT a ``TimeoutError`` subclass, and usually
the FIRST bound to trip since it defaults to 30s, shorter than a leg's
own budget) every timeout-shaped test below is parametrized over both
exception shapes.
2. The conventions leg no longer wraps ``_qa_convention_findings`` in an
outer ``run_bounded_leg`` that raced ``conventions_check_for_task``'s
own inner timeout+cleanup and leaked the validator subprocess. It now
self-bounds via the ``timeout`` kwarg alone and reports its own gap.
3. ``fetch_branch_for_inspection`` takes an optional ``subprocess_timeout``
so its fetch subprocess self-bounds near the leg's own budget instead of
occupying a thread on the shared default executor for up to 300s.
4. A shared ``LegBudget`` (one per evidence build) makes every leg's
``wait_for`` draw from one TOTAL budget instead of getting its own full
allotment summed per-leg budgets can no longer exceed the total.
"""
from __future__ import annotations
import asyncio
import time
from datetime import UTC, datetime
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.exceptions import GitCommandError, GitTimeoutError
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
from roboco.services.gateway.choreographer.evidence_legs import (
LegBudget,
run_bounded_leg,
)
# Every timeout-shaped test is parametrized over both real timeout shapes:
# asyncio's own cancellation-converted TimeoutError, and GitTimeoutError
# (GitService._run_git's own internal subprocess bound — a GitError/
# RobocoError subclass, NOT a TimeoutError subclass, and the most common
# real-world single-hung-git-call shape since it defaults to a SHORTER
# window, 30s, than a leg's own budget).
_TIMEOUT_EXCEPTIONS = (
TimeoutError("hung"),
GitTimeoutError("git diff", 30),
)
_TIMEOUT_IDS = ("asyncio_timeout", "git_timeout")
# ---------------------------------------------------------------------------
# run_bounded_leg / LegBudget themselves
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_run_bounded_leg_passes_through_on_success() -> None:
async def fast() -> str:
return "value"
gaps: list[str] = []
result = await run_bounded_leg(
fast(),
default="fallback",
budget=LegBudget(5.0),
leg="unit leg",
hint="check manually",
task_id=uuid4(),
gaps=gaps,
)
assert result == "value"
assert gaps == []
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_run_bounded_leg_degrades_to_default_on_timeout(exc: Exception) -> None:
async def hangs() -> str:
raise exc
gaps: list[str] = []
result = await run_bounded_leg(
hangs(),
default="fallback",
budget=LegBudget(5.0),
leg="unit leg",
hint="check manually",
task_id=uuid4(),
gaps=gaps,
)
assert result == "fallback"
assert len(gaps) == 1
assert "unit leg unavailable" in gaps[0]
assert "check manually" in gaps[0]
@pytest.mark.asyncio
async def test_run_bounded_leg_actually_bounds_a_slow_coroutine() -> None:
"""A genuinely slow (not pre-raised) coroutine is cancelled at the
budget's deadline, not awaited to completion."""
async def slow() -> str:
await asyncio.sleep(10)
return "too late"
gaps: list[str] = []
result = await run_bounded_leg(
slow(),
default="fallback",
budget=LegBudget(0.05),
leg="unit leg",
hint="check manually",
task_id=uuid4(),
gaps=gaps,
)
assert result == "fallback"
assert len(gaps) == 1
@pytest.mark.asyncio
async def test_run_bounded_leg_other_git_error_still_propagates() -> None:
"""A real command failure (not a timeout) is NOT a degrade case — it
must still propagate uncaught, same as any other unexpected exception."""
async def fails() -> str:
raise GitCommandError("git diff", "fatal: bad revision")
gaps: list[str] = []
with pytest.raises(GitCommandError):
await run_bounded_leg(
fails(),
default="fallback",
budget=LegBudget(5.0),
leg="unit leg",
hint="check manually",
task_id=uuid4(),
gaps=gaps,
)
assert gaps == []
def test_leg_budget_remaining_shrinks_over_time() -> None:
# Total well above _MIN_LEG_SECONDS (1.0) so the floor never engages
# here — otherwise both readings would clamp to 1.0 and look equal.
budget = LegBudget(3.0)
first = budget.remaining()
time.sleep(0.1)
second = budget.remaining()
assert second < first
assert first == pytest.approx(3.0, abs=0.05)
assert (first - second) == pytest.approx(0.1, abs=0.05)
def test_leg_budget_floors_at_minimum() -> None:
"""A budget already past its deadline still yields a positive window
(the floor) rather than 0 or a negative timeout a leg always gets a
real chance to run, even a badly-exhausted one."""
budget = LegBudget(0.01)
time.sleep(0.05)
assert budget.remaining() == pytest.approx(1.0, abs=0.05)
@pytest.mark.asyncio
async def test_run_bounded_leg_shares_one_shrinking_budget_across_legs() -> None:
"""Three legs sharing ONE LegBudget: the first two finish fast and
consume real budget; the last two never finish on their own and get
progressively SMALLER windows (shrinking, not each getting the full
total) both record their own gap, and total wall time stays bounded
near the shared total instead of the naive per-leg sum (0.1+0.1+5+5s).
"""
budget = LegBudget(1.2)
gaps: list[str] = []
async def _takes(seconds: float) -> str:
await asyncio.sleep(seconds)
return "done"
start = time.monotonic()
remaining_before_1 = budget.remaining()
r1 = await run_bounded_leg(
_takes(0.1),
default="gap1",
budget=budget,
leg="leg1",
hint="h",
task_id="t",
gaps=gaps,
)
remaining_before_2 = budget.remaining()
r2 = await run_bounded_leg(
_takes(5.0),
default="gap2",
budget=budget,
leg="leg2",
hint="h",
task_id="t",
gaps=gaps,
)
remaining_before_3 = budget.remaining()
r3 = await run_bounded_leg(
_takes(5.0),
default="gap3",
budget=budget,
leg="leg3",
hint="h",
task_id="t",
gaps=gaps,
)
elapsed = time.monotonic() - start
expected_gap_count = 2 # leg2 + leg3 both timed out; leg1 completed
assert r1 == "done"
assert r2 == "gap2"
assert r3 == "gap3"
assert len(gaps) == expected_gap_count
assert "leg2" in gaps[0]
assert "leg3" in gaps[1]
# Each leg's own remaining() reading is strictly smaller than the last
# — the shared deadline never resets.
assert remaining_before_1 > remaining_before_2 > remaining_before_3
# ponytail: the floor (max 1.0s) can inflate the LAST leg's window past
# what a naive "budget minus elapsed" would give once the deadline is
# already exhausted — a single floor engagement caps the worst-case
# overage at _MIN_LEG_SECONDS (1.0s), so budget total + ~1.3s is a safe,
# honest ceiling rather than a strict `<= budget` bound. Upgrade path:
# make the floor configurable if a caller ever needs a tighter cap.
budget_total_seconds = 1.2
floor_overage_tolerance_seconds = 1.3
assert elapsed <= budget_total_seconds + floor_overage_tolerance_seconds
# And it's nowhere near the naive per-leg-gets-its-own-full-timeout sum
# (0.1 + 5.0 + 5.0 = 10.1s) that pre-LegBudget behavior would produce.
naive_per_leg_sum_seconds = 5.0
assert elapsed < naive_per_leg_sum_seconds
# ---------------------------------------------------------------------------
# Shared choreographer test harness (mirrors test_choreographer_qa.py /
# test_claim_doc_task_checkout.py / test_claim_gate_review_guards.py)
# ---------------------------------------------------------------------------
def _make_deps(**overrides: Any) -> ChoreographerDeps:
base: dict[str, Any] = {
"task": AsyncMock(),
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base.update(overrides)
repo = base["evidence_repo"]
for method in (
"list_unread_a2a",
"list_unread_mentions",
"list_pending_notifications",
"task_metadata_gaps",
"recent_team_activity",
"blockers_in_lane",
"journal_highlights_for_task",
):
getattr(repo, method).return_value = []
_ldef = base["journal"].latest_decision_at.return_value
if type(_ldef).__name__ in ("MagicMock", "AsyncMock"):
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
return ChoreographerDeps(**base)
def _stub_empty_ledger(session: MagicMock) -> None:
session.execute = AsyncMock(
return_value=MagicMock(
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
)
)
# ---------------------------------------------------------------------------
# claim_review (qa.py)
# ---------------------------------------------------------------------------
_PR_NUMBER = 8
_PR_URL = "https://github.com/x/y/pull/8"
def _qa_task(task_id: Any) -> MagicMock:
return MagicMock(
id=task_id,
status="awaiting_qa",
assigned_to=None,
pr_number=_PR_NUMBER,
pr_url=_PR_URL,
commits=[{"sha": "abc123", "message": "feat: x"}],
team="backend",
branch_name="feature/backend/abc--def",
work_session_id=uuid4(),
documents=[],
dev_notes="implemented x",
acceptance_criteria=["AC1"],
acceptance_criteria_status=[
{"criterion": "AC1", "referencing_artifact_id": "abc123"},
],
parent_task_id=None,
)
def _qa_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
"""Does NOT touch ``settings.conventions_enabled`` — callers that care
set it themselves via their own ``monkeypatch`` fixture; a shared
forced-False here would silently clobber a caller's forced-True set
moments earlier (``monkeypatch.setattr`` doesn't stack, last write
wins), which is exactly what broke the conventions-specific tests."""
qa_id = uuid4()
task_id = uuid4()
t_initial = _qa_task(task_id)
t_claimed = MagicMock(**{**t_initial.__dict__, "assigned_to": qa_id})
task_svc = AsyncMock()
task_svc.get.return_value = t_initial
task_svc.agent_for.return_value = MagicMock(role="qa", team="backend")
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.qa_claim.return_value = t_claimed
_stub_empty_ledger(task_svc.session)
deps = _make_deps(task=task_svc, git=git_svc)
return Choreographer(deps), qa_id, task_id
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_claim_review_diff_timeout_degrades_with_gap(
monkeypatch: pytest.MonkeyPatch, exc: Exception
) -> None:
"""A hung git.diff on claim_review must not hang the verb: it degrades
to an empty diff, records the gap, and the OTHER leg (list_changed_files)
still comes through untouched."""
monkeypatch.setattr(settings, "conventions_enabled", False)
git_svc = AsyncMock()
git_svc.diff.side_effect = exc
git_svc.list_changed_files.return_value = ["README.md"]
c, qa_id, task_id = _qa_harness(git_svc)
env = await c.claim_review(qa_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == ""
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" in ev
assert len(ev["evidence_gaps"]) == 1
assert "pr diff unavailable" in ev["evidence_gaps"][0]
@pytest.mark.asyncio
async def test_claim_review_conventions_timeout_degrades_with_gap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""conventions_check_for_task's OWN internal timeout (proc.kill()'d and
reaped inside git.py, never raising) surfaces as could_not_run=True with
a "timed out" reason not an exception. The advisory call site
(_qa_convention_findings) detects that shape and records the gap
itself; NO outer run_bounded_leg wraps this leg (that's the fix — see
module docstring point 2)."""
monkeypatch.setattr(settings, "conventions_enabled", True)
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
git_svc.conventions_check_for_task.return_value = {
"findings": [],
"could_not_run": True,
"reason": "validator timed out after 30.0s",
}
c, qa_id, task_id = _qa_harness(git_svc)
env = await c.claim_review(qa_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert ev["convention_findings"] == [
{"could_not_run": True, "reason": "validator timed out after 30.0s"}
]
assert "evidence_gaps" in ev
assert any("conventions findings unavailable" in g for g in ev["evidence_gaps"])
# The advisory (shorter) ceiling reached the validator call, not the
# fail-closed i_am_done/pr_pass default (None -> hardcoded 120s).
git_svc.conventions_check_for_task.assert_awaited_once()
call_kwargs = git_svc.conventions_check_for_task.await_args.kwargs
assert (
call_kwargs["timeout"]
<= settings.conventions_validator_advisory_timeout_seconds
)
assert call_kwargs["timeout"] > 0
@pytest.mark.asyncio
async def test_claim_review_conventions_non_timeout_could_not_run_no_gap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A genuine resolution failure (not a timeout) still surfaces in
convention_findings (existing fail-open shape) but must NOT also spam
evidence_gaps that's reserved for actual degraded-advisory-leg notes."""
monkeypatch.setattr(settings, "conventions_enabled", True)
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
git_svc.conventions_check_for_task.return_value = {
"findings": [],
"could_not_run": True,
"reason": "resolution failed: NotFoundError: Branch not found",
}
c, qa_id, task_id = _qa_harness(git_svc)
env = await c.claim_review(qa_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["convention_findings"][0]["could_not_run"] is True
assert "evidence_gaps" not in ev
@pytest.mark.asyncio
async def test_qa_convention_findings_not_cancelled_by_outer_budget(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Regression for the orphaned-subprocess bug: even with a tiny shared
evidence-assembly budget, _qa_convention_findings must NOT be cut short
by an outer wait_for it awaits conventions_check_for_task to
completion. A slow-but-real mock (0.15s) run against a budget whose
total is far smaller (0.02s) proves there is no outer wrap: if there
still were one, this would return the default/empty shape instead of
the real result."""
monkeypatch.setattr(settings, "conventions_enabled", True)
async def _slow_check(*_args: object, **_kwargs: object) -> dict[str, Any]:
await asyncio.sleep(0.15)
return {"findings": [{"file": "x.py", "line": 1}], "could_not_run": False}
git_svc = AsyncMock()
git_svc.conventions_check_for_task.side_effect = _slow_check
c, _qa_id, _task_id = _qa_harness(git_svc)
cc: Any = c
gaps: list[str] = []
result = await cc._qa_convention_findings(
uuid4(), MagicMock(), timeout=0.02, gaps=gaps
)
assert result == [{"file": "x.py", "line": 1}]
assert gaps == []
@pytest.mark.asyncio
async def test_claim_review_normal_path_has_no_evidence_gaps(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Byte-for-byte unchanged normal path: no evidence_gaps key at all when
nothing times out."""
monkeypatch.setattr(settings, "conventions_enabled", False)
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
c, qa_id, task_id = _qa_harness(git_svc)
env = await c.claim_review(qa_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" not in ev
# ---------------------------------------------------------------------------
# claim_doc_task (doc.py)
# ---------------------------------------------------------------------------
def _doc_task(task_id: Any, branch: str) -> MagicMock:
return MagicMock(
id=task_id,
status="awaiting_documentation",
assigned_to=None,
task_type="documentation",
team="backend",
branch_name=branch,
quick_context=None,
documents=[],
commits=[{"sha": "abc123", "message": "[x] work"}],
pr_number=7,
pr_url="https://github.com/x/y/pull/7",
dev_notes="done",
acceptance_criteria_status=[],
work_session_id=uuid4(),
)
def _doc_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
doc_id = uuid4()
task_id = uuid4()
branch = "feature/backend/root1234--cellpm56--dev78901"
t_initial = _doc_task(task_id, branch)
t_claimed = MagicMock(**{**t_initial.__dict__, "assigned_to": doc_id})
task_svc = AsyncMock()
task_svc.get.return_value = t_initial
task_svc.agent_for.return_value = MagicMock(role="documenter", team="backend")
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.doc_claim.return_value = t_claimed
_stub_empty_ledger(task_svc.session)
deps = _make_deps(task=task_svc, git=git_svc)
return Choreographer(deps), doc_id, task_id
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_claim_doc_task_diff_timeout_degrades_with_gap(exc: Exception) -> None:
git_svc = AsyncMock()
git_svc.diff.side_effect = exc
git_svc.list_changed_files.return_value = ["README.md"]
c, doc_id, task_id = _doc_harness(git_svc)
env = await c.claim_doc_task(doc_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == ""
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" in ev
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_claim_doc_task_checkout_timeout_degrades_with_gap(
exc: Exception,
) -> None:
"""The checkout leg (run before evidence assembly) also degrades bounded
instead of an unbounded suppress(Exception) its gap folds into the
same evidence_gaps list the diff/list_changed_files legs use, drawing
from the SAME shared LegBudget."""
git_svc = AsyncMock()
git_svc.checkout_branch_in_agent_workspace.side_effect = exc
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
c, doc_id, task_id = _doc_harness(git_svc)
env = await c.claim_doc_task(doc_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
# The other legs are untouched by the checkout's own timeout.
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" in ev
assert any("workspace checkout unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
async def test_claim_doc_task_normal_path_has_no_evidence_gaps() -> None:
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
c, doc_id, task_id = _doc_harness(git_svc)
env = await c.claim_doc_task(doc_id, task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" not in ev
# ---------------------------------------------------------------------------
# claim_gate_review (pr_gate.py)
# ---------------------------------------------------------------------------
def _gate_task() -> MagicMock:
return MagicMock(
id=uuid4(),
status="awaiting_pr_review",
assigned_to=uuid4(),
parent_task_id=None,
task_type="planning",
dependency_ids=[],
team="main_pm",
pr_number=139,
pr_url="https://example/pr/139",
branch_name="feature/main_pm/root",
batch_id=None,
description=None,
acceptance_criteria=[],
)
def _gate_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
task_svc = AsyncMock()
t = _gate_task()
task_svc.get.return_value = t
task_svc.agent_for.return_value = MagicMock(
role="pr_reviewer", slug="be-pr-reviewer"
)
task_svc.list_in_progress_for_agent.return_value = []
task_svc.list_paused_for_agent.return_value = []
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
task_svc.has_earlier_incomplete_code_sibling.return_value = False
task_svc.pr_gate_claim = AsyncMock(return_value=t)
_stub_empty_ledger(task_svc.session)
deps = _make_deps(task=task_svc, git=git_svc)
return Choreographer(deps), t, uuid4()
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_claim_gate_review_diff_timeout_degrades_with_gap(exc: Exception) -> None:
git_svc = AsyncMock()
git_svc.diff.side_effect = exc
git_svc.list_changed_files.return_value = ["README.md"]
c, t, reviewer_id = _gate_harness(git_svc)
env = await c.claim_gate_review(reviewer_id, t.id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff"] == ""
assert "evidence_gaps" in ev
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
async def test_claim_gate_review_files_changed_timeout_degrades_with_gap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""list_changed_files hanging must not sink the whole gate claim, and the
diff leg (which succeeded) must remain intact in the evidence.
``_gate_changed_files`` has its own internal ``except Exception`` (an
existing hard-failure fail-open, untouched by this fix) that would
swallow a synchronously-raised exception (of either timeout shape)
before the outer ``run_bounded_leg`` ever saw it so this uses a
genuinely slow coroutine + a monkeypatched short budget to exercise the
real cancel-at-the-wall path (``asyncio.wait_for`` cancelling the
awaited task via ``CancelledError``, which that ``except Exception``
does NOT catch), matching what a real hang does in production.
"""
monkeypatch.setattr(settings, "evidence_assembly_timeout_seconds", 0.02)
async def _hangs(*_args: object, **_kwargs: object) -> list[str]:
await asyncio.sleep(5)
return ["should-not-be-reached"]
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.side_effect = _hangs
c, t, reviewer_id = _gate_harness(git_svc)
env = await c.claim_gate_review(reviewer_id, t.id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff"] == "diff content"
assert "evidence_gaps" in ev
assert any("files_changed unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
async def test_claim_gate_review_normal_path_has_no_evidence_gaps() -> None:
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
c, t, reviewer_id = _gate_harness(git_svc)
env = await c.claim_gate_review(reviewer_id, t.id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff"] == "diff content"
assert "evidence_gaps" not in ev
# ---------------------------------------------------------------------------
# _build_i_am_done_ok (i_am_done's success-envelope evidence). Runs strictly
# AFTER the composed transition already committed — advisory, not gating —
# so its list_changed_files leg is bounded exactly like the claim paths.
# ---------------------------------------------------------------------------
def _done_task(task_id: Any) -> MagicMock:
return MagicMock(
id=task_id,
branch_name="feature/backend/abc",
commits=[{"sha": "abc123", "message": "x"}],
dev_notes="done",
acceptance_criteria_status=[],
pr_number=5,
pr_url="https://github.com/x/y/pull/5",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_build_i_am_done_ok_files_changed_timeout_degrades_with_gap(
exc: Exception,
) -> None:
"""A hung list_changed_files leg in i_am_done's already-committed
success-envelope builder must not hang the dev's response — it degrades
to an empty files_changed and records the gap."""
agent_id = uuid4()
task_id = uuid4()
t = _done_task(task_id)
task_svc = AsyncMock()
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
_stub_empty_ledger(task_svc.session)
git_svc = AsyncMock()
git_svc.list_changed_files.side_effect = exc
deps = _make_deps(task=task_svc, git=git_svc)
c = Choreographer(deps)
env = await c._build_i_am_done_ok(agent_id, task_id, t)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["files_changed"] == []
assert "evidence_gaps" in ev
assert any("files_changed unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
async def test_build_i_am_done_ok_normal_path_has_no_evidence_gaps() -> None:
agent_id = uuid4()
task_id = uuid4()
t = _done_task(task_id)
task_svc = AsyncMock()
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
_stub_empty_ledger(task_svc.session)
git_svc = AsyncMock()
git_svc.list_changed_files.return_value = ["README.md"]
deps = _make_deps(task=task_svc, git=git_svc)
c = Choreographer(deps)
env = await c._build_i_am_done_ok(agent_id, task_id, t)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" not in ev
@@ -20,6 +20,8 @@ from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.exceptions import GitTimeoutError
from roboco.services.gateway.content_actions import ContentActions, ContentActionsDeps
@@ -223,3 +225,143 @@ async def test_evidence_no_branch_skips_git_calls() -> None:
assert body["evidence"]["pr_diff_summary"] == ""
git_svc.diff.assert_not_awaited()
git_svc.list_changed_files.assert_not_awaited()
# ---------------------------------------------------------------------------
# Bounded advisory-evidence legs: evidence() must not hang on a slow branch
# fetch / diff / list_changed_files leg — it degrades and records a note in
# evidence_gaps instead (same run_bounded_leg treatment as claim_review /
# claim_doc_task / claim_gate_review). Every timeout-shaped test is
# parametrized over both real timeout shapes: asyncio's own
# cancellation-converted TimeoutError, and GitTimeoutError (_run_git's own
# internal subprocess bound — a GitError/RobocoError subclass, NOT a
# TimeoutError subclass, and the most common real-world single-hung-git-call
# shape since it defaults to a SHORTER window than a leg's own budget).
# ---------------------------------------------------------------------------
_TIMEOUT_EXCEPTIONS = (
TimeoutError("hung"),
GitTimeoutError("git diff", 30),
)
_TIMEOUT_IDS = ("asyncio_timeout", "git_timeout")
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_evidence_diff_timeout_degrades_with_gap(exc: Exception) -> None:
"""A hung git.diff must not hang evidence(): it degrades to an empty
diff, records the gap, and list_changed_files (the other leg) still
comes through untouched."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
git_svc = AsyncMock()
git_svc.diff.side_effect = exc
git_svc.list_changed_files.return_value = ["README.md"]
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == ""
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" in ev
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
async def test_evidence_branch_fetch_timeout_degrades_with_gap(exc: Exception) -> None:
"""A hung workspace branch-fetch must not hang evidence() either — the
subsequent diff/list_changed_files legs still run (against whatever the
workspace already has) and the gap is recorded."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
workspace_svc = AsyncMock()
workspace_svc.fetch_branch_for_inspection.side_effect = exc
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" in ev
assert any("branch fetch unavailable" in g for g in ev["evidence_gaps"])
@pytest.mark.asyncio
async def test_evidence_branch_fetch_passes_subprocess_timeout_from_budget() -> None:
"""The branch-fetch leg passes its own remaining LegBudget share down as
fetch_branch_for_inspection's subprocess_timeout, so a hung fetch
subprocess self-terminates near the leg's own budget instead of
occupying a thread on the shared default executor for up to
workspace_clone_timeout (300s) after evidence() already gave up on it."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
workspace_svc = AsyncMock()
workspace_svc.fetch_branch_for_inspection.return_value = None
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
assert env.as_dict()["error"] is None
workspace_svc.fetch_branch_for_inspection.assert_awaited_once()
call_kwargs = workspace_svc.fetch_branch_for_inspection.await_args.kwargs
assert (
call_kwargs["subprocess_timeout"] <= settings.evidence_assembly_timeout_seconds
)
assert call_kwargs["subprocess_timeout"] > 0
@pytest.mark.asyncio
async def test_evidence_normal_path_has_no_evidence_gaps() -> None:
"""Byte-for-byte unchanged normal path: no evidence_gaps key at all when
nothing times out."""
agent_id = uuid4()
task_id = uuid4()
task_svc = AsyncMock()
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
git_svc = AsyncMock()
git_svc.diff.return_value = "diff content"
git_svc.list_changed_files.return_value = ["README.md"]
workspace_svc = AsyncMock()
evidence_repo = AsyncMock()
evidence_repo.journal_highlights_for_task.return_value = []
ca = ContentActions(
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
)
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
body = env.as_dict()
assert body["error"] is None, body
ev = body["evidence"]
assert ev["pr_diff_summary"] == "diff content"
assert ev["files_changed"] == ["README.md"]
assert "evidence_gaps" not in ev
@@ -7,6 +7,12 @@ gate runs. This removes the dominant stall where a loaded/weak-model PM
forgot the separate note(scope='decision') call and looped on a
tracing_gap respawn. The gate itself is unchanged (see
test_pm_decision_window.py) this only ensures a fresh decision exists.
``_ensure_pm_decision`` returns a ``PmDecisionOutcome`` ("fresh" / "wrote" /
"transient_failure" / "absent") the caller threads into the gate helper that
runs right after (see test_pm_decision_transient_failure.py for the
gate-satisfaction behavior itself) these tests pin the outcome value for
each branch.
"""
from __future__ import annotations
@@ -54,8 +60,11 @@ async def test_writes_decision_when_none_exists() -> None:
journal.latest_decision_at.return_value = None
c = Choreographer(_make_deps(journal=journal))
await c._ensure_pm_decision(agent_id, task_id, "Merging PR #120; all ACs verified")
outcome = await c._ensure_pm_decision(
agent_id, task_id, "Merging PR #120; all ACs verified"
)
assert outcome == "wrote"
journal.write_decision.assert_awaited_once()
_args, kwargs = journal.write_decision.call_args
assert kwargs["agent_id"] == agent_id
@@ -69,8 +78,9 @@ async def test_skips_when_fresh_decision_already_exists() -> None:
journal.latest_decision_at.return_value = datetime.now(UTC) - timedelta(seconds=60)
c = Choreographer(_make_deps(journal=journal))
await c._ensure_pm_decision(uuid4(), uuid4(), "rationale text here")
outcome = await c._ensure_pm_decision(uuid4(), uuid4(), "rationale text here")
assert outcome == "fresh"
journal.write_decision.assert_not_awaited()
@@ -82,8 +92,11 @@ async def test_writes_when_existing_decision_is_stale() -> None:
)
c = Choreographer(_make_deps(journal=journal))
await c._ensure_pm_decision(uuid4(), uuid4(), "fresh rationale around this point")
outcome = await c._ensure_pm_decision(
uuid4(), uuid4(), "fresh rationale around this point"
)
assert outcome == "wrote"
journal.write_decision.assert_awaited_once()
@@ -92,21 +105,29 @@ async def test_noop_on_empty_rationale() -> None:
journal = AsyncMock()
c = Choreographer(_make_deps(journal=journal))
await c._ensure_pm_decision(uuid4(), uuid4(), " ")
await c._ensure_pm_decision(uuid4(), uuid4(), None)
outcome_blank = await c._ensure_pm_decision(uuid4(), uuid4(), " ")
outcome_none = await c._ensure_pm_decision(uuid4(), uuid4(), None)
assert outcome_blank == "absent"
assert outcome_none == "absent"
journal.latest_decision_at.assert_not_awaited()
journal.write_decision.assert_not_awaited()
@pytest.mark.asyncio
async def test_swallows_write_failure_best_effort() -> None:
"""A journal write failure must not crash the verb — the gate then
rejects normally (the pre-fix behaviour), never a 500."""
"""A journal write failure must not crash the verb. Round-2 fix: the
outcome is "transient_failure" (not swallowed into a bare None) so the
caller's gate can treat a DB hiccup as satisfied by the rationale
already in hand see test_pm_decision_transient_failure.py for the
gate-satisfaction behavior itself."""
journal = AsyncMock()
journal.latest_decision_at.return_value = None
journal.write_decision.side_effect = RuntimeError("db down")
c = Choreographer(_make_deps(journal=journal))
# Must not raise.
await c._ensure_pm_decision(uuid4(), uuid4(), "rationale that triggers a write")
outcome = await c._ensure_pm_decision(
uuid4(), uuid4(), "rationale that triggers a write"
)
assert outcome == "transient_failure"
@@ -0,0 +1,285 @@
"""PM-decision write-then-gate: transient DB failure must not launder into a
durable rejection/block.
Live bug: every PM verb (complete / submit_up / submit_root / unblock /
escalate_up / escalate_to_ceo / delegate) runs ``_ensure_pm_decision`` to
auto-record its own rationale as a journal:decision before the freshness
gate (``_check_pm_decision_required`` / ``_check_complete_gates`` /
``_check_submit_up_gates``) runs. The write can lock-timeout under DB
contention (a concurrent claim holding the task row's FK share lock); the
old contract swallowed that and let the gate reject a "missing decision"
the PM's own rationale already answered — a PM retries, keeps getting
rejected while contention lasts, then escalates, and the task ends up
BLOCKED. Transient congestion laundered into a durable blocked task.
Fix: ``_ensure_pm_decision`` returns a ``PmDecisionOutcome`` "fresh" /
"wrote" / "transient_failure" / "absent". Every gate helper accepts
``pm_decision_outcome`` (default ``None`` = legacy behavior unchanged) and
treats "transient_failure" as gate-satisfied for THIS call the rationale
is in the verb payload; the write was only ever a convenience. "absent" (no
rationale at all) still rejects exactly as before.
These tests exercise the gate helpers directly (the leanest harness that
reaches the actual decision-point) for precise, fast coverage of the
mechanism, plus one near-real-call-site test per verb family
(``_cell_pm_complete_guard``, ``escalate_up``) proving the real wiring.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4
import pytest
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
from roboco.services.gateway.choreographer import _impl as _impl_module
from sqlalchemy.exc import OperationalError
def _make_choreographer(**overrides: Any) -> Choreographer:
base: dict[str, Any] = {
"task": AsyncMock(),
"work_session": AsyncMock(),
"git": AsyncMock(),
"a2a": AsyncMock(),
"journal": AsyncMock(),
"audit": AsyncMock(),
"evidence_repo": AsyncMock(),
}
base.update(overrides)
return Choreographer(ChoreographerDeps(**base))
_TRANSIENT_WARNING_SUBSTRING = "gate satisfied by verb rationale"
# ---------------------------------------------------------------------------
# _check_pm_decision_required (unblock / escalate_up / escalate_to_ceo / delegate)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_check_pm_decision_required_transient_failure_satisfies_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A "transient_failure" outcome satisfies the gate for this call even
though no fresh decision exists the verb's own rationale is the
substance, the write was a convenience.
``_impl.py`` logs via ``structlog.get_logger()`` directly; absent this
process having called ``roboco.logging.setup_logging()`` (never true in
a bare unit-test run), structlog uses its own default global config and
never touches stdlib ``logging`` so ``caplog`` cannot see it. Patching
the module-level ``logger`` object is the reliable way to assert a
structlog call in this harness.
"""
mock_logger = MagicMock()
monkeypatch.setattr(_impl_module, "logger", mock_logger)
journal_svc = AsyncMock()
journal_svc.latest_decision_at.return_value = None # no fresh decision
c = _make_choreographer(journal=journal_svc)
t = MagicMock(id=uuid4())
env = await c._check_pm_decision_required(
"unblock",
uuid4(),
t.id,
t,
pm_decision_outcome="transient_failure",
)
assert env is None
mock_logger.warning.assert_called_once()
call = mock_logger.warning.call_args
assert _TRANSIENT_WARNING_SUBSTRING in call.args[0]
assert call.kwargs.get("verb") == "unblock"
@pytest.mark.asyncio
async def test_check_pm_decision_required_absent_still_rejects() -> None:
""" "absent" (no rationale, no fresh decision) rejects exactly as before
defense-in-depth unchanged."""
journal_svc = AsyncMock()
journal_svc.latest_decision_at.return_value = None
c = _make_choreographer(journal=journal_svc)
t = MagicMock(id=uuid4())
env = await c._check_pm_decision_required(
"unblock", uuid4(), t.id, t, pm_decision_outcome="absent"
)
assert env is not None
assert env.as_dict()["error"] == "tracing_gap"
@pytest.mark.asyncio
async def test_check_pm_decision_required_none_default_unchanged() -> None:
"""Every call site that hasn't threaded the outcome through (there are
none left in production, but the param defaults to None for legacy
parity) behaves byte-for-byte as before: no fresh decision rejects."""
journal_svc = AsyncMock()
journal_svc.latest_decision_at.return_value = None
c = _make_choreographer(journal=journal_svc)
t = MagicMock(id=uuid4())
env = await c._check_pm_decision_required("unblock", uuid4(), t.id, t)
assert env is not None
assert env.as_dict()["error"] == "tracing_gap"
@pytest.mark.asyncio
async def test_check_pm_decision_required_fresh_path_unchanged() -> None:
"""A genuinely fresh decision passes regardless of pm_decision_outcome
"fresh" is not a special-case bypass, it's the ordinary passing path."""
journal_svc = AsyncMock()
journal_svc.latest_decision_at.return_value = datetime.now(UTC)
c = _make_choreographer(journal=journal_svc)
t = MagicMock(id=uuid4())
env = await c._check_pm_decision_required(
"unblock", uuid4(), t.id, t, pm_decision_outcome="fresh"
)
assert env is None
# ---------------------------------------------------------------------------
# _check_complete_gates (cell_pm / main_pm complete)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_check_complete_gates_transient_failure_satisfies_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""See ``test_check_pm_decision_required_transient_failure_satisfies_gate``
for why the module logger is patched directly instead of using caplog."""
mock_logger = MagicMock()
monkeypatch.setattr(_impl_module, "logger", mock_logger)
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = False
journal_svc.has_reflect_for_task.return_value = False
c = _make_choreographer(journal=journal_svc)
env = await c._check_complete_gates(
uuid4(),
uuid4(),
"closing this task: the cell's contribution merged cleanly",
pm_decision_outcome="transient_failure",
)
assert env is None
mock_logger.warning.assert_called_once()
call = mock_logger.warning.call_args
assert _TRANSIENT_WARNING_SUBSTRING in call.args[0]
assert call.kwargs.get("verb") == "complete"
@pytest.mark.asyncio
async def test_check_complete_gates_absent_still_rejects() -> None:
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = False
journal_svc.has_reflect_for_task.return_value = False
c = _make_choreographer(journal=journal_svc)
env = await c._check_complete_gates(
uuid4(),
uuid4(),
"closing this task: the cell's contribution merged cleanly",
pm_decision_outcome="absent",
)
assert env is not None
assert env.as_dict()["error"] == "tracing_gap"
@pytest.mark.asyncio
async def test_check_complete_gates_fresh_path_unchanged() -> None:
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = True
journal_svc.has_reflect_for_task.return_value = True
c = _make_choreographer(journal=journal_svc)
env = await c._check_complete_gates(
uuid4(),
uuid4(),
"closing this task: the cell's contribution merged cleanly",
pm_decision_outcome="fresh",
)
assert env is None
# ---------------------------------------------------------------------------
# _cell_pm_complete_guard — the real complete() call site, one hop above the
# gate helper, proving the outcome actually reaches it end to end (without
# dragging in the full cell_pm_complete verb's PR-merge/finalize machinery).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_cell_pm_complete_guard_survives_journal_write_lock_timeout() -> None:
"""journal.write_decision raising (a lock-timeout under DB contention)
with a substantive ``notes`` rationale in hand must not reject the PM's
complete the guard clears (returns None) instead of tracing_gap."""
pm_id = uuid4()
task_id = uuid4()
t = MagicMock(
id=task_id,
assigned_to=pm_id,
status="awaiting_pm_review",
pr_number=42,
)
task_svc = AsyncMock()
task_svc.all_subtasks_terminal.return_value = True
task_svc.uncovered_parent_acceptance_criteria.return_value = []
# _ensure_pm_decision opens a session.begin_nested() savepoint before
# the write raises — an unshaped AsyncMock's auto-attribute return
# doesn't support `async with`, orphaning the mock's internal coroutine
# (AsyncMockMixin._execute_mock_call never awaited).
task_svc.session = MagicMock()
task_svc.session.begin_nested = MagicMock(
return_value=MagicMock(
__aenter__=AsyncMock(return_value=None),
__aexit__=AsyncMock(return_value=False),
)
)
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = False
journal_svc.has_reflect_for_task.return_value = False
journal_svc.latest_decision_at.return_value = None
journal_svc.write_decision.side_effect = OperationalError(
"INSERT INTO journal_entries (id, ...) VALUES (...)",
{},
Exception("canceling statement due to lock timeout"),
)
c = _make_choreographer(task=task_svc, journal=journal_svc)
env = await c._cell_pm_complete_guard(
pm_id, task_id, t, "closing this task: the cell's contribution merged cleanly"
)
task_svc.session.begin_nested.assert_called()
assert env is None, env.as_dict() if env is not None else None
@pytest.mark.asyncio
async def test_cell_pm_complete_guard_empty_notes_still_rejects() -> None:
"""No rationale at all (empty notes) AND no fresh decision on record
still rejects "absent" is not a bypass."""
pm_id = uuid4()
task_id = uuid4()
t = MagicMock(
id=task_id,
assigned_to=pm_id,
status="awaiting_pm_review",
pr_number=42,
)
task_svc = AsyncMock()
task_svc.all_subtasks_terminal.return_value = True
task_svc.uncovered_parent_acceptance_criteria.return_value = []
journal_svc = AsyncMock()
journal_svc.has_decision_for_task.return_value = False
journal_svc.has_reflect_for_task.return_value = False
journal_svc.latest_decision_at.return_value = None
c = _make_choreographer(task=task_svc, journal=journal_svc)
env = await c._cell_pm_complete_guard(pm_id, task_id, t, "")
assert env is not None
assert env.as_dict()["error"] == "tracing_gap"
-1
View File
@@ -88,7 +88,6 @@ def test_agent_instance_default_factory_assigns_id() -> None:
def test_orchestrator_agent_config_defaults() -> None:
cfg = OrchestratorAgentConfig(
agent_id="be-dev-1",
blueprint_path=Path("/tmp/blueprint"),
@@ -49,7 +49,6 @@ async def test_get_bool_parses_and_defaults() -> None:
async def test_apply_overrides_stored_flags_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# Baseline env defaults.
monkeypatch.setattr(cfg, "external_pr_enabled", False)
monkeypatch.setattr(cfg, "research_enabled", True)
@@ -73,7 +72,6 @@ async def test_apply_overrides_stored_flags_only(
async def test_effective_values_use_env_default_when_unset(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(cfg, "strategy_engine_enabled", True)
async def fake_get(_self: SettingsService, _key: str) -> str | None:
@@ -149,3 +149,80 @@ async def test_validator_timeout_fails_closed_and_reaps(
assert "timed out" in (result.get("reason") or "")
fake_proc.kill.assert_called_once()
fake_proc.wait.assert_awaited_once()
@pytest.mark.asyncio
async def test_run_conventions_validator_timeout_override_used_over_hardcoded(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""An explicit ``timeout`` kwarg wins over the module-level hardcoded
default the advisory claim_review path's shorter budget must actually
reach the subprocess wait, not the fail-closed 120s cap. Sets the module
constant to something LONG (would never fire in the test's real time
budget) so a failure here would prove the override was ignored, not a
coincidence of both values being short."""
fake_proc = MagicMock()
fake_proc.returncode = None
async def _communicate() -> tuple[bytes, bytes]:
await asyncio.sleep(30)
return (b"", b"")
fake_proc.communicate = _communicate
fake_proc.kill = MagicMock()
fake_proc.wait = AsyncMock(return_value=-9)
async def _fake_exec(*_args: object, **_kwargs: object) -> object:
return fake_proc
monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec)
monkeypatch.setattr(git_module, "_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS", 300)
svc = _service()
result = await svc._run_conventions_validator(tmp_path, ["a.py"], timeout=0.01)
assert result["could_not_run"] is True
assert "timed out after 0.01s" in (result.get("reason") or "")
def _task_with_id(branch_name: str) -> MagicMock:
"""Like ``_task`` but with a real UUID id — ``conventions_check_for_task``
calls ``require_uuid(task.id)`` outside the resolution try/except, so a
bare MagicMock id would raise before reaching the validator call."""
return MagicMock(branch_name=branch_name, id=uuid4())
@pytest.mark.asyncio
async def test_conventions_check_for_task_forwards_timeout_override() -> None:
"""``conventions_check_for_task``'s ``timeout`` kwarg must reach
``_run_conventions_validator`` the seam ``claim_review`` uses to pin
the ADVISORY (shorter) budget instead of the fail-closed default."""
svc = _service()
_bind(svc, "_workspace_for_branch", AsyncMock(return_value=Path("/tmp/ws")))
_bind(svc, "list_changed_files", AsyncMock(return_value=["a.py"]))
_bind(svc, "_worktree_for_task", MagicMock(return_value=Path("/tmp/wt")))
_bind(svc, "_ensure_worktree_for_commit", AsyncMock(return_value=None))
validator = AsyncMock(return_value={"findings": [], "could_not_run": False})
_bind(svc, "_run_conventions_validator", validator)
await svc.conventions_check_for_task(
uuid4(), _task_with_id("feature/backend/abc"), timeout=30.0
)
validator.assert_awaited_once_with(Path("/tmp/wt"), ["a.py"], timeout=30.0)
@pytest.mark.asyncio
async def test_conventions_check_for_task_default_timeout_is_none() -> None:
"""The fail-closed callers (i_am_done's ``_conventions_gate``, pr_pass's
``_conventions_guard``) never pass ``timeout`` confirming the default
forwards ``None`` so ``_run_conventions_validator`` falls back to its
hardcoded fail-closed cap, unchanged."""
svc = _service()
_bind(svc, "_workspace_for_branch", AsyncMock(return_value=Path("/tmp/ws")))
_bind(svc, "list_changed_files", AsyncMock(return_value=["a.py"]))
_bind(svc, "_worktree_for_task", MagicMock(return_value=Path("/tmp/wt")))
_bind(svc, "_ensure_worktree_for_commit", AsyncMock(return_value=None))
validator = AsyncMock(return_value={"findings": [], "could_not_run": False})
_bind(svc, "_run_conventions_validator", validator)
await svc.conventions_check_for_task(uuid4(), _task_with_id("feature/backend/abc"))
validator.assert_awaited_once_with(Path("/tmp/wt"), ["a.py"], timeout=None)
@@ -158,7 +158,7 @@ async def test_conventions_check_runs_validator_in_worktree_not_clone() -> None:
captured: list[Path] = []
async def _capture_validator(
workspace: Path, _files: list[str]
workspace: Path, _files: list[str], **_kwargs: object
) -> dict[str, object]:
captured.append(Path(workspace))
return {"findings": [], "could_not_run": False}
@@ -169,7 +169,6 @@ async def test_create_rejects_cell_pm_assignee_plus_code() -> None:
@pytest.mark.asyncio
async def test_create_allows_cell_pm_assignee_plus_planning() -> None:
be_pm_uuid = AGENTS["be-pm"].uuid
svc = TaskService(
MagicMock(add=MagicMock(), flush=AsyncMock(), execute=AsyncMock())
@@ -0,0 +1,132 @@
"""fetch_branch_for_inspection's subprocess_timeout override (adversarial-review
round-2 fix 3): the fetch subprocess must self-bound near the caller's own leg
budget instead of running up to workspace_clone_timeout (300s) on the shared
DEFAULT asyncio executor (asyncio.to_thread, not git.py's dedicated
_GIT_EXECUTOR) after the caller has already given up waiting on it. A
timeout there now raises GitTimeoutError (mirroring _run_git's own
TimeoutExpired -> GitTimeoutError conversion in git.py), not a raw
subprocess.TimeoutExpired, so a bounded caller (run_bounded_leg) catches it
the same way as every other git-touching leg.
The clone-CREATION step (ensure_workspace, stubbed out here) is untouched by
this fix and keeps its own workspace_clone_timeout (300s) unconditionally
these tests isolate the FETCH subprocess only.
"""
from __future__ import annotations
import subprocess
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4
import pytest
from roboco.config import settings
from roboco.exceptions import GitTimeoutError
from roboco.services.workspace import WorkspaceService
if TYPE_CHECKING:
from pathlib import Path
# Named constant to satisfy ruff PLR2004 (magic value in comparison).
_EXPECTED_TIMEOUT_SECONDS = 5
def _service() -> WorkspaceService:
session = MagicMock()
session.execute = AsyncMock()
return WorkspaceService(session)
def _bind(svc: WorkspaceService, name: str, value: object) -> None:
"""Stub `name` on `svc` without tripping mypy's method-assign check."""
object.__setattr__(svc, name, value)
def _wire_resolution(svc: WorkspaceService, workspace: Path) -> None:
"""Stub the resolution steps ahead of the fetch subprocess so only the
fetch itself is under test."""
_bind(svc, "_resolve_branch_to_project_slug", AsyncMock(return_value="roboco"))
_bind(svc, "ensure_workspace", AsyncMock(return_value=workspace))
def _no_project_service_patch() -> Any:
"""No git token (project=None skips the decrypt path entirely)."""
return patch(
"roboco.services.project.get_project_service",
return_value=MagicMock(get_by_slug=AsyncMock(return_value=None)),
)
@pytest.mark.asyncio
async def test_default_subprocess_timeout_is_workspace_clone_timeout(
tmp_path: Path,
) -> None:
"""Every EXISTING caller omits subprocess_timeout — behavior byte-for-byte
unchanged: the fetch subprocess keeps the 300s workspace_clone_timeout."""
svc = _service()
_wire_resolution(svc, tmp_path)
captured: list[object] = []
def _fake_run(*_args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
captured.append(kwargs.get("timeout"))
return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="")
with (
patch("roboco.services.workspace.subprocess.run", side_effect=_fake_run),
patch("roboco.services.workspace._ensure_agent_owned"),
_no_project_service_patch(),
):
await svc.fetch_branch_for_inspection(agent_id=uuid4(), branch_name="feature/x")
assert captured == [settings.workspace_clone_timeout]
@pytest.mark.asyncio
async def test_subprocess_timeout_override_reaches_the_fetch(tmp_path: Path) -> None:
svc = _service()
_wire_resolution(svc, tmp_path)
captured: list[object] = []
def _fake_run(*_args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
captured.append(kwargs.get("timeout"))
return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="")
with (
patch("roboco.services.workspace.subprocess.run", side_effect=_fake_run),
patch("roboco.services.workspace._ensure_agent_owned"),
_no_project_service_patch(),
):
await svc.fetch_branch_for_inspection(
agent_id=uuid4(), branch_name="feature/x", subprocess_timeout=12.5
)
assert captured == [12.5]
@pytest.mark.asyncio
async def test_timeout_expired_becomes_git_timeout_error(tmp_path: Path) -> None:
"""Mirrors _run_git's own TimeoutExpired -> GitTimeoutError conversion
(git.py) so run_bounded_leg catches this uniformly with every other
git-touching leg, instead of a raw subprocess.TimeoutExpired propagating
uncaught to the RobocoError handler."""
svc = _service()
_wire_resolution(svc, tmp_path)
def _fake_run(*_args: object, **kwargs: object) -> subprocess.CompletedProcess[str]:
timeout = kwargs.get("timeout")
raise subprocess.TimeoutExpired(
cmd="git fetch",
timeout=float(timeout) if isinstance(timeout, int | float) else 0,
)
with (
patch("roboco.services.workspace.subprocess.run", side_effect=_fake_run),
patch("roboco.services.workspace._ensure_agent_owned"),
_no_project_service_patch(),
pytest.raises(GitTimeoutError) as exc_info,
):
await svc.fetch_branch_for_inspection(
agent_id=uuid4(), branch_name="feature/x", subprocess_timeout=5.0
)
assert exc_info.value.timeout == _EXPECTED_TIMEOUT_SECONDS
+1 -1
View File
@@ -350,7 +350,7 @@ def test_issue_agent_token_with_ttl_uses_expiring_format(
assert payload["id"] == "be-dev-1"
assert payload["role"] == "developer"
assert payload["team"] == "backend"
assert isinstance(payload["iat"], (int, float))
assert isinstance(payload["iat"], int | float)
assert payload["exp"] == payload["iat"] + 3600