mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(gateway): bounded fail-open evidence assembly + PM decision transient-failure bypass
Evidence-assembly git legs (diff, changed-files, branch fetch, advisory conventions run) ran unbounded inside claim_review / claim_doc_task / claim_gate_review / evidence() / i_am_done's envelope build, so a slow clone turned the whole verb into a silent 120s FlowVerbTimeout 504. Each leg now runs through run_bounded_leg under a shared LegBudget (evidence_assembly_timeout_seconds, 45s total): a timed-out leg — both asyncio TimeoutError and git's own GitTimeoutError — degrades into an evidence_gaps note on the envelope instead of hanging the verb, while non-timeout git errors still propagate. The advisory conventions run gets an inner-only timeout (conventions_validator_advisory_timeout_ seconds, 30s) threaded down to the subprocess so it is never orphaned by an outer cancel; the fail-closed i_am_done/pr_pass conventions gates keep their hardcoded 120s. _ensure_pm_decision now reports a PmDecisionOutcome: a transient DB failure recording the PM's decision journal (e.g. lock timeout under load) no longer launders into a journal:decision gate rejection that escalates and BLOCKS the task — the verb's own rationale satisfies the gate with a structured warning, across all seven PM verbs. Also: repo-wide ruff realignment to the lockfile-pinned ruff (8 format-only diffs, 14 UP038 isinstance conversions) that a transiently newer venv ruff had masked. Gate: 15474 passed, 459 skipped; ruff/mypy/xenon/vulture/bandit/ pip-audit/deptry/import-linter/foundation-check all green.
This commit is contained in:
@@ -978,18 +978,27 @@ async def test_escalate_up_survives_journal_write_lock_timeout() -> None:
|
||||
with no rollback/savepoint, poisoning the session so the very next
|
||||
attribute touch (``_escalate_up_preflight`` reading ``t.id``) raised an
|
||||
unhandled ``PendingRollbackError``. The write is now savepoint-guarded
|
||||
(``begin_nested()``): the failure is contained, the verb falls through
|
||||
cleanly to the normal tracing_gap rejection (no decision was actually
|
||||
persisted), and the task stays fully readable — no unhandled exception
|
||||
escapes ``escalate_up``."""
|
||||
(``begin_nested()``): the failure is contained and no unhandled
|
||||
exception escapes ``escalate_up``.
|
||||
|
||||
Round-2 fix (transient-failure gate bypass): a lock-timeout with a
|
||||
non-empty rationale already in hand (``reason``) no longer falls through
|
||||
to a tracing_gap rejection either — the PM answered the gate's actual
|
||||
question (why); the write was only ever a convenience. Laundering
|
||||
transient DB congestion into a rejected/blocked PM verb is exactly the
|
||||
bug this closes — see test_pm_decision_transient_failure_* below for the
|
||||
gate-helper-level unit coverage.
|
||||
"""
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = MagicMock(id=task_id, status="blocked", assigned_to=pm_id, team="backend")
|
||||
after = MagicMock(**{**t.__dict__, "assigned_to": uuid4()})
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
role="cell_pm", escalation_target="main-pm"
|
||||
)
|
||||
task_svc.escalate.return_value = after
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
@@ -1006,11 +1015,11 @@ async def test_escalate_up_survives_journal_write_lock_timeout() -> None:
|
||||
# The savepoint was actually engaged — proves the fix is wired in, not
|
||||
# merely that AsyncMock happened to swallow the raise on its own.
|
||||
task_svc.session.begin_nested.assert_called()
|
||||
# No unhandled exception escaped escalate_up: the gate falls through to
|
||||
# its normal clean rejection since the decision write never landed.
|
||||
# No unhandled exception escaped escalate_up, AND the gate is satisfied
|
||||
# by the verb's own rationale instead of rejecting a DB hiccup.
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "tracing_gap"
|
||||
assert "journal:decision" in body["missing"]
|
||||
assert body["error"] is None, body
|
||||
task_svc.escalate.assert_awaited_once()
|
||||
# The task is still fully readable afterward — this is exactly where
|
||||
# the production trace crashed with PendingRollbackError on t.id.
|
||||
assert t.id == task_id
|
||||
|
||||
@@ -109,3 +109,18 @@ async def test_gate_records_findings_even_when_blocking(
|
||||
env = await c._conventions_gate(_ctx())
|
||||
assert env is not None # still blocks
|
||||
assert recorded and recorded[0] is _BLOCK_RESULT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gate_never_overrides_the_fail_closed_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""i_am_done's conventions gate is fail-closed and must keep the
|
||||
validator's hardcoded 120s cap — unlike claim_review's advisory path, it
|
||||
must never pass a ``timeout`` override down to ``conventions_check_for_task``."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
c = _make_choreographer(check_result={"findings": [], "could_not_run": False})
|
||||
await c._conventions_gate(_ctx())
|
||||
check = c.git.conventions_check_for_task
|
||||
check.assert_awaited_once()
|
||||
assert "timeout" not in check.await_args.kwargs
|
||||
|
||||
@@ -125,3 +125,18 @@ async def test_pr_pass_guard_inert_when_flag_off(
|
||||
monkeypatch.setattr(settings, "conventions_enabled", False)
|
||||
c = _make_choreographer(check_result=_BLOCK_RESULT)
|
||||
assert await c._conventions_guard(uuid4(), MagicMock(), {}) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pr_pass_guard_never_overrides_the_fail_closed_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pr_pass gate is fail-closed and must keep the validator's
|
||||
hardcoded 120s cap — unlike claim_review's advisory path, it must
|
||||
never pass a ``timeout`` override down to ``conventions_check_for_task``."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
c = _make_choreographer(check_result={"findings": [], "could_not_run": False})
|
||||
await c._conventions_guard(uuid4(), MagicMock(), {})
|
||||
check = c.git.conventions_check_for_task
|
||||
check.assert_awaited_once()
|
||||
assert "timeout" not in check.await_args.kwargs
|
||||
|
||||
@@ -31,7 +31,12 @@ async def test_findings_surfaced_when_flag_on(monkeypatch: pytest.MonkeyPatch) -
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
findings = [{"file": "x.py", "line": 1, "level": "warn", "fix_hint": "h"}]
|
||||
c = _make_choreographer(check_result={"findings": findings, "could_not_run": False})
|
||||
assert await c._qa_convention_findings(uuid4(), MagicMock()) == findings
|
||||
gaps: list[str] = []
|
||||
assert (
|
||||
await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
|
||||
== findings
|
||||
)
|
||||
assert gaps == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -40,21 +45,32 @@ async def test_empty_when_flag_off(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
c = _make_choreographer(
|
||||
check_result={"findings": [{"file": "x"}], "could_not_run": False}
|
||||
)
|
||||
assert await c._qa_convention_findings(uuid4(), MagicMock()) == []
|
||||
gaps: list[str] = []
|
||||
assert (
|
||||
await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
|
||||
== []
|
||||
)
|
||||
assert gaps == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_could_not_run_surfaced_as_single_entry(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A non-timeout could_not_run reason ("boom") stays fail-open in
|
||||
convention_findings but must NOT also spam evidence_gaps — only a
|
||||
detected timeout does (see test_claim_review_conventions_timeout_
|
||||
degrades_with_gap in test_evidence_assembly_bounded_legs.py)."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
c = _make_choreographer(
|
||||
check_result={"findings": [], "could_not_run": True, "reason": "boom"}
|
||||
)
|
||||
out = await c._qa_convention_findings(uuid4(), MagicMock())
|
||||
gaps: list[str] = []
|
||||
out = await c._qa_convention_findings(uuid4(), MagicMock(), timeout=30.0, gaps=gaps)
|
||||
assert len(out) == 1
|
||||
assert out[0]["could_not_run"] is True
|
||||
assert out[0]["reason"] == "boom"
|
||||
assert gaps == []
|
||||
|
||||
|
||||
def _stub_task() -> MagicMock:
|
||||
|
||||
@@ -0,0 +1,763 @@
|
||||
"""Bounded advisory-evidence legs on claim_review / claim_doc_task /
|
||||
claim_gate_review / evidence() / i_am_done's success envelope.
|
||||
|
||||
Live bug: claim-evidence assembly's slow legs (branch-fetch-backed diff,
|
||||
list_changed_files, the conventions-validator subprocess) had no per-leg
|
||||
budget, so a hung leg silently ate the whole ``flow_verb_timeout_seconds``
|
||||
(120s) and died as a FlowVerbTimeout 504 — holding every row the request
|
||||
touched for the duration. Fix: each slow leg runs bounded via
|
||||
``run_bounded_leg`` (``asyncio.wait_for``) against a SHARED ``LegBudget`` per
|
||||
build; a timeout skips that piece, records a human-readable note in the
|
||||
evidence's ``evidence_gaps``, and lets the claim verb succeed with partial
|
||||
evidence instead of hanging.
|
||||
|
||||
Adversarial-review follow-up (round 2) covers four confirmed gaps:
|
||||
1. ``run_bounded_leg`` must catch ``GitTimeoutError`` too (``_run_git``'s own
|
||||
internal subprocess bound — NOT a ``TimeoutError`` subclass, and usually
|
||||
the FIRST bound to trip since it defaults to 30s, shorter than a leg's
|
||||
own budget) — every timeout-shaped test below is parametrized over both
|
||||
exception shapes.
|
||||
2. The conventions leg no longer wraps ``_qa_convention_findings`` in an
|
||||
outer ``run_bounded_leg`` — that raced ``conventions_check_for_task``'s
|
||||
own inner timeout+cleanup and leaked the validator subprocess. It now
|
||||
self-bounds via the ``timeout`` kwarg alone and reports its own gap.
|
||||
3. ``fetch_branch_for_inspection`` takes an optional ``subprocess_timeout``
|
||||
so its fetch subprocess self-bounds near the leg's own budget instead of
|
||||
occupying a thread on the shared default executor for up to 300s.
|
||||
4. A shared ``LegBudget`` (one per evidence build) makes every leg's
|
||||
``wait_for`` draw from one TOTAL budget instead of getting its own full
|
||||
allotment — summed per-leg budgets can no longer exceed the total.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.exceptions import GitCommandError, GitTimeoutError
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from roboco.services.gateway.choreographer.evidence_legs import (
|
||||
LegBudget,
|
||||
run_bounded_leg,
|
||||
)
|
||||
|
||||
# Every timeout-shaped test is parametrized over both real timeout shapes:
|
||||
# asyncio's own cancellation-converted TimeoutError, and GitTimeoutError
|
||||
# (GitService._run_git's own internal subprocess bound — a GitError/
|
||||
# RobocoError subclass, NOT a TimeoutError subclass, and the most common
|
||||
# real-world single-hung-git-call shape since it defaults to a SHORTER
|
||||
# window, 30s, than a leg's own budget).
|
||||
_TIMEOUT_EXCEPTIONS = (
|
||||
TimeoutError("hung"),
|
||||
GitTimeoutError("git diff", 30),
|
||||
)
|
||||
_TIMEOUT_IDS = ("asyncio_timeout", "git_timeout")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_bounded_leg / LegBudget themselves
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_bounded_leg_passes_through_on_success() -> None:
|
||||
async def fast() -> str:
|
||||
return "value"
|
||||
|
||||
gaps: list[str] = []
|
||||
result = await run_bounded_leg(
|
||||
fast(),
|
||||
default="fallback",
|
||||
budget=LegBudget(5.0),
|
||||
leg="unit leg",
|
||||
hint="check manually",
|
||||
task_id=uuid4(),
|
||||
gaps=gaps,
|
||||
)
|
||||
assert result == "value"
|
||||
assert gaps == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_run_bounded_leg_degrades_to_default_on_timeout(exc: Exception) -> None:
|
||||
async def hangs() -> str:
|
||||
raise exc
|
||||
|
||||
gaps: list[str] = []
|
||||
result = await run_bounded_leg(
|
||||
hangs(),
|
||||
default="fallback",
|
||||
budget=LegBudget(5.0),
|
||||
leg="unit leg",
|
||||
hint="check manually",
|
||||
task_id=uuid4(),
|
||||
gaps=gaps,
|
||||
)
|
||||
assert result == "fallback"
|
||||
assert len(gaps) == 1
|
||||
assert "unit leg unavailable" in gaps[0]
|
||||
assert "check manually" in gaps[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_bounded_leg_actually_bounds_a_slow_coroutine() -> None:
|
||||
"""A genuinely slow (not pre-raised) coroutine is cancelled at the
|
||||
budget's deadline, not awaited to completion."""
|
||||
|
||||
async def slow() -> str:
|
||||
await asyncio.sleep(10)
|
||||
return "too late"
|
||||
|
||||
gaps: list[str] = []
|
||||
result = await run_bounded_leg(
|
||||
slow(),
|
||||
default="fallback",
|
||||
budget=LegBudget(0.05),
|
||||
leg="unit leg",
|
||||
hint="check manually",
|
||||
task_id=uuid4(),
|
||||
gaps=gaps,
|
||||
)
|
||||
assert result == "fallback"
|
||||
assert len(gaps) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_bounded_leg_other_git_error_still_propagates() -> None:
|
||||
"""A real command failure (not a timeout) is NOT a degrade case — it
|
||||
must still propagate uncaught, same as any other unexpected exception."""
|
||||
|
||||
async def fails() -> str:
|
||||
raise GitCommandError("git diff", "fatal: bad revision")
|
||||
|
||||
gaps: list[str] = []
|
||||
with pytest.raises(GitCommandError):
|
||||
await run_bounded_leg(
|
||||
fails(),
|
||||
default="fallback",
|
||||
budget=LegBudget(5.0),
|
||||
leg="unit leg",
|
||||
hint="check manually",
|
||||
task_id=uuid4(),
|
||||
gaps=gaps,
|
||||
)
|
||||
assert gaps == []
|
||||
|
||||
|
||||
def test_leg_budget_remaining_shrinks_over_time() -> None:
|
||||
# Total well above _MIN_LEG_SECONDS (1.0) so the floor never engages
|
||||
# here — otherwise both readings would clamp to 1.0 and look equal.
|
||||
budget = LegBudget(3.0)
|
||||
first = budget.remaining()
|
||||
time.sleep(0.1)
|
||||
second = budget.remaining()
|
||||
assert second < first
|
||||
assert first == pytest.approx(3.0, abs=0.05)
|
||||
assert (first - second) == pytest.approx(0.1, abs=0.05)
|
||||
|
||||
|
||||
def test_leg_budget_floors_at_minimum() -> None:
|
||||
"""A budget already past its deadline still yields a positive window
|
||||
(the floor) rather than 0 or a negative timeout — a leg always gets a
|
||||
real chance to run, even a badly-exhausted one."""
|
||||
budget = LegBudget(0.01)
|
||||
time.sleep(0.05)
|
||||
assert budget.remaining() == pytest.approx(1.0, abs=0.05)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_bounded_leg_shares_one_shrinking_budget_across_legs() -> None:
|
||||
"""Three legs sharing ONE LegBudget: the first two finish fast and
|
||||
consume real budget; the last two never finish on their own and get
|
||||
progressively SMALLER windows (shrinking, not each getting the full
|
||||
total) — both record their own gap, and total wall time stays bounded
|
||||
near the shared total instead of the naive per-leg sum (0.1+0.1+5+5s).
|
||||
"""
|
||||
budget = LegBudget(1.2)
|
||||
gaps: list[str] = []
|
||||
|
||||
async def _takes(seconds: float) -> str:
|
||||
await asyncio.sleep(seconds)
|
||||
return "done"
|
||||
|
||||
start = time.monotonic()
|
||||
remaining_before_1 = budget.remaining()
|
||||
r1 = await run_bounded_leg(
|
||||
_takes(0.1),
|
||||
default="gap1",
|
||||
budget=budget,
|
||||
leg="leg1",
|
||||
hint="h",
|
||||
task_id="t",
|
||||
gaps=gaps,
|
||||
)
|
||||
remaining_before_2 = budget.remaining()
|
||||
r2 = await run_bounded_leg(
|
||||
_takes(5.0),
|
||||
default="gap2",
|
||||
budget=budget,
|
||||
leg="leg2",
|
||||
hint="h",
|
||||
task_id="t",
|
||||
gaps=gaps,
|
||||
)
|
||||
remaining_before_3 = budget.remaining()
|
||||
r3 = await run_bounded_leg(
|
||||
_takes(5.0),
|
||||
default="gap3",
|
||||
budget=budget,
|
||||
leg="leg3",
|
||||
hint="h",
|
||||
task_id="t",
|
||||
gaps=gaps,
|
||||
)
|
||||
elapsed = time.monotonic() - start
|
||||
expected_gap_count = 2 # leg2 + leg3 both timed out; leg1 completed
|
||||
|
||||
assert r1 == "done"
|
||||
assert r2 == "gap2"
|
||||
assert r3 == "gap3"
|
||||
assert len(gaps) == expected_gap_count
|
||||
assert "leg2" in gaps[0]
|
||||
assert "leg3" in gaps[1]
|
||||
# Each leg's own remaining() reading is strictly smaller than the last
|
||||
# — the shared deadline never resets.
|
||||
assert remaining_before_1 > remaining_before_2 > remaining_before_3
|
||||
# ponytail: the floor (max 1.0s) can inflate the LAST leg's window past
|
||||
# what a naive "budget minus elapsed" would give once the deadline is
|
||||
# already exhausted — a single floor engagement caps the worst-case
|
||||
# overage at _MIN_LEG_SECONDS (1.0s), so budget total + ~1.3s is a safe,
|
||||
# honest ceiling rather than a strict `<= budget` bound. Upgrade path:
|
||||
# make the floor configurable if a caller ever needs a tighter cap.
|
||||
budget_total_seconds = 1.2
|
||||
floor_overage_tolerance_seconds = 1.3
|
||||
assert elapsed <= budget_total_seconds + floor_overage_tolerance_seconds
|
||||
# And it's nowhere near the naive per-leg-gets-its-own-full-timeout sum
|
||||
# (0.1 + 5.0 + 5.0 = 10.1s) that pre-LegBudget behavior would produce.
|
||||
naive_per_leg_sum_seconds = 5.0
|
||||
assert elapsed < naive_per_leg_sum_seconds
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared choreographer test harness (mirrors test_choreographer_qa.py /
|
||||
# test_claim_doc_task_checkout.py / test_claim_gate_review_guards.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
base: dict[str, Any] = {
|
||||
"task": AsyncMock(),
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
"list_unread_mentions",
|
||||
"list_pending_notifications",
|
||||
"task_metadata_gaps",
|
||||
"recent_team_activity",
|
||||
"blockers_in_lane",
|
||||
"journal_highlights_for_task",
|
||||
):
|
||||
getattr(repo, method).return_value = []
|
||||
_ldef = base["journal"].latest_decision_at.return_value
|
||||
if type(_ldef).__name__ in ("MagicMock", "AsyncMock"):
|
||||
base["journal"].latest_decision_at.return_value = datetime.now(UTC)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
def _stub_empty_ledger(session: MagicMock) -> None:
|
||||
session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# claim_review (qa.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PR_NUMBER = 8
|
||||
_PR_URL = "https://github.com/x/y/pull/8"
|
||||
|
||||
|
||||
def _qa_task(task_id: Any) -> MagicMock:
|
||||
return MagicMock(
|
||||
id=task_id,
|
||||
status="awaiting_qa",
|
||||
assigned_to=None,
|
||||
pr_number=_PR_NUMBER,
|
||||
pr_url=_PR_URL,
|
||||
commits=[{"sha": "abc123", "message": "feat: x"}],
|
||||
team="backend",
|
||||
branch_name="feature/backend/abc--def",
|
||||
work_session_id=uuid4(),
|
||||
documents=[],
|
||||
dev_notes="implemented x",
|
||||
acceptance_criteria=["AC1"],
|
||||
acceptance_criteria_status=[
|
||||
{"criterion": "AC1", "referencing_artifact_id": "abc123"},
|
||||
],
|
||||
parent_task_id=None,
|
||||
)
|
||||
|
||||
|
||||
def _qa_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
|
||||
"""Does NOT touch ``settings.conventions_enabled`` — callers that care
|
||||
set it themselves via their own ``monkeypatch`` fixture; a shared
|
||||
forced-False here would silently clobber a caller's forced-True set
|
||||
moments earlier (``monkeypatch.setattr`` doesn't stack, last write
|
||||
wins), which is exactly what broke the conventions-specific tests."""
|
||||
qa_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t_initial = _qa_task(task_id)
|
||||
t_claimed = MagicMock(**{**t_initial.__dict__, "assigned_to": qa_id})
|
||||
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t_initial
|
||||
task_svc.agent_for.return_value = MagicMock(role="qa", team="backend")
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.qa_claim.return_value = t_claimed
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
return Choreographer(deps), qa_id, task_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_claim_review_diff_timeout_degrades_with_gap(
|
||||
monkeypatch: pytest.MonkeyPatch, exc: Exception
|
||||
) -> None:
|
||||
"""A hung git.diff on claim_review must not hang the verb: it degrades
|
||||
to an empty diff, records the gap, and the OTHER leg (list_changed_files)
|
||||
still comes through untouched."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", False)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.side_effect = exc
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, qa_id, task_id = _qa_harness(git_svc)
|
||||
|
||||
env = await c.claim_review(qa_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == ""
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" in ev
|
||||
assert len(ev["evidence_gaps"]) == 1
|
||||
assert "pr diff unavailable" in ev["evidence_gaps"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_review_conventions_timeout_degrades_with_gap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""conventions_check_for_task's OWN internal timeout (proc.kill()'d and
|
||||
reaped inside git.py, never raising) surfaces as could_not_run=True with
|
||||
a "timed out" reason — not an exception. The advisory call site
|
||||
(_qa_convention_findings) detects that shape and records the gap
|
||||
itself; NO outer run_bounded_leg wraps this leg (that's the fix — see
|
||||
module docstring point 2)."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
git_svc.conventions_check_for_task.return_value = {
|
||||
"findings": [],
|
||||
"could_not_run": True,
|
||||
"reason": "validator timed out after 30.0s",
|
||||
}
|
||||
c, qa_id, task_id = _qa_harness(git_svc)
|
||||
|
||||
env = await c.claim_review(qa_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert ev["convention_findings"] == [
|
||||
{"could_not_run": True, "reason": "validator timed out after 30.0s"}
|
||||
]
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("conventions findings unavailable" in g for g in ev["evidence_gaps"])
|
||||
# The advisory (shorter) ceiling reached the validator call, not the
|
||||
# fail-closed i_am_done/pr_pass default (None -> hardcoded 120s).
|
||||
git_svc.conventions_check_for_task.assert_awaited_once()
|
||||
call_kwargs = git_svc.conventions_check_for_task.await_args.kwargs
|
||||
assert (
|
||||
call_kwargs["timeout"]
|
||||
<= settings.conventions_validator_advisory_timeout_seconds
|
||||
)
|
||||
assert call_kwargs["timeout"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_review_conventions_non_timeout_could_not_run_no_gap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A genuine resolution failure (not a timeout) still surfaces in
|
||||
convention_findings (existing fail-open shape) but must NOT also spam
|
||||
evidence_gaps — that's reserved for actual degraded-advisory-leg notes."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
git_svc.conventions_check_for_task.return_value = {
|
||||
"findings": [],
|
||||
"could_not_run": True,
|
||||
"reason": "resolution failed: NotFoundError: Branch not found",
|
||||
}
|
||||
c, qa_id, task_id = _qa_harness(git_svc)
|
||||
|
||||
env = await c.claim_review(qa_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["convention_findings"][0]["could_not_run"] is True
|
||||
assert "evidence_gaps" not in ev
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_qa_convention_findings_not_cancelled_by_outer_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Regression for the orphaned-subprocess bug: even with a tiny shared
|
||||
evidence-assembly budget, _qa_convention_findings must NOT be cut short
|
||||
by an outer wait_for — it awaits conventions_check_for_task to
|
||||
completion. A slow-but-real mock (0.15s) run against a budget whose
|
||||
total is far smaller (0.02s) proves there is no outer wrap: if there
|
||||
still were one, this would return the default/empty shape instead of
|
||||
the real result."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", True)
|
||||
|
||||
async def _slow_check(*_args: object, **_kwargs: object) -> dict[str, Any]:
|
||||
await asyncio.sleep(0.15)
|
||||
return {"findings": [{"file": "x.py", "line": 1}], "could_not_run": False}
|
||||
|
||||
git_svc = AsyncMock()
|
||||
git_svc.conventions_check_for_task.side_effect = _slow_check
|
||||
c, _qa_id, _task_id = _qa_harness(git_svc)
|
||||
cc: Any = c
|
||||
|
||||
gaps: list[str] = []
|
||||
result = await cc._qa_convention_findings(
|
||||
uuid4(), MagicMock(), timeout=0.02, gaps=gaps
|
||||
)
|
||||
assert result == [{"file": "x.py", "line": 1}]
|
||||
assert gaps == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_review_normal_path_has_no_evidence_gaps(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Byte-for-byte unchanged normal path: no evidence_gaps key at all when
|
||||
nothing times out."""
|
||||
monkeypatch.setattr(settings, "conventions_enabled", False)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, qa_id, task_id = _qa_harness(git_svc)
|
||||
|
||||
env = await c.claim_review(qa_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" not in ev
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# claim_doc_task (doc.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _doc_task(task_id: Any, branch: str) -> MagicMock:
|
||||
return MagicMock(
|
||||
id=task_id,
|
||||
status="awaiting_documentation",
|
||||
assigned_to=None,
|
||||
task_type="documentation",
|
||||
team="backend",
|
||||
branch_name=branch,
|
||||
quick_context=None,
|
||||
documents=[],
|
||||
commits=[{"sha": "abc123", "message": "[x] work"}],
|
||||
pr_number=7,
|
||||
pr_url="https://github.com/x/y/pull/7",
|
||||
dev_notes="done",
|
||||
acceptance_criteria_status=[],
|
||||
work_session_id=uuid4(),
|
||||
)
|
||||
|
||||
|
||||
def _doc_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
|
||||
doc_id = uuid4()
|
||||
task_id = uuid4()
|
||||
branch = "feature/backend/root1234--cellpm56--dev78901"
|
||||
t_initial = _doc_task(task_id, branch)
|
||||
t_claimed = MagicMock(**{**t_initial.__dict__, "assigned_to": doc_id})
|
||||
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t_initial
|
||||
task_svc.agent_for.return_value = MagicMock(role="documenter", team="backend")
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.doc_claim.return_value = t_claimed
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
return Choreographer(deps), doc_id, task_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_claim_doc_task_diff_timeout_degrades_with_gap(exc: Exception) -> None:
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.side_effect = exc
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, doc_id, task_id = _doc_harness(git_svc)
|
||||
|
||||
env = await c.claim_doc_task(doc_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == ""
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_claim_doc_task_checkout_timeout_degrades_with_gap(
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
"""The checkout leg (run before evidence assembly) also degrades bounded
|
||||
instead of an unbounded suppress(Exception) — its gap folds into the
|
||||
same evidence_gaps list the diff/list_changed_files legs use, drawing
|
||||
from the SAME shared LegBudget."""
|
||||
git_svc = AsyncMock()
|
||||
git_svc.checkout_branch_in_agent_workspace.side_effect = exc
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, doc_id, task_id = _doc_harness(git_svc)
|
||||
|
||||
env = await c.claim_doc_task(doc_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
# The other legs are untouched by the checkout's own timeout.
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("workspace checkout unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_doc_task_normal_path_has_no_evidence_gaps() -> None:
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, doc_id, task_id = _doc_harness(git_svc)
|
||||
|
||||
env = await c.claim_doc_task(doc_id, task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" not in ev
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# claim_gate_review (pr_gate.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _gate_task() -> MagicMock:
|
||||
return MagicMock(
|
||||
id=uuid4(),
|
||||
status="awaiting_pr_review",
|
||||
assigned_to=uuid4(),
|
||||
parent_task_id=None,
|
||||
task_type="planning",
|
||||
dependency_ids=[],
|
||||
team="main_pm",
|
||||
pr_number=139,
|
||||
pr_url="https://example/pr/139",
|
||||
branch_name="feature/main_pm/root",
|
||||
batch_id=None,
|
||||
description=None,
|
||||
acceptance_criteria=[],
|
||||
)
|
||||
|
||||
|
||||
def _gate_harness(git_svc: AsyncMock) -> tuple[Choreographer, Any, Any]:
|
||||
task_svc = AsyncMock()
|
||||
t = _gate_task()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
role="pr_reviewer", slug="be-pr-reviewer"
|
||||
)
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.unmet_dependency_ids = AsyncMock(return_value=[])
|
||||
task_svc.has_earlier_incomplete_code_sibling.return_value = False
|
||||
task_svc.pr_gate_claim = AsyncMock(return_value=t)
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
return Choreographer(deps), t, uuid4()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_claim_gate_review_diff_timeout_degrades_with_gap(exc: Exception) -> None:
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.side_effect = exc
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, t, reviewer_id = _gate_harness(git_svc)
|
||||
|
||||
env = await c.claim_gate_review(reviewer_id, t.id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff"] == ""
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_gate_review_files_changed_timeout_degrades_with_gap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""list_changed_files hanging must not sink the whole gate claim, and the
|
||||
diff leg (which succeeded) must remain intact in the evidence.
|
||||
|
||||
``_gate_changed_files`` has its own internal ``except Exception`` (an
|
||||
existing hard-failure fail-open, untouched by this fix) that would
|
||||
swallow a synchronously-raised exception (of either timeout shape)
|
||||
before the outer ``run_bounded_leg`` ever saw it — so this uses a
|
||||
genuinely slow coroutine + a monkeypatched short budget to exercise the
|
||||
real cancel-at-the-wall path (``asyncio.wait_for`` cancelling the
|
||||
awaited task via ``CancelledError``, which that ``except Exception``
|
||||
does NOT catch), matching what a real hang does in production.
|
||||
"""
|
||||
monkeypatch.setattr(settings, "evidence_assembly_timeout_seconds", 0.02)
|
||||
|
||||
async def _hangs(*_args: object, **_kwargs: object) -> list[str]:
|
||||
await asyncio.sleep(5)
|
||||
return ["should-not-be-reached"]
|
||||
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.side_effect = _hangs
|
||||
c, t, reviewer_id = _gate_harness(git_svc)
|
||||
|
||||
env = await c.claim_gate_review(reviewer_id, t.id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff"] == "diff content"
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("files_changed unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_gate_review_normal_path_has_no_evidence_gaps() -> None:
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
c, t, reviewer_id = _gate_harness(git_svc)
|
||||
|
||||
env = await c.claim_gate_review(reviewer_id, t.id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff"] == "diff content"
|
||||
assert "evidence_gaps" not in ev
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_i_am_done_ok (i_am_done's success-envelope evidence). Runs strictly
|
||||
# AFTER the composed transition already committed — advisory, not gating —
|
||||
# so its list_changed_files leg is bounded exactly like the claim paths.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _done_task(task_id: Any) -> MagicMock:
|
||||
return MagicMock(
|
||||
id=task_id,
|
||||
branch_name="feature/backend/abc",
|
||||
commits=[{"sha": "abc123", "message": "x"}],
|
||||
dev_notes="done",
|
||||
acceptance_criteria_status=[],
|
||||
pr_number=5,
|
||||
pr_url="https://github.com/x/y/pull/5",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_build_i_am_done_ok_files_changed_timeout_degrades_with_gap(
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
"""A hung list_changed_files leg in i_am_done's already-committed
|
||||
success-envelope builder must not hang the dev's response — it degrades
|
||||
to an empty files_changed and records the gap."""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = _done_task(task_id)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.list_changed_files.side_effect = exc
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c._build_i_am_done_ok(agent_id, task_id, t)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["files_changed"] == []
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("files_changed unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_i_am_done_ok_normal_path_has_no_evidence_gaps() -> None:
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = _done_task(task_id)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.agent_for.return_value = MagicMock(role="developer", team="backend")
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c._build_i_am_done_ok(agent_id, task_id, t)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" not in ev
|
||||
@@ -20,6 +20,8 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings
|
||||
from roboco.exceptions import GitTimeoutError
|
||||
from roboco.services.gateway.content_actions import ContentActions, ContentActionsDeps
|
||||
|
||||
|
||||
@@ -223,3 +225,143 @@ async def test_evidence_no_branch_skips_git_calls() -> None:
|
||||
assert body["evidence"]["pr_diff_summary"] == ""
|
||||
git_svc.diff.assert_not_awaited()
|
||||
git_svc.list_changed_files.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bounded advisory-evidence legs: evidence() must not hang on a slow branch
|
||||
# fetch / diff / list_changed_files leg — it degrades and records a note in
|
||||
# evidence_gaps instead (same run_bounded_leg treatment as claim_review /
|
||||
# claim_doc_task / claim_gate_review). Every timeout-shaped test is
|
||||
# parametrized over both real timeout shapes: asyncio's own
|
||||
# cancellation-converted TimeoutError, and GitTimeoutError (_run_git's own
|
||||
# internal subprocess bound — a GitError/RobocoError subclass, NOT a
|
||||
# TimeoutError subclass, and the most common real-world single-hung-git-call
|
||||
# shape since it defaults to a SHORTER window than a leg's own budget).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TIMEOUT_EXCEPTIONS = (
|
||||
TimeoutError("hung"),
|
||||
GitTimeoutError("git diff", 30),
|
||||
)
|
||||
_TIMEOUT_IDS = ("asyncio_timeout", "git_timeout")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_evidence_diff_timeout_degrades_with_gap(exc: Exception) -> None:
|
||||
"""A hung git.diff must not hang evidence(): it degrades to an empty
|
||||
diff, records the gap, and list_changed_files (the other leg) still
|
||||
comes through untouched."""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.side_effect = exc
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
workspace_svc = AsyncMock()
|
||||
evidence_repo = AsyncMock()
|
||||
evidence_repo.journal_highlights_for_task.return_value = []
|
||||
|
||||
ca = ContentActions(
|
||||
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
|
||||
)
|
||||
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == ""
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("pr diff unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exc", _TIMEOUT_EXCEPTIONS, ids=_TIMEOUT_IDS)
|
||||
async def test_evidence_branch_fetch_timeout_degrades_with_gap(exc: Exception) -> None:
|
||||
"""A hung workspace branch-fetch must not hang evidence() either — the
|
||||
subsequent diff/list_changed_files legs still run (against whatever the
|
||||
workspace already has) and the gap is recorded."""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
workspace_svc = AsyncMock()
|
||||
workspace_svc.fetch_branch_for_inspection.side_effect = exc
|
||||
evidence_repo = AsyncMock()
|
||||
evidence_repo.journal_highlights_for_task.return_value = []
|
||||
|
||||
ca = ContentActions(
|
||||
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
|
||||
)
|
||||
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" in ev
|
||||
assert any("branch fetch unavailable" in g for g in ev["evidence_gaps"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_branch_fetch_passes_subprocess_timeout_from_budget() -> None:
|
||||
"""The branch-fetch leg passes its own remaining LegBudget share down as
|
||||
fetch_branch_for_inspection's subprocess_timeout, so a hung fetch
|
||||
subprocess self-terminates near the leg's own budget instead of
|
||||
occupying a thread on the shared default executor for up to
|
||||
workspace_clone_timeout (300s) after evidence() already gave up on it."""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
workspace_svc = AsyncMock()
|
||||
workspace_svc.fetch_branch_for_inspection.return_value = None
|
||||
evidence_repo = AsyncMock()
|
||||
evidence_repo.journal_highlights_for_task.return_value = []
|
||||
|
||||
ca = ContentActions(
|
||||
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
|
||||
)
|
||||
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
|
||||
assert env.as_dict()["error"] is None
|
||||
|
||||
workspace_svc.fetch_branch_for_inspection.assert_awaited_once()
|
||||
call_kwargs = workspace_svc.fetch_branch_for_inspection.await_args.kwargs
|
||||
assert (
|
||||
call_kwargs["subprocess_timeout"] <= settings.evidence_assembly_timeout_seconds
|
||||
)
|
||||
assert call_kwargs["subprocess_timeout"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_normal_path_has_no_evidence_gaps() -> None:
|
||||
"""Byte-for-byte unchanged normal path: no evidence_gaps key at all when
|
||||
nothing times out."""
|
||||
agent_id = uuid4()
|
||||
task_id = uuid4()
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = _task_with_pr(task_id, commits=["abc"])
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
git_svc.list_changed_files.return_value = ["README.md"]
|
||||
workspace_svc = AsyncMock()
|
||||
evidence_repo = AsyncMock()
|
||||
evidence_repo.journal_highlights_for_task.return_value = []
|
||||
|
||||
ca = ContentActions(
|
||||
_deps_for_evidence(task_svc, git_svc, workspace_svc, evidence_repo)
|
||||
)
|
||||
env = await ca.evidence(agent_id=agent_id, task_id=task_id)
|
||||
body = env.as_dict()
|
||||
assert body["error"] is None, body
|
||||
ev = body["evidence"]
|
||||
assert ev["pr_diff_summary"] == "diff content"
|
||||
assert ev["files_changed"] == ["README.md"]
|
||||
assert "evidence_gaps" not in ev
|
||||
|
||||
@@ -7,6 +7,12 @@ gate runs. This removes the dominant stall where a loaded/weak-model PM
|
||||
forgot the separate note(scope='decision') call and looped on a
|
||||
tracing_gap → respawn. The gate itself is unchanged (see
|
||||
test_pm_decision_window.py) — this only ensures a fresh decision exists.
|
||||
|
||||
``_ensure_pm_decision`` returns a ``PmDecisionOutcome`` ("fresh" / "wrote" /
|
||||
"transient_failure" / "absent") the caller threads into the gate helper that
|
||||
runs right after (see test_pm_decision_transient_failure.py for the
|
||||
gate-satisfaction behavior itself) — these tests pin the outcome value for
|
||||
each branch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -54,8 +60,11 @@ async def test_writes_decision_when_none_exists() -> None:
|
||||
journal.latest_decision_at.return_value = None
|
||||
c = Choreographer(_make_deps(journal=journal))
|
||||
|
||||
await c._ensure_pm_decision(agent_id, task_id, "Merging PR #120; all ACs verified")
|
||||
outcome = await c._ensure_pm_decision(
|
||||
agent_id, task_id, "Merging PR #120; all ACs verified"
|
||||
)
|
||||
|
||||
assert outcome == "wrote"
|
||||
journal.write_decision.assert_awaited_once()
|
||||
_args, kwargs = journal.write_decision.call_args
|
||||
assert kwargs["agent_id"] == agent_id
|
||||
@@ -69,8 +78,9 @@ async def test_skips_when_fresh_decision_already_exists() -> None:
|
||||
journal.latest_decision_at.return_value = datetime.now(UTC) - timedelta(seconds=60)
|
||||
c = Choreographer(_make_deps(journal=journal))
|
||||
|
||||
await c._ensure_pm_decision(uuid4(), uuid4(), "rationale text here")
|
||||
outcome = await c._ensure_pm_decision(uuid4(), uuid4(), "rationale text here")
|
||||
|
||||
assert outcome == "fresh"
|
||||
journal.write_decision.assert_not_awaited()
|
||||
|
||||
|
||||
@@ -82,8 +92,11 @@ async def test_writes_when_existing_decision_is_stale() -> None:
|
||||
)
|
||||
c = Choreographer(_make_deps(journal=journal))
|
||||
|
||||
await c._ensure_pm_decision(uuid4(), uuid4(), "fresh rationale around this point")
|
||||
outcome = await c._ensure_pm_decision(
|
||||
uuid4(), uuid4(), "fresh rationale around this point"
|
||||
)
|
||||
|
||||
assert outcome == "wrote"
|
||||
journal.write_decision.assert_awaited_once()
|
||||
|
||||
|
||||
@@ -92,21 +105,29 @@ async def test_noop_on_empty_rationale() -> None:
|
||||
journal = AsyncMock()
|
||||
c = Choreographer(_make_deps(journal=journal))
|
||||
|
||||
await c._ensure_pm_decision(uuid4(), uuid4(), " ")
|
||||
await c._ensure_pm_decision(uuid4(), uuid4(), None)
|
||||
outcome_blank = await c._ensure_pm_decision(uuid4(), uuid4(), " ")
|
||||
outcome_none = await c._ensure_pm_decision(uuid4(), uuid4(), None)
|
||||
|
||||
assert outcome_blank == "absent"
|
||||
assert outcome_none == "absent"
|
||||
journal.latest_decision_at.assert_not_awaited()
|
||||
journal.write_decision.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swallows_write_failure_best_effort() -> None:
|
||||
"""A journal write failure must not crash the verb — the gate then
|
||||
rejects normally (the pre-fix behaviour), never a 500."""
|
||||
"""A journal write failure must not crash the verb. Round-2 fix: the
|
||||
outcome is "transient_failure" (not swallowed into a bare None) so the
|
||||
caller's gate can treat a DB hiccup as satisfied by the rationale
|
||||
already in hand — see test_pm_decision_transient_failure.py for the
|
||||
gate-satisfaction behavior itself."""
|
||||
journal = AsyncMock()
|
||||
journal.latest_decision_at.return_value = None
|
||||
journal.write_decision.side_effect = RuntimeError("db down")
|
||||
c = Choreographer(_make_deps(journal=journal))
|
||||
|
||||
# Must not raise.
|
||||
await c._ensure_pm_decision(uuid4(), uuid4(), "rationale that triggers a write")
|
||||
outcome = await c._ensure_pm_decision(
|
||||
uuid4(), uuid4(), "rationale that triggers a write"
|
||||
)
|
||||
assert outcome == "transient_failure"
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
"""PM-decision write-then-gate: transient DB failure must not launder into a
|
||||
durable rejection/block.
|
||||
|
||||
Live bug: every PM verb (complete / submit_up / submit_root / unblock /
|
||||
escalate_up / escalate_to_ceo / delegate) runs ``_ensure_pm_decision`` to
|
||||
auto-record its own rationale as a journal:decision before the freshness
|
||||
gate (``_check_pm_decision_required`` / ``_check_complete_gates`` /
|
||||
``_check_submit_up_gates``) runs. The write can lock-timeout under DB
|
||||
contention (a concurrent claim holding the task row's FK share lock); the
|
||||
old contract swallowed that and let the gate reject a "missing decision"
|
||||
the PM's own rationale already answered — a PM retries, keeps getting
|
||||
rejected while contention lasts, then escalates, and the task ends up
|
||||
BLOCKED. Transient congestion laundered into a durable blocked task.
|
||||
|
||||
Fix: ``_ensure_pm_decision`` returns a ``PmDecisionOutcome`` — "fresh" /
|
||||
"wrote" / "transient_failure" / "absent". Every gate helper accepts
|
||||
``pm_decision_outcome`` (default ``None`` = legacy behavior unchanged) and
|
||||
treats "transient_failure" as gate-satisfied for THIS call — the rationale
|
||||
is in the verb payload; the write was only ever a convenience. "absent" (no
|
||||
rationale at all) still rejects exactly as before.
|
||||
|
||||
These tests exercise the gate helpers directly (the leanest harness that
|
||||
reaches the actual decision-point) for precise, fast coverage of the
|
||||
mechanism, plus one near-real-call-site test per verb family
|
||||
(``_cell_pm_complete_guard``, ``escalate_up``) proving the real wiring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from roboco.services.gateway.choreographer import _impl as _impl_module
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
|
||||
def _make_choreographer(**overrides: Any) -> Choreographer:
|
||||
base: dict[str, Any] = {
|
||||
"task": AsyncMock(),
|
||||
"work_session": AsyncMock(),
|
||||
"git": AsyncMock(),
|
||||
"a2a": AsyncMock(),
|
||||
"journal": AsyncMock(),
|
||||
"audit": AsyncMock(),
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
return Choreographer(ChoreographerDeps(**base))
|
||||
|
||||
|
||||
_TRANSIENT_WARNING_SUBSTRING = "gate satisfied by verb rationale"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_pm_decision_required (unblock / escalate_up / escalate_to_ceo / delegate)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pm_decision_required_transient_failure_satisfies_gate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A "transient_failure" outcome satisfies the gate for this call even
|
||||
though no fresh decision exists — the verb's own rationale is the
|
||||
substance, the write was a convenience.
|
||||
|
||||
``_impl.py`` logs via ``structlog.get_logger()`` directly; absent this
|
||||
process having called ``roboco.logging.setup_logging()`` (never true in
|
||||
a bare unit-test run), structlog uses its own default global config and
|
||||
never touches stdlib ``logging`` — so ``caplog`` cannot see it. Patching
|
||||
the module-level ``logger`` object is the reliable way to assert a
|
||||
structlog call in this harness.
|
||||
"""
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(_impl_module, "logger", mock_logger)
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.latest_decision_at.return_value = None # no fresh decision
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
t = MagicMock(id=uuid4())
|
||||
|
||||
env = await c._check_pm_decision_required(
|
||||
"unblock",
|
||||
uuid4(),
|
||||
t.id,
|
||||
t,
|
||||
pm_decision_outcome="transient_failure",
|
||||
)
|
||||
assert env is None
|
||||
mock_logger.warning.assert_called_once()
|
||||
call = mock_logger.warning.call_args
|
||||
assert _TRANSIENT_WARNING_SUBSTRING in call.args[0]
|
||||
assert call.kwargs.get("verb") == "unblock"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pm_decision_required_absent_still_rejects() -> None:
|
||||
""" "absent" (no rationale, no fresh decision) rejects exactly as before
|
||||
— defense-in-depth unchanged."""
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
t = MagicMock(id=uuid4())
|
||||
|
||||
env = await c._check_pm_decision_required(
|
||||
"unblock", uuid4(), t.id, t, pm_decision_outcome="absent"
|
||||
)
|
||||
assert env is not None
|
||||
assert env.as_dict()["error"] == "tracing_gap"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pm_decision_required_none_default_unchanged() -> None:
|
||||
"""Every call site that hasn't threaded the outcome through (there are
|
||||
none left in production, but the param defaults to None for legacy
|
||||
parity) behaves byte-for-byte as before: no fresh decision rejects."""
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
t = MagicMock(id=uuid4())
|
||||
|
||||
env = await c._check_pm_decision_required("unblock", uuid4(), t.id, t)
|
||||
assert env is not None
|
||||
assert env.as_dict()["error"] == "tracing_gap"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_pm_decision_required_fresh_path_unchanged() -> None:
|
||||
"""A genuinely fresh decision passes regardless of pm_decision_outcome
|
||||
— "fresh" is not a special-case bypass, it's the ordinary passing path."""
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.latest_decision_at.return_value = datetime.now(UTC)
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
t = MagicMock(id=uuid4())
|
||||
|
||||
env = await c._check_pm_decision_required(
|
||||
"unblock", uuid4(), t.id, t, pm_decision_outcome="fresh"
|
||||
)
|
||||
assert env is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_complete_gates (cell_pm / main_pm complete)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_complete_gates_transient_failure_satisfies_gate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""See ``test_check_pm_decision_required_transient_failure_satisfies_gate``
|
||||
for why the module logger is patched directly instead of using caplog."""
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(_impl_module, "logger", mock_logger)
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.has_reflect_for_task.return_value = False
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
|
||||
env = await c._check_complete_gates(
|
||||
uuid4(),
|
||||
uuid4(),
|
||||
"closing this task: the cell's contribution merged cleanly",
|
||||
pm_decision_outcome="transient_failure",
|
||||
)
|
||||
assert env is None
|
||||
mock_logger.warning.assert_called_once()
|
||||
call = mock_logger.warning.call_args
|
||||
assert _TRANSIENT_WARNING_SUBSTRING in call.args[0]
|
||||
assert call.kwargs.get("verb") == "complete"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_complete_gates_absent_still_rejects() -> None:
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.has_reflect_for_task.return_value = False
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
|
||||
env = await c._check_complete_gates(
|
||||
uuid4(),
|
||||
uuid4(),
|
||||
"closing this task: the cell's contribution merged cleanly",
|
||||
pm_decision_outcome="absent",
|
||||
)
|
||||
assert env is not None
|
||||
assert env.as_dict()["error"] == "tracing_gap"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_complete_gates_fresh_path_unchanged() -> None:
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = True
|
||||
journal_svc.has_reflect_for_task.return_value = True
|
||||
c = _make_choreographer(journal=journal_svc)
|
||||
|
||||
env = await c._check_complete_gates(
|
||||
uuid4(),
|
||||
uuid4(),
|
||||
"closing this task: the cell's contribution merged cleanly",
|
||||
pm_decision_outcome="fresh",
|
||||
)
|
||||
assert env is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _cell_pm_complete_guard — the real complete() call site, one hop above the
|
||||
# gate helper, proving the outcome actually reaches it end to end (without
|
||||
# dragging in the full cell_pm_complete verb's PR-merge/finalize machinery).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cell_pm_complete_guard_survives_journal_write_lock_timeout() -> None:
|
||||
"""journal.write_decision raising (a lock-timeout under DB contention)
|
||||
with a substantive ``notes`` rationale in hand must not reject the PM's
|
||||
complete — the guard clears (returns None) instead of tracing_gap."""
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = MagicMock(
|
||||
id=task_id,
|
||||
assigned_to=pm_id,
|
||||
status="awaiting_pm_review",
|
||||
pr_number=42,
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.all_subtasks_terminal.return_value = True
|
||||
task_svc.uncovered_parent_acceptance_criteria.return_value = []
|
||||
# _ensure_pm_decision opens a session.begin_nested() savepoint before
|
||||
# the write raises — an unshaped AsyncMock's auto-attribute return
|
||||
# doesn't support `async with`, orphaning the mock's internal coroutine
|
||||
# (AsyncMockMixin._execute_mock_call never awaited).
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.has_reflect_for_task.return_value = False
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
journal_svc.write_decision.side_effect = OperationalError(
|
||||
"INSERT INTO journal_entries (id, ...) VALUES (...)",
|
||||
{},
|
||||
Exception("canceling statement due to lock timeout"),
|
||||
)
|
||||
c = _make_choreographer(task=task_svc, journal=journal_svc)
|
||||
|
||||
env = await c._cell_pm_complete_guard(
|
||||
pm_id, task_id, t, "closing this task: the cell's contribution merged cleanly"
|
||||
)
|
||||
|
||||
task_svc.session.begin_nested.assert_called()
|
||||
assert env is None, env.as_dict() if env is not None else None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cell_pm_complete_guard_empty_notes_still_rejects() -> None:
|
||||
"""No rationale at all (empty notes) AND no fresh decision on record
|
||||
still rejects — "absent" is not a bypass."""
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = MagicMock(
|
||||
id=task_id,
|
||||
assigned_to=pm_id,
|
||||
status="awaiting_pm_review",
|
||||
pr_number=42,
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.all_subtasks_terminal.return_value = True
|
||||
task_svc.uncovered_parent_acceptance_criteria.return_value = []
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.has_reflect_for_task.return_value = False
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
c = _make_choreographer(task=task_svc, journal=journal_svc)
|
||||
|
||||
env = await c._cell_pm_complete_guard(pm_id, task_id, t, "")
|
||||
assert env is not None
|
||||
assert env.as_dict()["error"] == "tracing_gap"
|
||||
Reference in New Issue
Block a user