mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(runtime): stop the chown storm from starving claims, and savepoint the PM journal auto-record
Claim-shaped verbs were failing 7/7 (claim_review) and 6/6 (claim_doc_task) as silent 120s FlowVerbTimeout 504s on the NAS: the per-claim ownership repair walked the whole clone issuing two stat syscalls per entry (chown_ms 39502 vs git_ms 8 in the live log), several passes stacked per claim, and the claim transaction held the task row the whole time — so concurrent writers queued behind it into the 60s lock_timeout. The walk now does one stat per entry shared by the chown-skip and chmod-skip checks, and a .git/roboco-owned sentinel (worktree-aware via _resolve_clone_root, written only after a zero-failure pass) skips the walk entirely when the tree is already agent-owned. Every root-side git write invalidates the sentinel BEFORE its subprocess runs — GitService._run_git for scope != none, plus the three raw-subprocess paths inside WorkspaceService the adversarial pass proved bypass it deterministically on the common respawn shape (_worktree_git for mutating verbs, _fetch_branch_ref, _fetch_origin_best_effort) — so a live marker can never vouch for files a root write is about to create. One of those queued writers was the PM journal-decision auto-record: its INSERT hit the lock timeout, _ensure_pm_decision's catch-all swallowed it without rollback, and the poisoned session blew up escalate_up with PendingRollbackError (live incident). The helper's try body now runs in a savepoint — one fix covering all seven PM verbs that route through it — verified empirically against real Postgres in both directions: the failure path leaves the session healthy and the task object readable, and create_entry's internal commit inside the savepoint drains the transactional outbox exactly once.
This commit is contained in:
@@ -12,6 +12,7 @@ from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.services.gateway.choreographer import Choreographer, ChoreographerDeps
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
|
||||
def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
@@ -969,6 +970,52 @@ async def test_escalate_up_blocks_without_journal_decision() -> None:
|
||||
assert "journal:decision" in body["missing"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_escalate_up_survives_journal_write_lock_timeout() -> None:
|
||||
"""Regression: a journal:decision INSERT that lock-times out (a
|
||||
concurrent claim transaction holding the task row's FK share lock —
|
||||
live production 500) used to be swallowed by ``_ensure_pm_decision``
|
||||
with no rollback/savepoint, poisoning the session so the very next
|
||||
attribute touch (``_escalate_up_preflight`` reading ``t.id``) raised an
|
||||
unhandled ``PendingRollbackError``. The write is now savepoint-guarded
|
||||
(``begin_nested()``): the failure is contained, the verb falls through
|
||||
cleanly to the normal tracing_gap rejection (no decision was actually
|
||||
persisted), and the task stays fully readable — no unhandled exception
|
||||
escapes ``escalate_up``."""
|
||||
pm_id = uuid4()
|
||||
task_id = uuid4()
|
||||
t = MagicMock(id=task_id, status="blocked", assigned_to=pm_id, team="backend")
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = t
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
role="cell_pm", escalation_target="main-pm"
|
||||
)
|
||||
journal_svc = AsyncMock()
|
||||
journal_svc.has_decision_for_task.return_value = False
|
||||
journal_svc.latest_decision_at.return_value = None
|
||||
journal_svc.write_decision.side_effect = OperationalError(
|
||||
"INSERT INTO journal_entries (id, ...) VALUES (...)",
|
||||
{},
|
||||
Exception("canceling statement due to lock timeout"),
|
||||
)
|
||||
deps = _make_deps(task=task_svc, journal=journal_svc)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c.escalate_up(pm_id, task_id, reason="needs cross-cell coordination")
|
||||
|
||||
# The savepoint was actually engaged — proves the fix is wired in, not
|
||||
# merely that AsyncMock happened to swallow the raise on its own.
|
||||
task_svc.session.begin_nested.assert_called()
|
||||
# No unhandled exception escaped escalate_up: the gate falls through to
|
||||
# its normal clean rejection since the decision write never landed.
|
||||
body = env.as_dict()
|
||||
assert body["error"] == "tracing_gap"
|
||||
assert "journal:decision" in body["missing"]
|
||||
# The task is still fully readable afterward — this is exactly where
|
||||
# the production trace crashed with PendingRollbackError on t.id.
|
||||
assert t.id == task_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_escalate_up_no_target_returns_invalid_state() -> None:
|
||||
"""Verb-specific preflight: PM whose escalation_target is unconfigured.
|
||||
|
||||
Reference in New Issue
Block a user