fix(runtime): stop the chown storm from starving claims, and savepoint the PM journal auto-record

Claim-shaped verbs were failing 7/7 (claim_review) and 6/6
(claim_doc_task) as silent 120s FlowVerbTimeout 504s on the NAS: the
per-claim ownership repair walked the whole clone issuing two stat
syscalls per entry (chown_ms 39502 vs git_ms 8 in the live log), several
passes stacked per claim, and the claim transaction held the task row
the whole time — so concurrent writers queued behind it into the 60s
lock_timeout. The walk now does one stat per entry shared by the
chown-skip and chmod-skip checks, and a .git/roboco-owned sentinel
(worktree-aware via _resolve_clone_root, written only after a
zero-failure pass) skips the walk entirely when the tree is already
agent-owned. Every root-side git write invalidates the sentinel BEFORE
its subprocess runs — GitService._run_git for scope != none, plus the
three raw-subprocess paths inside WorkspaceService the adversarial pass
proved bypass it deterministically on the common respawn shape
(_worktree_git for mutating verbs, _fetch_branch_ref,
_fetch_origin_best_effort) — so a live marker can never vouch for files
a root write is about to create.

One of those queued writers was the PM journal-decision auto-record:
its INSERT hit the lock timeout, _ensure_pm_decision's catch-all
swallowed it without rollback, and the poisoned session blew up
escalate_up with PendingRollbackError (live incident). The helper's try
body now runs in a savepoint — one fix covering all seven PM verbs that
route through it — verified empirically against real Postgres in both
directions: the failure path leaves the session healthy and the task
object readable, and create_entry's internal commit inside the savepoint
drains the transactional outbox exactly once.
This commit is contained in:
Renn F
2026-07-31 01:53:07 +02:00
parent d87e2d9b4e
commit e6c9dde2a9
15 changed files with 874 additions and 42 deletions
@@ -194,6 +194,72 @@ async def test_full_scope_op_calls_full_repair_not_git_repair(
git_repair.assert_not_called()
@pytest.mark.asyncio
async def test_read_only_op_does_not_invalidate_owned_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A read-only op writes nothing, so the ownership-sentinel marker
(`_ensure_agent_owned`'s root short-circuit) stays valid — invalidating
it here would force a needless full walk on the very next call."""
(tmp_path / ".git").mkdir()
monkeypatch.setattr(
"roboco.services.git.subprocess.run", lambda *_a, **_k: _ok(["status"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
await _svc()._run_git(tmp_path, ["status", "--porcelain"])
invalidate.assert_not_called()
@pytest.mark.asyncio
async def test_git_scoped_op_invalidates_owned_marker_before_running(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A `.git`-only-writing op (commit) can still create new root-owned
files, so it must invalidate the marker too, not just checkout/reset/
etc. — and it must do so BEFORE the subprocess runs, so a marker still
trusted by a concurrent `_ensure_agent_owned` call can never straddle
the write."""
(tmp_path / ".git").mkdir()
order: list[str] = []
def _run_subprocess(*_a: object, **_k: object) -> subprocess.CompletedProcess[str]:
order.append("subprocess.run")
return _ok(["commit"])
monkeypatch.setattr("roboco.services.git.subprocess.run", _run_subprocess)
monkeypatch.setattr(
"roboco.services.workspace.invalidate_owned_marker",
lambda _ws: order.append("invalidate_owned_marker"),
)
monkeypatch.setattr("roboco.services.workspace._ensure_git_dir_owned", MagicMock())
await _svc()._run_git(tmp_path, ["commit", "-m", "msg"])
assert order == ["invalidate_owned_marker", "subprocess.run"]
@pytest.mark.asyncio
async def test_full_scope_op_invalidates_owned_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""checkout/reset/rebase/pull can create root-owned working-tree files
too — the marker invalidation isn't scoped to `.git`-only writes."""
(tmp_path / ".git").mkdir()
monkeypatch.setattr(
"roboco.services.git.subprocess.run", lambda *_a, **_k: _ok(["checkout"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
monkeypatch.setattr("roboco.services.workspace._ensure_agent_owned", MagicMock())
await _svc()._run_git(tmp_path, ["checkout", "some-branch"])
invalidate.assert_called_once_with(tmp_path)
@pytest.mark.asyncio
async def test_reown_after_git_op_returns_zero_ms_when_skipped() -> None:
"""The instrumentation must see a true near-zero cost for a skipped repair,
@@ -10,14 +10,19 @@ approach) left the working tree root-owned and broke every agent file write.
from __future__ import annotations
from typing import TYPE_CHECKING
import os
import stat as stat_module
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from roboco.services import workspace as workspace_module
from roboco.services.workspace import _ensure_agent_owned
if TYPE_CHECKING:
from pathlib import Path
from roboco.services.workspace import (
_AGENT_GID,
_AGENT_UID,
_ensure_agent_owned,
_own_and_grant_rw,
)
def _build_workspace(root: Path) -> None:
@@ -48,11 +53,11 @@ def _record_touched(monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""Record every path _ensure_agent_owned tries to chown/chmod."""
touched: list[str] = []
def fake_chown_entry(entry: str) -> bool:
def fake_chown_entry(entry: str, _st: os.stat_result) -> bool:
touched.append(entry)
return True
def fake_make_rw(entry: str) -> None:
def fake_make_rw(entry: str, _st: os.stat_result) -> None:
touched.append(entry)
monkeypatch.setattr(workspace_module, "_chown_entry", fake_chown_entry)
@@ -105,9 +110,11 @@ def test_chown_failure_falls_back_to_chmod_and_warns(
(tmp_path / "file.py").write_text("x = 1\n")
chmod_calls: list[str] = []
monkeypatch.setattr(workspace_module, "_chown_entry", lambda _entry: False)
monkeypatch.setattr(workspace_module, "_chown_entry", lambda _entry, _st: False)
monkeypatch.setattr(
workspace_module, "_make_owner_and_group_rw", chmod_calls.append
workspace_module,
"_make_owner_and_group_rw",
lambda entry, _st: chmod_calls.append(entry),
)
warning_calls: list[tuple[str, dict[str, object]]] = []
monkeypatch.setattr(
@@ -123,3 +130,229 @@ def test_chown_failure_falls_back_to_chmod_and_warns(
# The failure is surfaced, not swallowed.
assert warning_calls
assert warning_calls[0][1]["failures"]
# ---------------------------------------------------------------------------
# `_own_and_grant_rw`: one shared stat now backs both the chown-needed and
# chmod-needed checks (previously two separate `Path.stat()` calls, one per
# helper) — the fix for chown_ms: 39502 on the production NAS, where every
# chown/chmod is a copy-on-write metadata write and the tree is almost always
# ALREADY correctly owned on a re-claim. These exercise the real (unmocked)
# `_own_and_grant_rw` / `_chown_entry` / `_make_owner_and_group_rw` at the
# os-syscall boundary.
# ---------------------------------------------------------------------------
def _stat_result(mode: int, uid: int = 0, gid: int = 0) -> os.stat_result:
"""A real ``os.stat_result`` exposing only the fields the ownership
helpers read (st_mode/st_uid/st_gid) — no filesystem entry needed."""
return os.stat_result((mode, 0, 0, 0, uid, gid, 0, 0, 0, 0))
def _fake_stat(result: os.stat_result) -> object:
"""A stand-in for ``os.stat`` accepting the ``(path, *, follow_symlinks)``
signature ``Path(entry).stat()`` actually calls it with underneath."""
def _stat(_path: object, **_kwargs: object) -> os.stat_result:
return result
return _stat
_ALREADY_RW_MODE = (
stat_module.S_IFREG
| stat_module.S_IRUSR
| stat_module.S_IWUSR
| stat_module.S_IRGRP
| stat_module.S_IWGRP
)
_ROOT_NARROW_MODE = stat_module.S_IFREG | stat_module.S_IRUSR | stat_module.S_IWUSR
def test_own_and_grant_rw_skips_syscalls_when_already_correct(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""An already agent-owned, already rw entry costs one stat and ZERO
metadata-write syscalls — the no-op-write cost that stacked to
chown_ms: 39502 across tens of thousands of files when the tree was
already correctly owned, as it almost always is on a re-claim."""
chown_mock = MagicMock()
chmod_mock = MagicMock()
monkeypatch.setattr(
workspace_module.os,
"stat",
_fake_stat(_stat_result(_ALREADY_RW_MODE, _AGENT_UID, _AGENT_GID)),
)
monkeypatch.setattr(workspace_module.os, "chown", chown_mock)
monkeypatch.setattr(workspace_module.os, "chmod", chmod_mock)
failed = _own_and_grant_rw("/fake/already-owned")
assert failed == 0
chown_mock.assert_not_called()
chmod_mock.assert_not_called()
def test_own_and_grant_rw_still_repairs_a_wrong_owned_entry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A genuinely wrong-owned entry (fresh root-side clone/fetch/checkout
output — root uid/gid, no group-write bit) still gets chowned AND
chmodded exactly as before the single-stat merge."""
chown_mock = MagicMock()
chmod_mock = MagicMock()
monkeypatch.setattr(
workspace_module.os, "stat", _fake_stat(_stat_result(_ROOT_NARROW_MODE))
)
monkeypatch.setattr(workspace_module.os, "chown", chown_mock)
monkeypatch.setattr(workspace_module.os, "chmod", chmod_mock)
failed = _own_and_grant_rw("/fake/root-owned")
assert failed == 0
chown_mock.assert_called_once_with("/fake/root-owned", _AGENT_UID, _AGENT_GID)
expected_mode = _ROOT_NARROW_MODE | stat_module.S_IRGRP | stat_module.S_IWGRP
# chmod runs via Path(entry).chmod(...), which calls os.chmod with a
# Path-wrapped first arg + follow_symlinks=True — not the bare string.
chmod_mock.assert_called_once_with(
Path("/fake/root-owned"), expected_mode, follow_symlinks=True
)
def test_own_and_grant_rw_chown_failure_still_counted_and_chmod_still_attempted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Unchanged failure-counting contract: a rejected chown (rootless /
userns host) still counts as one failure AND the chmod best-effort
fallback still runs (belt + suspenders for ACL-inheriting NAS volumes)."""
monkeypatch.setattr(
workspace_module.os, "stat", _fake_stat(_stat_result(_ROOT_NARROW_MODE))
)
def _raise_chown(*_args: object, **_kwargs: object) -> None:
raise OSError("Operation not permitted")
chmod_mock = MagicMock()
monkeypatch.setattr(workspace_module.os, "chown", _raise_chown)
monkeypatch.setattr(workspace_module.os, "chmod", chmod_mock)
failed = _own_and_grant_rw("/fake/rootless-host")
assert failed == 1
chmod_mock.assert_called_once()
def test_own_and_grant_rw_broken_symlink_counts_as_failure_without_crashing(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Symlink decision: the shared stat call follows symlinks — matching
os.chown/os.chmod's own default follow behavior, unchanged from before
the merge (the old code's two separate `Path(entry).stat()` calls also
followed). A dangling symlink's stat raises OSError exactly as it did
from inside the old `_chown_entry`'s own stat call: counted as one
chown failure, and chmod is never attempted — the old chmod path hit
the identical OSError from its own separate stat call and silently
swallowed it, so the net effect (one counted failure, no chmod) is
unchanged."""
broken_link = tmp_path / "dangling"
broken_link.symlink_to(tmp_path / "does-not-exist")
chown_mock = MagicMock()
chmod_mock = MagicMock()
monkeypatch.setattr(workspace_module.os, "chown", chown_mock)
monkeypatch.setattr(workspace_module.os, "chmod", chmod_mock)
failed = _own_and_grant_rw(str(broken_link))
assert failed == 1
chown_mock.assert_not_called()
chmod_mock.assert_not_called()
# ---------------------------------------------------------------------------
# Root-sentinel short-circuit: `_ensure_agent_owned` skips the ENTIRE walk
# when the workspace root is already agent-owned with the right bits AND a
# `.git/roboco-owned` marker from the last zero-failure pass exists. The
# marker is invalidated by `GitService._run_git`
# (tests/unit/services/test_git_ownership_scope.py) before any root-side git
# write, so a live marker is trustworthy — this only removes the remaining
# per-entry-stat cost the earlier collapse (above) couldn't, the walk itself.
# ---------------------------------------------------------------------------
_OWNED_DIR_MODE = (
stat_module.S_IFDIR
| stat_module.S_IRUSR
| stat_module.S_IWUSR
| stat_module.S_IXUSR
| stat_module.S_IRGRP
| stat_module.S_IWGRP
| stat_module.S_IXGRP
)
def _fake_root_owned_stat(root: Path) -> object:
"""Real ``os.stat`` for every path except ``root``, which reports as
agent-owned with the required rw+x bits. Lets the marker file's own
existence check (``Path.is_file()``, which also routes through
``os.stat``) reflect the real filesystem instead of a blanket fake."""
real_stat = os.stat
def _stat(path: Path, *, follow_symlinks: bool = True) -> os.stat_result:
if path == root:
return _stat_result(_OWNED_DIR_MODE, _AGENT_UID, _AGENT_GID)
return real_stat(path, follow_symlinks=follow_symlinks)
return _stat
def test_skips_walk_when_root_owned_and_marker_present(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, _record_touched: list[str]
) -> None:
_build_workspace(tmp_path)
(tmp_path / ".git" / "roboco-owned").touch()
monkeypatch.setattr(workspace_module.os, "stat", _fake_root_owned_stat(tmp_path))
walk_mock = MagicMock(return_value=iter(()))
monkeypatch.setattr(workspace_module.os, "walk", walk_mock)
_ensure_agent_owned(tmp_path)
walk_mock.assert_not_called()
assert _record_touched == []
def test_full_walk_when_marker_absent(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, _record_touched: list[str]
) -> None:
"""Root already agent-owned, but no marker: the last pass over this
clone is unattested, so the real walk still runs."""
_build_workspace(tmp_path)
monkeypatch.setattr(workspace_module.os, "stat", _fake_root_owned_stat(tmp_path))
_ensure_agent_owned(tmp_path)
assert str(tmp_path) in _record_touched
assert str(tmp_path / "README.md") in _record_touched
def test_marker_written_only_on_zero_failure_pass(tmp_path: Path) -> None:
_build_workspace(tmp_path)
marker = tmp_path / ".git" / "roboco-owned"
assert not marker.exists()
# A real pass: chown to uid 1000 fails under the test's real (non-root)
# uid, exactly like a rootless/userns host — so no marker should land.
_ensure_agent_owned(tmp_path)
assert not marker.exists()
def test_marker_written_after_successful_pass(
tmp_path: Path, _record_touched: list[str]
) -> None:
"""`_record_touched`'s fakes report every chown/chmod as succeeding, so
this exercises the zero-failure branch without needing real root."""
_build_workspace(tmp_path)
marker = tmp_path / ".git" / "roboco-owned"
_ensure_agent_owned(tmp_path)
assert marker.is_file()
assert _record_touched # the walk actually ran (no marker existed yet)
@@ -0,0 +1,212 @@
"""The ownership-sentinel marker (`_ensure_agent_owned`'s root short-circuit,
see test_workspace_ensure_agent_owned_scope.py) must be invalidated by EVERY
root-side git-write path, not just `GitService._run_git`
(test_git_ownership_scope.py covers that one).
Adversarial review found a deterministic hole: `WorkspaceService`'s raw-
subprocess git helpers — `_worktree_git`, `_fetch_branch_ref`,
`_fetch_origin_best_effort` — never invalidated the marker, and the most
common spawn path hits them on (nearly) every respawn
(`ensure_worktree_self_heal` -> `_refresh_present_worktree` ->
`_fetch_branch_ref` + `_worktree_git(["reset", "--hard", ...])`). A stale
marker then let `_ensure_agent_owned` skip the walk that would have repaired
the root-owned files those calls had just created — a live Permission
denied for the agent. These tests cover the bypass paths directly.
"""
from __future__ import annotations
import subprocess
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from roboco.services.workspace import WorkspaceService, _ensure_agent_owned
from tests.unit.services.test_workspace_ensure_agent_owned_scope import (
_build_workspace,
)
if TYPE_CHECKING:
from pathlib import Path
def _svc() -> WorkspaceService:
return WorkspaceService(MagicMock())
def _ok(args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.CompletedProcess(
args=["git", *args], returncode=0, stdout="", stderr=""
)
# ---------------------------------------------------------------------------
# `_worktree_git`: mutating verbs invalidate, read-only verbs don't.
# ---------------------------------------------------------------------------
def test_worktree_git_reset_hard_invalidates_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run", lambda *_a, **_k: _ok(["reset"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
WorkspaceService._worktree_git(tmp_path, ["reset", "--hard", "origin/x"])
invalidate.assert_called_once_with(tmp_path)
def test_worktree_git_rev_parse_does_not_invalidate_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run",
lambda *_a, **_k: _ok(["rev-parse"]),
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
WorkspaceService._worktree_git(
tmp_path, ["rev-parse", "--verify", "--quiet", "refs/heads/x"], check=False
)
invalidate.assert_not_called()
def test_worktree_git_branch_show_current_does_not_invalidate_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The ambiguous verb's query form: `branch --show-current` only reads."""
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run", lambda *_a, **_k: _ok(["branch"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
WorkspaceService._worktree_git(tmp_path, ["branch", "--show-current"], check=False)
invalidate.assert_not_called()
def test_worktree_git_branch_delete_invalidates_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The ambiguous verb's write forms: `branch -d/-D <name>` writes."""
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run", lambda *_a, **_k: _ok(["branch"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
WorkspaceService._worktree_git(
tmp_path, ["branch", "-D", "task-branch"], check=False
)
invalidate.assert_called_once_with(tmp_path)
# ---------------------------------------------------------------------------
# `_fetch_branch_ref` — always mutating (fetch writes .git/objects + refs).
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_branch_ref_invalidates_marker_before_subprocess(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
order: list[str] = []
def _run_subprocess(*_a: object, **_k: object) -> subprocess.CompletedProcess[str]:
order.append("subprocess.run")
return _ok(["fetch"])
monkeypatch.setattr("roboco.services.workspace.subprocess.run", _run_subprocess)
monkeypatch.setattr(
"roboco.services.workspace.invalidate_owned_marker",
lambda _ws: order.append("invalidate_owned_marker"),
)
mock_project_service = MagicMock()
mock_project_service.get_by_slug = AsyncMock(return_value=None)
with patch(
"roboco.services.project.get_project_service",
return_value=mock_project_service,
):
await _svc()._fetch_branch_ref(tmp_path, "task-branch", "roboco-api")
assert order == ["invalidate_owned_marker", "subprocess.run"]
# ---------------------------------------------------------------------------
# `_fetch_origin_best_effort` — same shape, scoped multi-ref fetch.
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_origin_best_effort_invalidates_marker(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run", lambda *_a, **_k: _ok(["fetch"])
)
invalidate = MagicMock()
monkeypatch.setattr("roboco.services.workspace.invalidate_owned_marker", invalidate)
await WorkspaceService._fetch_origin_best_effort(tmp_path, "roboco-api")
invalidate.assert_called_once_with(tmp_path)
# ---------------------------------------------------------------------------
# End-to-end-shaped repro: a full zero-failure pass writes the marker; a
# bypass-path root write (through the now-fixed helper) invalidates it; the
# NEXT _ensure_agent_owned call walks again instead of trusting stale state.
# ---------------------------------------------------------------------------
def test_bypass_path_write_forces_next_ensure_agent_owned_to_walk(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_build_workspace(tmp_path)
touched_pass_1: list[str] = []
touched_pass_2: list[str] = []
def make_fakes(sink: list[str]) -> tuple[object, object]:
def fake_chown_entry(entry: str, _st: object) -> bool:
sink.append(entry)
return True
def fake_make_rw(entry: str, _st: object) -> None:
sink.append(entry)
return fake_chown_entry, fake_make_rw
# Pass 1: a normal zero-failure ensure_agent_owned — writes the marker.
fake_chown, fake_rw = make_fakes(touched_pass_1)
monkeypatch.setattr("roboco.services.workspace._chown_entry", fake_chown)
monkeypatch.setattr("roboco.services.workspace._make_owner_and_group_rw", fake_rw)
_ensure_agent_owned(tmp_path)
marker = tmp_path / ".git" / "roboco-owned"
assert marker.is_file()
assert touched_pass_1 # the walk actually ran
# Bypass-path root write: a mutating _worktree_git call (the exact class
# of call ensure_worktree_self_heal's self-heal makes on nearly every
# respawn) — with subprocess mocked so no real git repo is needed, but
# invalidate_owned_marker running for real.
monkeypatch.setattr(
"roboco.services.workspace.subprocess.run", lambda *_a, **_k: _ok(["reset"])
)
WorkspaceService._worktree_git(tmp_path, ["reset", "--hard", "origin/x"])
assert not marker.exists() # the fix: the bypass path invalidated it
# Pass 2: _ensure_agent_owned must walk again (marker gone), not trust
# the stale "fully owned" state from before the bypass-path write.
fake_chown_2, fake_rw_2 = make_fakes(touched_pass_2)
monkeypatch.setattr("roboco.services.workspace._chown_entry", fake_chown_2)
monkeypatch.setattr("roboco.services.workspace._make_owner_and_group_rw", fake_rw_2)
_ensure_agent_owned(tmp_path)
assert touched_pass_2 # the walk ran again — nothing was silently skipped