[F132] timeout the conventions validator + reap on hang

_run_conventions_validator awaited proc.communicate() with no timeout —
a hung subprocess (tree-sitter deadlock, huge repo) hung the
i_am_done/pr_pass gate forever and orphaned the python subprocess on
orchestrator restart. Wrap communicate() in wait_for(120s); on timeout
kill+wait the proc and fail closed (could_not_run=True → block gate
refuses the submit), matching the validator's own fail-loud philosophy.
This commit is contained in:
Renn F
2026-06-29 00:25:26 +02:00
parent 7f5967bc02
commit 4eb17ff7db
2 changed files with 58 additions and 1 deletions
+22 -1
View File
@@ -181,6 +181,9 @@ _CI_RUN_WINDOW = 20
_CI_FETCH_ATTEMPTS = 3 _CI_FETCH_ATTEMPTS = 3
_CI_FETCH_BACKOFF_SECONDS = 0.5 _CI_FETCH_BACKOFF_SECONDS = 0.5
_CI_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504}) _CI_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
# Cap a conventions-validator run so a hung subprocess (tree-sitter deadlock,
# huge repo) can't hang the i_am_done/pr_pass gate forever.
_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS = 120
def _select_ci_head_run(runs: list[dict[str, Any]]) -> dict[str, Any]: def _select_ci_head_run(runs: list[dict[str, Any]]) -> dict[str, Any]:
@@ -4347,7 +4350,25 @@ class GitService(BaseService):
stdout=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
) )
out, err = await proc.communicate() try:
out, err = await asyncio.wait_for(
proc.communicate(),
timeout=_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS,
)
except TimeoutError:
# Fail closed (could_not_run=True → block gate refuses the submit),
# matching the validator's own fail-loud philosophy, and reap the
# killed proc so it isn't orphaned on orchestrator restart.
proc.kill()
await proc.wait()
return {
"findings": [],
"could_not_run": True,
"reason": (
f"validator timed out after "
f"{_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS}s"
),
}
if proc.returncode != 0: if proc.returncode != 0:
reason = err.decode(errors="replace").strip() or "validator crashed" reason = err.decode(errors="replace").strip() or "validator crashed"
return {"findings": [], "could_not_run": True, "reason": reason[:300]} return {"findings": [], "could_not_run": True, "reason": reason[:300]}
@@ -16,11 +16,13 @@ These are empty-result cases, not errors, so the gate correctly passes.
from __future__ import annotations from __future__ import annotations
import asyncio
from pathlib import Path from pathlib import Path
from unittest.mock import AsyncMock, MagicMock from unittest.mock import AsyncMock, MagicMock
from uuid import uuid4 from uuid import uuid4
import pytest import pytest
from roboco.services import git as git_module
from roboco.services.git import GitService from roboco.services.git import GitService
@@ -91,3 +93,37 @@ async def test_no_changed_files_still_fails_open() -> None:
result = await svc.conventions_check_for_task(uuid4(), _task("feature/backend/abc")) result = await svc.conventions_check_for_task(uuid4(), _task("feature/backend/abc"))
assert result["could_not_run"] is False assert result["could_not_run"] is False
assert result["findings"] == [] assert result["findings"] == []
@pytest.mark.asyncio
async def test_validator_timeout_fails_closed_and_reaps(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A hung conventions validator subprocess (tree-sitter deadlock, huge repo)
must time out, fail closed (could_not_run=True so the block gate refuses the
submit), and kill+wait the proc not hang the gate forever nor orphan the
subprocess on orchestrator restart.
"""
fake_proc = MagicMock()
fake_proc.returncode = None
async def _communicate() -> tuple[bytes, bytes]:
await asyncio.sleep(30)
return (b"", b"")
fake_proc.communicate = _communicate
fake_proc.kill = MagicMock()
fake_proc.wait = AsyncMock(return_value=-9)
async def _fake_exec(*_args: object, **_kwargs: object) -> object:
return fake_proc
monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec)
monkeypatch.setattr(git_module, "_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS", 0.01)
svc = _service()
result = await svc._run_conventions_validator(tmp_path, ["a.py"])
assert result["could_not_run"] is True
assert "timed out" in (result.get("reason") or "")
fake_proc.kill.assert_called_once()
fake_proc.wait.assert_awaited_once()