mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F132] timeout the conventions validator + reap on hang
_run_conventions_validator awaited proc.communicate() with no timeout — a hung subprocess (tree-sitter deadlock, huge repo) hung the i_am_done/pr_pass gate forever and orphaned the python subprocess on orchestrator restart. Wrap communicate() in wait_for(120s); on timeout kill+wait the proc and fail closed (could_not_run=True → block gate refuses the submit), matching the validator's own fail-loud philosophy.
This commit is contained in:
+22
-1
@@ -181,6 +181,9 @@ _CI_RUN_WINDOW = 20
|
|||||||
_CI_FETCH_ATTEMPTS = 3
|
_CI_FETCH_ATTEMPTS = 3
|
||||||
_CI_FETCH_BACKOFF_SECONDS = 0.5
|
_CI_FETCH_BACKOFF_SECONDS = 0.5
|
||||||
_CI_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
|
_CI_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504})
|
||||||
|
# Cap a conventions-validator run so a hung subprocess (tree-sitter deadlock,
|
||||||
|
# huge repo) can't hang the i_am_done/pr_pass gate forever.
|
||||||
|
_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS = 120
|
||||||
|
|
||||||
|
|
||||||
def _select_ci_head_run(runs: list[dict[str, Any]]) -> dict[str, Any]:
|
def _select_ci_head_run(runs: list[dict[str, Any]]) -> dict[str, Any]:
|
||||||
@@ -4347,7 +4350,25 @@ class GitService(BaseService):
|
|||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
)
|
)
|
||||||
out, err = await proc.communicate()
|
try:
|
||||||
|
out, err = await asyncio.wait_for(
|
||||||
|
proc.communicate(),
|
||||||
|
timeout=_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
# Fail closed (could_not_run=True → block gate refuses the submit),
|
||||||
|
# matching the validator's own fail-loud philosophy, and reap the
|
||||||
|
# killed proc so it isn't orphaned on orchestrator restart.
|
||||||
|
proc.kill()
|
||||||
|
await proc.wait()
|
||||||
|
return {
|
||||||
|
"findings": [],
|
||||||
|
"could_not_run": True,
|
||||||
|
"reason": (
|
||||||
|
f"validator timed out after "
|
||||||
|
f"{_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS}s"
|
||||||
|
),
|
||||||
|
}
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
reason = err.decode(errors="replace").strip() or "validator crashed"
|
reason = err.decode(errors="replace").strip() or "validator crashed"
|
||||||
return {"findings": [], "could_not_run": True, "reason": reason[:300]}
|
return {"findings": [], "could_not_run": True, "reason": reason[:300]}
|
||||||
|
|||||||
@@ -16,11 +16,13 @@ These are empty-result cases, not errors, so the gate correctly passes.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from roboco.services import git as git_module
|
||||||
from roboco.services.git import GitService
|
from roboco.services.git import GitService
|
||||||
|
|
||||||
|
|
||||||
@@ -91,3 +93,37 @@ async def test_no_changed_files_still_fails_open() -> None:
|
|||||||
result = await svc.conventions_check_for_task(uuid4(), _task("feature/backend/abc"))
|
result = await svc.conventions_check_for_task(uuid4(), _task("feature/backend/abc"))
|
||||||
assert result["could_not_run"] is False
|
assert result["could_not_run"] is False
|
||||||
assert result["findings"] == []
|
assert result["findings"] == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_validator_timeout_fails_closed_and_reaps(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""A hung conventions validator subprocess (tree-sitter deadlock, huge repo)
|
||||||
|
must time out, fail closed (could_not_run=True so the block gate refuses the
|
||||||
|
submit), and kill+wait the proc — not hang the gate forever nor orphan the
|
||||||
|
subprocess on orchestrator restart.
|
||||||
|
"""
|
||||||
|
fake_proc = MagicMock()
|
||||||
|
fake_proc.returncode = None
|
||||||
|
|
||||||
|
async def _communicate() -> tuple[bytes, bytes]:
|
||||||
|
await asyncio.sleep(30)
|
||||||
|
return (b"", b"")
|
||||||
|
|
||||||
|
fake_proc.communicate = _communicate
|
||||||
|
fake_proc.kill = MagicMock()
|
||||||
|
fake_proc.wait = AsyncMock(return_value=-9)
|
||||||
|
|
||||||
|
async def _fake_exec(*_args: object, **_kwargs: object) -> object:
|
||||||
|
return fake_proc
|
||||||
|
|
||||||
|
monkeypatch.setattr(asyncio, "create_subprocess_exec", _fake_exec)
|
||||||
|
monkeypatch.setattr(git_module, "_CONVENTIONS_VALIDATOR_TIMEOUT_SECONDS", 0.01)
|
||||||
|
|
||||||
|
svc = _service()
|
||||||
|
result = await svc._run_conventions_validator(tmp_path, ["a.py"])
|
||||||
|
assert result["could_not_run"] is True
|
||||||
|
assert "timed out" in (result.get("reason") or "")
|
||||||
|
fake_proc.kill.assert_called_once()
|
||||||
|
fake_proc.wait.assert_awaited_once()
|
||||||
|
|||||||
Reference in New Issue
Block a user