mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
[F078] release_executor: deadline every subprocess (git/make/gh/clone)
A hung git/make/gh/clone would block the CEO-gated release loop indefinitely. Wrap each proc.communicate() in asyncio.wait_for via a shared _await_proc helper; on expiry proc.kill() the child and return a non-zero rc (124) so every caller's fail-closed branch fires. Mirrors the quality-gate _run_one kill-on-timeout idiom. Deadlines are generous (30min gate / 10min clone / 5min push+gh) so a legitimate slow op is never wrongly aborted — floor-assertion tests pin the floors to guard exactly that logical regression. Green path returns the real rc unchanged.
This commit is contained in:
@@ -132,6 +132,33 @@ class ReleaseExecutor:
|
||||
_CI_POLL_INTERVAL_SECONDS = 30
|
||||
_CI_MAX_POLLS = 80 # ~40 min ceiling
|
||||
|
||||
# Subprocess deadlines. A hung git / make / gh would otherwise block the
|
||||
# CEO-gated release loop indefinitely. Each is generous enough that a
|
||||
# legitimate, slow operation is never wrongly aborted — only a true hang fails
|
||||
# closed. Mirrors the quality-gate ``_run_one`` kill-on-timeout idiom.
|
||||
_GIT_OP_TIMEOUT_SECONDS = 300 # git add/commit/rev-parse/ls-remote/push
|
||||
_RELEASE_GATE_TIMEOUT_SECONDS = 1800 # make quality — full ruff/mypy/pytest suite
|
||||
_PUBLISH_TIMEOUT_SECONDS = 300 # gh release create
|
||||
_CLONE_TIMEOUT_SECONDS = 600 # git clone / rm -rf the release clone
|
||||
|
||||
# The conventional non-zero rc a timed-out subprocess reports so every caller's
|
||||
# fail-closed branch (rc != 0) fires instead of hanging the release loop.
|
||||
_TIMEOUT_RC = 124
|
||||
|
||||
|
||||
async def _await_proc(
|
||||
proc: asyncio.subprocess.Process, timeout: float
|
||||
) -> tuple[int, str]:
|
||||
"""Communicate with a subprocess under a deadline; on expiry ``kill()`` the
|
||||
child so a hang fails closed instead of wedging the release loop, and
|
||||
return a non-zero rc so every caller's fail-closed branch fires."""
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
proc.kill()
|
||||
return _TIMEOUT_RC, f"subprocess timed out after {int(timeout)}s"
|
||||
return proc.returncode or 0, out.decode("utf-8", "replace")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ReleaseContext:
|
||||
@@ -164,8 +191,7 @@ class _GitReleaseOps:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
return proc.returncode or 0, out.decode("utf-8", "replace")
|
||||
return await _await_proc(proc, _GIT_OP_TIMEOUT_SECONDS)
|
||||
|
||||
async def is_already_published(self, version: str) -> bool:
|
||||
rc, out = await self._git("ls-remote", "--tags", "origin", f"v{version}")
|
||||
@@ -212,13 +238,13 @@ class _GitReleaseOps:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
rc, out = await _await_proc(proc, _RELEASE_GATE_TIMEOUT_SECONDS)
|
||||
if rc != 0:
|
||||
logger.warning(
|
||||
"release gate (make quality) failed",
|
||||
tail=out.decode("utf-8", "replace")[-2000:],
|
||||
"release gate (make quality) failed or timed out",
|
||||
tail=out[-2000:],
|
||||
)
|
||||
return proc.returncode == 0
|
||||
return rc == 0
|
||||
|
||||
async def commit_and_push(self, version: str) -> str:
|
||||
add_rc, add_out = await self._git("add", "-A")
|
||||
@@ -280,9 +306,9 @@ class _GitReleaseOps:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
text = out.decode("utf-8", "replace").strip()
|
||||
if proc.returncode != 0:
|
||||
rc, out = await _await_proc(proc, _PUBLISH_TIMEOUT_SECONDS)
|
||||
text = out.strip()
|
||||
if rc != 0:
|
||||
logger.error("gh release create failed", error=text[:300])
|
||||
raise RuntimeError(f"gh release create failed: {text[:200]}")
|
||||
url = next(
|
||||
@@ -360,5 +386,4 @@ async def _run(cmd: list[str]) -> tuple[int, str]:
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await proc.communicate()
|
||||
return proc.returncode or 0, out.decode("utf-8", "replace")
|
||||
return await _await_proc(proc, _CLONE_TIMEOUT_SECONDS)
|
||||
|
||||
Reference in New Issue
Block a user