[chore] logical-gaps: release-proposal already_published closes proposal + heartbeat-lock-loss cancels execute (2 gaps)

approve() closed the proposal only on status=='published'. A retry that finds
the tag already shipped returns 'already_published' (is_already_published),
so if a prior publish's route commit failed / HTTP 504'd, the proposal stayed
non-terminal forever — every retry returned already_published and never
closed it; only a manual cancel unstuck it. Close on both published and
already_published: the release shipped either way.

_heartbeat_loop returned silently when the lock was no longer owned (a >TTL
Redis outage let the mutex expire mid-execute), leaving executor.execute
running UNGUARDED — a concurrent approve (once Redis returns) could then
acquire the lock and _prepare_release_clone rm -rf the in-flight shared
release clone while the first execute was still mid-run_gate, re-opening the
very rm -rf-clone race the mutex+heartbeat exist to prevent. Run execute as a
task; on lock-loss the heartbeat sets a flag and cancels it, and approve()
turns the CancelledError into a structured 'lock_lost' result (an external
cancellation of approve itself still propagates — distinguished by the flag).

TDD: 2 red→green (already_published → COMPLETED not wedged; heartbeat lock-loss
→ lock_lost + execute cancelled, proposal not completed). 8 concurrency tests
green; ruff/mypy clean.
This commit is contained in:
Renn F
2026-06-30 18:09:48 +02:00
parent d8a5bb485f
commit 05616607e4
2 changed files with 150 additions and 8 deletions
@@ -393,3 +393,94 @@ async def test_heartbeat_refreshes_lock_and_is_cancelled_in_finally(
assert all(k == name for k, _ttl in fake_redis.expire_calls)
# And the lock was released at the end.
assert await fake_redis.get(name) is None
@pytest.mark.asyncio
async def test_already_published_closes_proposal_not_wedges_open() -> None:
"""#149: when a prior publish landed (tag exists) but the route commit
failed / 504'd, a retry sees the tag and execute returns ``already_published``.
approve() must STILL mark the proposal COMPLETED — the release shipped —
else the old ``if status == 'published'`` check wedged it open forever (every
retry returns already_published and never closes it; only a manual cancel
unsticks it)."""
task = _task()
fake_redis = _FakeRedis()
already = ReleaseResult(
status="already_published",
version="0.13.0",
files_changed=[],
commit_sha=None,
release_url=None,
detail="v0.13.0 is already published; nothing to do.",
)
w = _wire(task, _REPORT, already, fake_redis)
svc = ReleaseProposalService(_session())
with (
w["patches"][0],
w["patches"][1],
w["patches"][2],
w["patches"][3],
w["patches"][4],
):
result = await svc.approve(task.id)
assert result is not None
assert result.status == "already_published"
# The proposal is CLOSED — the release shipped — not wedged open.
assert task.status == TaskStatus.COMPLETED.value
@pytest.mark.asyncio
async def test_heartbeat_lock_loss_cancels_execute_fail_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""#218: when the heartbeat finds the lock no longer owned (a >TTL Redis
outage let the mutex expire and a usurper re-acquired), it must CANCEL the
in-flight execute fail-closed — not ``return`` silently and leave execute
running unguarded, which lets the usurper's approve ``rm -rf`` the in-flight
release clone (re-opening the race the mutex+heartbeat exist to prevent).
approve() surfaces a structured ``lock_lost`` result."""
monkeypatch.setattr(rp, "_RELEASE_LOCK_HEARTBEAT_SECONDS", 0.001)
task = _task()
fake_redis = _FakeRedis()
published = ReleaseResult(
status="published",
version="0.13.0",
files_changed=[],
commit_sha=None,
release_url=None,
detail="ok",
)
execute_started = asyncio.Event()
async def _blocking_execute(_report: Any) -> ReleaseResult:
# Block until the heartbeat cancels us — proves execute was running and
# got cancelled, not never-started.
execute_started.set()
await asyncio.sleep(60)
return published
w = _wire(task, _REPORT, published, fake_redis)
w["executor"].execute = AsyncMock(side_effect=_blocking_execute)
svc = ReleaseProposalService(_session())
# The heartbeat's compare-and-expire reports the lock lost (token mismatch —
# a usurper re-acquired after TTL expiry).
svc._heartbeat_release_lock = AsyncMock(return_value=False)
with (
w["patches"][0],
w["patches"][1],
w["patches"][2],
w["patches"][3],
w["patches"][4],
):
result = await svc.approve(task.id)
assert result is not None
assert result.status == "lock_lost"
assert execute_started.is_set() # execute did start, then was cancelled
# Fail-closed: the proposal is NOT marked COMPLETED.
assert task.status != TaskStatus.COMPLETED.value