fix(orchestrator): let a failed Board Program exploration retry (#706)

`_board_dispatched` is an in-memory, never-expiring set of
(agent_slug, task_id). The solo exploration dispatchers consulted it, so an
exploration whose propose verb rejected was never retried for the life of the
process — Periscope, Sentinel, Scales and Barfly each spawned once on
2026-07-25, failed, and sat PENDING until the stack restarted.

The guard was written for the two-reviewer board REVIEW pass, where it is
correct: a reviewer has no verb to advance the task, so a respawn can only
loop. An explorer is the opposite — `propose_*` is exactly such a verb, so a
respawn can and should advance it.

Drop it from the 15 `_dispatch_*_exploration` functions. Bounding falls to
`_pm_respawn_should_gate`, which is what actually bounds a loop: DB-persisted,
reset by a status change, and cooled down so a deploy that fixes the cause
lets the work resume. `_dispatch_board_reviewer` keeps the set (its
`_board_review_complete` reads it as a has-run signal) as does vault
curation (same-process race guard behind its own durable marker).

The 14 `*_dispatch_is_one_shot` tests asserted the old contract on the false
rationale that board roles have no progression verb; they now assert a second
tick re-attempts.

Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
Renzo F
2026-07-26 15:52:33 +02:00
committed by GitHub
co-authored by Renn F
parent 879afc14a4
commit a36a180b73
15 changed files with 208 additions and 125 deletions
+13 -4
View File
@@ -58,9 +58,15 @@ async def test_barfly_dispatch_spawns_only_head_marketing() -> None:
@pytest.mark.asyncio
async def test_barfly_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending exploration must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_barfly_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task — unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _barfly_task()
with (
@@ -71,7 +77,10 @@ async def test_barfly_dispatch_is_one_shot() -> None:
await orch._dispatch_barfly_exploration(task)
await orch._dispatch_barfly_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -67,8 +67,15 @@ async def test_coroner_dispatch_spawns_only_auditor() -> None:
@pytest.mark.asyncio
async def test_coroner_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending autopsy must NOT respawn."""
async def test_coroner_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task — unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _coroner_task()
with (
@@ -80,7 +87,10 @@ async def test_coroner_dispatch_is_one_shot() -> None:
await orch._dispatch_coroner_exploration(task)
await orch._dispatch_coroner_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -61,8 +61,15 @@ async def test_dogfood_dispatch_spawns_only_product_owner() -> None:
@pytest.mark.asyncio
async def test_dogfood_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
async def test_dogfood_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task — unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _dogfood_task()
with (
@@ -73,7 +80,10 @@ async def test_dogfood_dispatch_is_one_shot() -> None:
await orch._dispatch_dogfood_exploration(task)
await orch._dispatch_dogfood_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
@@ -57,9 +57,15 @@ async def test_feature_spotlight_dispatch_spawns_only_head_marketing() -> None:
@pytest.mark.asyncio
async def test_feature_spotlight_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending exploration must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_feature_spotlight_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _feature_task()
with (
@@ -70,7 +76,10 @@ async def test_feature_spotlight_dispatch_is_one_shot() -> None:
await orch._dispatch_feature_spotlight_exploration(task)
await orch._dispatch_feature_spotlight_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -4
View File
@@ -59,9 +59,15 @@ async def test_librarian_dispatch_spawns_only_auditor() -> None:
@pytest.mark.asyncio
async def test_librarian_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending mining task must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_librarian_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _librarian_task()
with (
@@ -72,7 +78,10 @@ async def test_librarian_dispatch_is_one_shot() -> None:
await orch._dispatch_librarian_exploration(task)
await orch._dispatch_librarian_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -4
View File
@@ -58,9 +58,15 @@ async def test_megaphone_dispatch_spawns_only_head_marketing() -> None:
@pytest.mark.asyncio
async def test_megaphone_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending exploration must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_megaphone_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _megaphone_task()
with (
@@ -71,7 +77,10 @@ async def test_megaphone_dispatch_is_one_shot() -> None:
await orch._dispatch_megaphone_exploration(task)
await orch._dispatch_megaphone_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -57,8 +57,15 @@ async def test_mirror_dispatch_spawns_only_head_of_marketing() -> None:
@pytest.mark.asyncio
async def test_mirror_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
async def test_mirror_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _mirror_task()
with (
@@ -69,7 +76,10 @@ async def test_mirror_dispatch_is_one_shot() -> None:
await orch._dispatch_mirror_exploration(task)
await orch._dispatch_mirror_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -4
View File
@@ -59,9 +59,15 @@ async def test_periscope_dispatch_spawns_only_head_marketing() -> None:
@pytest.mark.asyncio
async def test_periscope_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending exploration must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_periscope_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _periscope_task()
with (
@@ -72,7 +78,10 @@ async def test_periscope_dispatch_is_one_shot() -> None:
await orch._dispatch_periscope_exploration(task)
await orch._dispatch_periscope_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
@@ -57,8 +57,15 @@ async def test_pest_control_dispatch_spawns_only_product_owner() -> None:
@pytest.mark.asyncio
async def test_pest_control_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
async def test_pest_control_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _pest_control_task()
with (
@@ -69,7 +76,10 @@ async def test_pest_control_dispatch_is_one_shot() -> None:
await orch._dispatch_pest_control_exploration(task)
await orch._dispatch_pest_control_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -4
View File
@@ -57,9 +57,15 @@ async def test_roadmap_dispatch_spawns_only_product_owner() -> None:
@pytest.mark.asyncio
async def test_roadmap_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn —
board roles have no progression verb, so a respawn would just loop."""
async def test_roadmap_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _roadmap_task()
with (
@@ -70,7 +76,10 @@ async def test_roadmap_dispatch_is_one_shot() -> None:
await orch._dispatch_roadmap_exploration(task)
await orch._dispatch_roadmap_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -57,8 +57,15 @@ async def test_scales_dispatch_spawns_only_product_owner() -> None:
@pytest.mark.asyncio
async def test_scales_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
async def test_scales_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _scales_task()
with (
@@ -69,7 +76,10 @@ async def test_scales_dispatch_is_one_shot() -> None:
await orch._dispatch_scales_exploration(task)
await orch._dispatch_scales_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -4
View File
@@ -59,9 +59,15 @@ async def test_sentinel_dispatch_spawns_only_auditor() -> None:
@pytest.mark.asyncio
async def test_sentinel_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending exploration must NOT respawn — board roles
have no progression verb, so a respawn would just loop."""
async def test_sentinel_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _sentinel_task()
with (
@@ -72,7 +78,10 @@ async def test_sentinel_dispatch_is_one_shot() -> None:
await orch._dispatch_sentinel_exploration(task)
await orch._dispatch_sentinel_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -57,8 +57,15 @@ async def test_spackle_dispatch_spawns_only_product_owner() -> None:
@pytest.mark.asyncio
async def test_spackle_dispatch_is_one_shot() -> None:
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
async def test_spackle_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _spackle_task()
with (
@@ -69,7 +76,10 @@ async def test_spackle_dispatch_is_one_shot() -> None:
await orch._dispatch_spackle_exploration(task)
await orch._dispatch_spackle_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio
+13 -3
View File
@@ -67,8 +67,15 @@ async def test_war_room_dispatch_spawns_only_hom() -> None:
@pytest.mark.asyncio
async def test_war_room_dispatch_is_one_shot() -> None:
"""Re-ticking a still-pending campaign-planning task must NOT respawn."""
async def test_war_room_dispatch_retries_until_breaker() -> None:
"""A failed exploration must be retried, not abandoned.
The explorer has a progression verb (``propose_*``), so a respawn CAN
advance the task unlike the two-reviewer review pass this guard was
originally written for. Bounding belongs to
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
not to a never-expiring in-memory set.
"""
orch = _make_orch()
task = _war_room_task()
with (
@@ -79,7 +86,10 @@ async def test_war_room_dispatch_is_one_shot() -> None:
await orch._dispatch_war_room_exploration(task)
await orch._dispatch_war_room_exploration(task)
spawn.assert_awaited_once()
ticks = 2
assert spawn.await_count == ticks, (
"a second tick must re-attempt a failed exploration"
)
@pytest.mark.asyncio