mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(orchestrator): let a failed Board Program exploration retry (#706)
`_board_dispatched` is an in-memory, never-expiring set of (agent_slug, task_id). The solo exploration dispatchers consulted it, so an exploration whose propose verb rejected was never retried for the life of the process — Periscope, Sentinel, Scales and Barfly each spawned once on 2026-07-25, failed, and sat PENDING until the stack restarted. The guard was written for the two-reviewer board REVIEW pass, where it is correct: a reviewer has no verb to advance the task, so a respawn can only loop. An explorer is the opposite — `propose_*` is exactly such a verb, so a respawn can and should advance it. Drop it from the 15 `_dispatch_*_exploration` functions. Bounding falls to `_pm_respawn_should_gate`, which is what actually bounds a loop: DB-persisted, reset by a status change, and cooled down so a deploy that fixes the cause lets the work resume. `_dispatch_board_reviewer` keeps the set (its `_board_review_complete` reads it as a has-run signal) as does vault curation (same-process race guard behind its own durable marker). The 14 `*_dispatch_is_one_shot` tests asserted the old contract on the false rationale that board roles have no progression verb; they now assert a second tick re-attempts. Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -58,9 +58,15 @@ async def test_barfly_dispatch_spawns_only_head_marketing() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_barfly_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending exploration must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_barfly_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _barfly_task()
|
||||
with (
|
||||
@@ -71,7 +77,10 @@ async def test_barfly_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_barfly_exploration(task)
|
||||
await orch._dispatch_barfly_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -67,8 +67,15 @@ async def test_coroner_dispatch_spawns_only_auditor() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_coroner_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending autopsy must NOT respawn."""
|
||||
async def test_coroner_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _coroner_task()
|
||||
with (
|
||||
@@ -80,7 +87,10 @@ async def test_coroner_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_coroner_exploration(task)
|
||||
await orch._dispatch_coroner_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -61,8 +61,15 @@ async def test_dogfood_dispatch_spawns_only_product_owner() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dogfood_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
|
||||
async def test_dogfood_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _dogfood_task()
|
||||
with (
|
||||
@@ -73,7 +80,10 @@ async def test_dogfood_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_dogfood_exploration(task)
|
||||
await orch._dispatch_dogfood_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,9 +57,15 @@ async def test_feature_spotlight_dispatch_spawns_only_head_marketing() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feature_spotlight_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending exploration must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_feature_spotlight_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _feature_task()
|
||||
with (
|
||||
@@ -70,7 +76,10 @@ async def test_feature_spotlight_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_feature_spotlight_exploration(task)
|
||||
await orch._dispatch_feature_spotlight_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -59,9 +59,15 @@ async def test_librarian_dispatch_spawns_only_auditor() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_librarian_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending mining task must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_librarian_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _librarian_task()
|
||||
with (
|
||||
@@ -72,7 +78,10 @@ async def test_librarian_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_librarian_exploration(task)
|
||||
await orch._dispatch_librarian_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -58,9 +58,15 @@ async def test_megaphone_dispatch_spawns_only_head_marketing() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_megaphone_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending exploration must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_megaphone_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _megaphone_task()
|
||||
with (
|
||||
@@ -71,7 +77,10 @@ async def test_megaphone_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_megaphone_exploration(task)
|
||||
await orch._dispatch_megaphone_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,8 +57,15 @@ async def test_mirror_dispatch_spawns_only_head_of_marketing() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mirror_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
|
||||
async def test_mirror_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _mirror_task()
|
||||
with (
|
||||
@@ -69,7 +76,10 @@ async def test_mirror_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_mirror_exploration(task)
|
||||
await orch._dispatch_mirror_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -59,9 +59,15 @@ async def test_periscope_dispatch_spawns_only_head_marketing() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periscope_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending exploration must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_periscope_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _periscope_task()
|
||||
with (
|
||||
@@ -72,7 +78,10 @@ async def test_periscope_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_periscope_exploration(task)
|
||||
await orch._dispatch_periscope_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,8 +57,15 @@ async def test_pest_control_dispatch_spawns_only_product_owner() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pest_control_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
|
||||
async def test_pest_control_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _pest_control_task()
|
||||
with (
|
||||
@@ -69,7 +76,10 @@ async def test_pest_control_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_pest_control_exploration(task)
|
||||
await orch._dispatch_pest_control_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,9 +57,15 @@ async def test_roadmap_dispatch_spawns_only_product_owner() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_roadmap_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn —
|
||||
board roles have no progression verb, so a respawn would just loop."""
|
||||
async def test_roadmap_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _roadmap_task()
|
||||
with (
|
||||
@@ -70,7 +76,10 @@ async def test_roadmap_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_roadmap_exploration(task)
|
||||
await orch._dispatch_roadmap_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,8 +57,15 @@ async def test_scales_dispatch_spawns_only_product_owner() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scales_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
|
||||
async def test_scales_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _scales_task()
|
||||
with (
|
||||
@@ -69,7 +76,10 @@ async def test_scales_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_scales_exploration(task)
|
||||
await orch._dispatch_scales_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -59,9 +59,15 @@ async def test_sentinel_dispatch_spawns_only_auditor() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sentinel_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending exploration must NOT respawn — board roles
|
||||
have no progression verb, so a respawn would just loop."""
|
||||
async def test_sentinel_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _sentinel_task()
|
||||
with (
|
||||
@@ -72,7 +78,10 @@ async def test_sentinel_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_sentinel_exploration(task)
|
||||
await orch._dispatch_sentinel_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -57,8 +57,15 @@ async def test_spackle_dispatch_spawns_only_product_owner() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spackle_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-unauthored, still-pending cycle must NOT respawn."""
|
||||
async def test_spackle_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _spackle_task()
|
||||
with (
|
||||
@@ -69,7 +76,10 @@ async def test_spackle_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_spackle_exploration(task)
|
||||
await orch._dispatch_spackle_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -67,8 +67,15 @@ async def test_war_room_dispatch_spawns_only_hom() -> None:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_war_room_dispatch_is_one_shot() -> None:
|
||||
"""Re-ticking a still-pending campaign-planning task must NOT respawn."""
|
||||
async def test_war_room_dispatch_retries_until_breaker() -> None:
|
||||
"""A failed exploration must be retried, not abandoned.
|
||||
|
||||
The explorer has a progression verb (``propose_*``), so a respawn CAN
|
||||
advance the task — unlike the two-reviewer review pass this guard was
|
||||
originally written for. Bounding belongs to
|
||||
``_pm_respawn_should_gate`` (DB-persisted, reset by a status change),
|
||||
not to a never-expiring in-memory set.
|
||||
"""
|
||||
orch = _make_orch()
|
||||
task = _war_room_task()
|
||||
with (
|
||||
@@ -79,7 +86,10 @@ async def test_war_room_dispatch_is_one_shot() -> None:
|
||||
await orch._dispatch_war_room_exploration(task)
|
||||
await orch._dispatch_war_room_exploration(task)
|
||||
|
||||
spawn.assert_awaited_once()
|
||||
ticks = 2
|
||||
assert spawn.await_count == ticks, (
|
||||
"a second tick must re-attempt a failed exploration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user