mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(notifications): stop tick-wide row locks + poisoned-session swallows behind the PendingRollbackError 500s (#743)
* fix(notifications): release re-escalation row locks per row; stop swallowing DB errors in notify_get The re-escalation sweep ran one tick-wide transaction, so each CAS claim's row lock was held across every remaining delivery until the single commit — a concurrent mark-read UPDATE on a claimed row starved into the 60s lock_timeout. The sweep now commits per row (claim commit releases the lock before delivery and makes the burned slot durable), re-fetches each row by snapshotted id so one row's rollback can't expire the rest of the tick, and savepoints each recipient's delivery. notify_get's bare except swallowed the resulting LockNotAvailableError into a false "notification not found" and returned a poisoned session to the commit-at-send middleware, which blew up with PendingRollbackError; it now catches only the two domain outcomes. defer_after_commit's listeners fire on SAVEPOINT release too, which would have drained deferred telegram/bus work before real durability — they now skip savepoint boundaries via get_nested_transaction() (the root get_transaction() is non-None inside the listener even at a real commit). acknowledge_for_recipient's Redis dedup-clear moved before the flush so the row lock never spans a Redis round-trip. The five best-effort CEO-notify swallows that persist notification rows are savepointed. * fix(services): contain swallowed best-effort DB write failures instead of poisoning the session Sweep of the same class as the notify_get incident: broad except-Exception handlers that swallow a failure whose try-body writes through the shared session leave the session rollback-pending, and the verb/request then dies later with PendingRollbackError at commit-at-send. Confirmed-dangerous sites now run the write inside a savepoint (safe since defer_after_commit skips savepoint boundaries): ceo_approve's verified-stamp, completion/pitch/postmortem-style CEO notifies, _inherit_upstream_base, _link_commit_to_task (covers every commit route), board-program LEARN records, the QA/PR-gate/PM-merge verified-stamps, and the documenter->PM handoff. _ack_pending_wake_notifications gets the same treatment so a wake-ack failure can't fail the A2A read. telegram_inbound's per-update loop and intake confirm roll back explicitly instead (their success paths commit mid-flow, so a savepoint doesn't fit). A swallowed savepoint rollback fully expires any ORM object mutated inside the block, and the next attribute read raises MissingGreenlet — strictly worse than the original bug. The two paths that keep using the object after the swallow (doc handoff's envelope build, base inheritance's claim continuation) refresh it in the except path; regression tests run against a real session and were verified to fail with the refresh reverted. * test: shape mocked session.execute results so sync accessors stop leaking unawaited coroutines An AsyncMock's auto-created children are themselves AsyncMock, so production code that correctly awaits session.execute() and then calls sync accessors (.scalars().all(), .scalar_one_or_none()) on the result was silently collecting unawaited coroutines in 22 test files — 80 RuntimeWarnings per unit run, and in test_flow_soup_guard one mock raised a real TypeError that a coincidentally-matching invalid_state envelope masked. Each affected fixture now returns a plain MagicMock shaped like a real Result. Zero AsyncMock warnings remain. * docs: document per-row sweep commits and the savepoint/refresh containment pattern --------- Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -43,7 +43,7 @@ from roboco.models.base import (
|
||||
from roboco.models.events import EventType
|
||||
from roboco.services.a2a import _LIVE_VIEW_EXCERPT_CHARS, A2AService
|
||||
from roboco.services.gateway.evidence_repo import EvidenceRepo
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy import select as _sel
|
||||
from sqlalchemy.sql.dml import Update
|
||||
|
||||
@@ -2636,3 +2636,55 @@ async def test_read_a2a_acks_pending_wake_notification(a2a_setup: dict) -> None:
|
||||
refreshed = await db.get(NotificationTable, notif_id)
|
||||
assert refreshed is not None
|
||||
assert qa.id in refreshed.acked_by
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ack_pending_wake_notifications_savepoint_isolates_db_failure(
|
||||
a2a_setup: dict,
|
||||
) -> None:
|
||||
"""A genuine Postgres-level failure inside `_ack_pending_wake_notifications`
|
||||
(e.g. `bulk_acknowledge`'s flush hitting a lock timeout) must not poison
|
||||
the shared session for whatever the caller does next. Force a REAL DB
|
||||
error — not a bare Python exception, which alone never aborts the
|
||||
underlying Postgres transaction — via a failing raw statement patched
|
||||
into `bulk_acknowledge`, then prove the session is still usable
|
||||
afterwards (no `PendingRollbackError`)."""
|
||||
svc: A2AService = a2a_setup["svc"]
|
||||
db = a2a_setup["db"]
|
||||
qa = a2a_setup["qa"]
|
||||
dev = a2a_setup["dev"]
|
||||
|
||||
notif = NotificationTable(
|
||||
type=NotificationType.A2A_REQUEST,
|
||||
priority=NotificationPriority.NORMAL,
|
||||
from_agent=dev.id,
|
||||
to_agents=[qa.id],
|
||||
subject="A2A: ceo_dm",
|
||||
body="pending wake",
|
||||
requires_ack=True,
|
||||
)
|
||||
db.add(notif)
|
||||
await db.flush()
|
||||
notif_id = notif.id
|
||||
|
||||
async def _boom(_self: object, *_args: object, **_kwargs: object) -> None:
|
||||
await db.execute(text("SELECT 1/0"))
|
||||
|
||||
with patch(
|
||||
"roboco.services.notification_delivery.NotificationDeliveryService."
|
||||
"bulk_acknowledge",
|
||||
_boom,
|
||||
):
|
||||
await svc._ack_pending_wake_notifications(qa.id) # must not raise
|
||||
|
||||
# The savepoint rolled back the poisoned statement — the shared session
|
||||
# is still usable for later work. Must be a real round trip (`.get()` on
|
||||
# an already-identity-mapped, unexpired object would just return the
|
||||
# cached instance without touching the DB, silently hiding a poisoned
|
||||
# transaction) — `execute(select(...))` always issues the query.
|
||||
refreshed = (
|
||||
await db.execute(
|
||||
select(NotificationTable).where(NotificationTable.id == notif_id)
|
||||
)
|
||||
).scalar_one()
|
||||
assert qa.id not in refreshed.acked_by # the swallowed failure never acked
|
||||
|
||||
@@ -20,7 +20,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
@@ -653,6 +653,142 @@ async def test_doc_path(
|
||||
del cell_pm_agent # asserted indirectly via cell_pm_for_team.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_path_survives_handoff_failure_real_session(
|
||||
db_session: AsyncSession, lifecycle_setup: dict[str, Any]
|
||||
) -> None:
|
||||
"""Round-2 regression (#doc-savepoint-expiry): _handoff_to_cell_pm's
|
||||
`reassign()` mutates + flushes `t` inside the `begin_nested()` savepoint,
|
||||
then `a2a.send` raises. On a REAL AsyncSession the savepoint rollback
|
||||
fully expires every attribute of `t` — reading `t.status` /
|
||||
`with_introspection(task=t, ...)` right after the except without
|
||||
refreshing first raises `MissingGreenlet`, which propagates uncaught and
|
||||
rolls back the WHOLE request (discarding the docs_complete transition the
|
||||
warning claims survived). The equivalent unit test
|
||||
(`test_i_documented_survives_handoff_failure`) mocks the session, so it
|
||||
cannot reproduce this — a mock has no real ORM expiry semantics.
|
||||
"""
|
||||
task = lifecycle_setup["task"]
|
||||
doc_agent = lifecycle_setup["doc_agent"]
|
||||
|
||||
task.status = TaskStatus.AWAITING_DOCUMENTATION
|
||||
task.pr_number = _PR_NUMBER
|
||||
task.pr_url = _PR_URL
|
||||
task.pr_created = True
|
||||
task.qa_verified = True
|
||||
task.assigned_to = None
|
||||
task.commits = [
|
||||
{"sha": uuid4().hex[:40], "message": "feat: /healthz", "task_id": str(task.id)}
|
||||
]
|
||||
await db_session.flush()
|
||||
|
||||
task_service = TaskService(db_session)
|
||||
# reassign() (a real write) runs BEFORE a2a.send inside _handoff_to_cell_pm
|
||||
# — this raises only after that mutation has already flushed.
|
||||
broken_a2a = AsyncMock()
|
||||
broken_a2a.send.side_effect = RuntimeError("a2a down")
|
||||
deps = ChoreographerDeps(
|
||||
task=task_service,
|
||||
work_session=_mock_work_session(),
|
||||
git=_StubGit(db_session, task),
|
||||
a2a=broken_a2a,
|
||||
journal=_mock_journal_with_reflect(),
|
||||
audit=AsyncMock(),
|
||||
evidence_repo=_mock_evidence_repo(),
|
||||
)
|
||||
c = Choreographer(deps)
|
||||
|
||||
env = await c.claim_doc_task(doc_agent.id, task.id)
|
||||
assert env.error is None, f"claim_doc_task failed: {env.message}"
|
||||
|
||||
env = await c.i_documented(
|
||||
doc_agent.id,
|
||||
task.id,
|
||||
notes="Documented /healthz behaviour in docs/api/health.md",
|
||||
files=["docs/api/health.md"],
|
||||
)
|
||||
body = env.as_dict()
|
||||
# Must not 500 / propagate — this is the exact assertion that raises
|
||||
# MissingGreenlet without the session.refresh(t) fix, since `status`
|
||||
# reads `t.status` on the savepoint-expired object.
|
||||
assert body["error"] is None, body
|
||||
assert body.get("warning") is not None
|
||||
assert "handoff" in body["warning"].lower()
|
||||
assert body["status"] == Status.AWAITING_PM_REVIEW.value
|
||||
|
||||
final = await task_service.get(task.id)
|
||||
assert final is not None
|
||||
assert str(final.status) == Status.AWAITING_PM_REVIEW.value
|
||||
assert final.docs_complete is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inherit_upstream_base_survives_flush_failure_real_session(
|
||||
db_session: AsyncSession, lifecycle_setup: dict[str, Any]
|
||||
) -> None:
|
||||
"""Round-2 regression (#task-savepoint-expiry): the conflict branch
|
||||
mutates `task` (the conflict marker + dev_notes) then `flush()`es inside
|
||||
the `begin_nested()` savepoint. On a REAL AsyncSession a flush() failure
|
||||
there rolls back the savepoint and fully expires every attribute of
|
||||
`task` — the real caller (`claim_task_for_agent` ->
|
||||
`_create_work_session_if_needed`) reads `task.project_id`/
|
||||
`task.branch_name` right after this returns; `MissingGreenlet` is not an
|
||||
`AttributeError`, so a `getattr` guard would not shield it, killing the
|
||||
claim despite "never fails the claim". Forces a real, one-shot flush()
|
||||
failure via monkeypatch — the closest realistic trigger to an actual DB
|
||||
hiccup — mirroring `tests/unit/services/test_task_base_inheritance.py`'s
|
||||
own project/git stubbing, but against a REAL session (that unit test's
|
||||
mocked session cannot reproduce ORM expiry at all).
|
||||
"""
|
||||
task = lifecycle_setup["task"]
|
||||
project = lifecycle_setup["project"]
|
||||
task.branch_name = "feature/backend/AAA--BBB"
|
||||
await db_session.flush()
|
||||
|
||||
task_service = TaskService(db_session)
|
||||
|
||||
# Drive _inherit_upstream_base into the "conflict" branch (mutate +
|
||||
# flush), mirroring the unit test's own project/git stubbing.
|
||||
proj_svc = MagicMock()
|
||||
proj_svc.get = AsyncMock(return_value=project)
|
||||
git_svc = MagicMock()
|
||||
git_svc.get_workspace = AsyncMock(return_value=MagicMock())
|
||||
git_svc.merge_dependency_lineage = AsyncMock(
|
||||
return_value={"status": "conflict", "files": ["a.py"]}
|
||||
)
|
||||
object.__setattr__(
|
||||
task_service,
|
||||
"_resolve_parent_branch",
|
||||
AsyncMock(return_value="feature/main_pm/root"),
|
||||
)
|
||||
|
||||
real_flush = db_session.flush
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _flush_once_boom() -> None:
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise RuntimeError("simulated flush failure")
|
||||
await real_flush()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"roboco.services.project.get_project_service",
|
||||
MagicMock(return_value=proj_svc),
|
||||
),
|
||||
patch("roboco.services.git.get_git_service", MagicMock(return_value=git_svc)),
|
||||
patch.object(db_session, "flush", _flush_once_boom),
|
||||
):
|
||||
await task_service._inherit_upstream_base(task, uuid4()) # must not raise
|
||||
|
||||
# `task` must be readable afterward — the exact access pattern
|
||||
# `_create_work_session_if_needed` performs right after this call
|
||||
# returns in the real claim flow. Raises MissingGreenlet without the
|
||||
# session.refresh(task) fix in the except block.
|
||||
assert task.project_id == project.id
|
||||
assert task.branch_name == "feature/backend/AAA--BBB"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. PM complete (Cell PM, simple task): awaiting_pm_review → completed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -10,22 +10,104 @@ issues a real `expires_at < now()` query, so a mocked session (as
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from roboco.config import settings
|
||||
from roboco.db.tables import AgentTable, NotificationTable
|
||||
from roboco.models import AgentRole, AgentStatus, NotificationPriority, NotificationType
|
||||
from roboco.models.base import Team
|
||||
from roboco.models.notification import CreateNotificationParams
|
||||
from roboco.services.notification import NotificationService
|
||||
from roboco.services.notification_delivery import get_notification_delivery_service
|
||||
from sqlalchemy import select
|
||||
from roboco.services.notification_delivery import (
|
||||
NotificationDeliveryService,
|
||||
defer_after_commit,
|
||||
get_notification_delivery_service,
|
||||
)
|
||||
from sqlalchemy import and_, delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def _committed_notification_senders(
|
||||
_test_database_url: str,
|
||||
) -> AsyncIterator[list[UUID]]:
|
||||
"""Cleans up NotificationTable rows a test durably committed.
|
||||
|
||||
The sweep's per-row commit (#Correction 2) means a test that reaches
|
||||
`sweep_expired_notifications` leaves its notification row(s) durable in
|
||||
this session-scoped shared test DB — NOT teardown-rolled-back like
|
||||
`db_session`'s own uncommitted work — which then pollutes a LATER test
|
||||
file's system-wide notification listing (`test_notification_system_list`).
|
||||
|
||||
A test that commits appends its seeded sender agent id(s) to the yielded
|
||||
list. A re-escalation row always inherits the ORIGINAL notification's
|
||||
`from_agent` (see `_re_escalate_recipient`), so deleting every
|
||||
NotificationTable row `from_agent`-matched to a test's sender(s) catches
|
||||
both the original row and anything it spawned — the most precise handle
|
||||
available, and it needs no separate tracking of recipient/target ids or
|
||||
the re-escalation rows' own ids.
|
||||
|
||||
Runs on a wholly separate engine/connection (independent of `db_session`'s
|
||||
own transaction state) so it always reaches Postgres regardless of what
|
||||
`db_session` itself still has open when this fixture tears down.
|
||||
"""
|
||||
sender_ids: list[UUID] = []
|
||||
yield sender_ids
|
||||
if not sender_ids:
|
||||
return
|
||||
engine = create_async_engine(_test_database_url, future=True)
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(
|
||||
delete(NotificationTable).where(
|
||||
NotificationTable.from_agent.in_(sender_ids)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _await_drain(session: AsyncSession) -> None:
|
||||
"""Await any `defer_after_commit` drain tasks stashed on the session
|
||||
(mirrors `test_notification_delivery_phantom.py`'s helper) — the real
|
||||
drain is fire-and-forget via `asyncio.create_task`, so a test must await
|
||||
it explicitly rather than racing the event loop."""
|
||||
tasks = list(session.info.get("_roboco_drain_tasks", []))
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def _stale_unacked_count(session: AsyncSession) -> int:
|
||||
"""Mirrors `sweep_expired_notifications`'s own stale-unacked predicate.
|
||||
|
||||
The sweep's per-row commit (already true before this file's fixes —
|
||||
see the `>= 1` assertions above) makes prior tests' rows durable in this
|
||||
session-scoped shared test DB rather than teardown-rolled-back, so a new
|
||||
test can't assert an exact sweep `count` — it must diff against this
|
||||
baseline instead.
|
||||
"""
|
||||
result = await session.execute(
|
||||
select(NotificationTable).where(
|
||||
and_(
|
||||
NotificationTable.expires_at.is_not(None),
|
||||
NotificationTable.expires_at < datetime.now(UTC),
|
||||
NotificationTable.requires_ack.is_(True),
|
||||
)
|
||||
)
|
||||
)
|
||||
return sum(
|
||||
1
|
||||
for n in result.scalars().all()
|
||||
if not NotificationDeliveryService._notification_is_fully_acked(n)
|
||||
)
|
||||
|
||||
|
||||
async def _seed_agent(db: AsyncSession, *, role: AgentRole, slug: str) -> UUID:
|
||||
@@ -50,6 +132,7 @@ async def _seed_agent(db: AsyncSession, *, role: AgentRole, slug: str) -> UUID:
|
||||
@pytest.mark.asyncio
|
||||
async def test_created_notification_expires_at_is_stamped_and_matched_by_sweep(
|
||||
db_session: AsyncSession,
|
||||
_committed_notification_senders: list[UUID],
|
||||
) -> None:
|
||||
"""End-to-end: NotificationService._create_notification stamps expires_at
|
||||
for an ack-required row, and once that deadline is in the past,
|
||||
@@ -63,6 +146,7 @@ async def test_created_notification_expires_at_is_stamped_and_matched_by_sweep(
|
||||
recipient = await _seed_agent(
|
||||
db_session, role=AgentRole.CELL_PM, slug=f"pm-{unique}"
|
||||
)
|
||||
_committed_notification_senders.append(sender)
|
||||
|
||||
svc = NotificationService()
|
||||
await svc._create_notification(
|
||||
@@ -102,6 +186,7 @@ async def test_created_notification_expires_at_is_stamped_and_matched_by_sweep(
|
||||
@pytest.mark.asyncio
|
||||
async def test_directly_stamped_expired_row_is_matched_by_sweep_query(
|
||||
db_session: AsyncSession,
|
||||
_committed_notification_senders: list[UUID],
|
||||
) -> None:
|
||||
"""Isolates the sweep query mechanics from creation: a hand-built
|
||||
ack-required, unacked row with expires_at in the past must be counted."""
|
||||
@@ -110,6 +195,7 @@ async def test_directly_stamped_expired_row_is_matched_by_sweep_query(
|
||||
db_session, role=AgentRole.DEVELOPER, slug=f"s2-{unique}"
|
||||
)
|
||||
recipient = await _seed_agent(db_session, role=AgentRole.QA, slug=f"r2-{unique}")
|
||||
_committed_notification_senders.append(sender)
|
||||
|
||||
notification = NotificationTable(
|
||||
type=NotificationType.ALERT,
|
||||
@@ -167,3 +253,297 @@ async def test_zero_ttl_disables_expires_at_stamping_end_to_end(
|
||||
)
|
||||
).scalar_one()
|
||||
assert row.expires_at is None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# defer_after_commit vs. begin_nested() savepoints
|
||||
#
|
||||
# `_re_escalate_recipient` wraps `_persist_and_deliver` (which calls
|
||||
# `deliver()` -> `defer_bus_publish` -> `defer_after_commit`) in a
|
||||
# `begin_nested()` savepoint, per row, inside this very sweep. SQLAlchemy
|
||||
# dispatches `after_commit`/`after_rollback` on a SAVEPOINT release/rollback
|
||||
# too (verified live against this Postgres), so without the fix the pending
|
||||
# work would drain right there — before the sweep's own real per-row
|
||||
# `session.commit()` — reintroducing the phantom-notification bug the
|
||||
# outbox exists to prevent.
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_defer_after_commit_does_not_drain_at_savepoint_release(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A `begin_nested()` release must NOT drain pending work — only the
|
||||
real root commit may."""
|
||||
ran: list[str] = []
|
||||
|
||||
async def _work() -> None:
|
||||
ran.append("ran")
|
||||
|
||||
defer_after_commit(db_session, _work)
|
||||
|
||||
async with db_session.begin_nested():
|
||||
pass # savepoint opens and releases; the root transaction stays open
|
||||
|
||||
await _await_drain(db_session)
|
||||
assert ran == [] # not drained at the savepoint boundary
|
||||
|
||||
await db_session.commit()
|
||||
await _await_drain(db_session)
|
||||
assert ran == ["ran"] # drained at the real root commit
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_defer_after_commit_discarded_on_root_rollback(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A real root rollback discards pending work — no phantom event for
|
||||
work whose enclosing transaction never became durable."""
|
||||
await db_session.execute(select(1)) # force a real root txn to open
|
||||
|
||||
ran: list[str] = []
|
||||
|
||||
async def _work() -> None:
|
||||
ran.append("ran")
|
||||
|
||||
defer_after_commit(db_session, _work)
|
||||
await db_session.rollback()
|
||||
await _await_drain(db_session)
|
||||
|
||||
assert ran == []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Sweep per-row isolation (#Correction 1/2): one row's failure must not
|
||||
# corrupt or block another row's processing in the same tick, and a row
|
||||
# that does succeed must be durably committed regardless.
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_delivers_via_resolvable_chain_and_commit_is_durable(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
_test_database_url: str,
|
||||
_committed_notification_senders: list[UUID],
|
||||
) -> None:
|
||||
"""The 3 tests above never seed a resolvable escalation chain, so
|
||||
`_re_escalate_recipient` always short-circuits at `get_escalation_target`
|
||||
returning None before ever reaching `_persist_and_deliver` — a due row's
|
||||
real delivery path stays unexercised. Seed one row with a resolvable
|
||||
chain plus a second row whose processing is forced to raise, and confirm
|
||||
the first row really delivers (a new escalation NotificationTable row
|
||||
addressed to the resolved target) and its `reescalation_count` bump is
|
||||
durably committed — visible from a wholly separate connection, not just
|
||||
this session's own (rollback-able) view — regardless of the other row's
|
||||
failure.
|
||||
"""
|
||||
unique = uuid4().hex[:8]
|
||||
sender = await _seed_agent(
|
||||
db_session, role=AgentRole.DEVELOPER, slug=f"snd-{unique}"
|
||||
)
|
||||
recipient = await _seed_agent(
|
||||
db_session, role=AgentRole.CELL_PM, slug=f"rcp-{unique}"
|
||||
)
|
||||
target = await _seed_agent(db_session, role=AgentRole.MAIN_PM, slug=f"tgt-{unique}")
|
||||
_committed_notification_senders.append(sender)
|
||||
baseline = await _stale_unacked_count(db_session)
|
||||
|
||||
good = NotificationTable(
|
||||
type=NotificationType.BLOCKER_ESCALATION,
|
||||
priority=NotificationPriority.HIGH,
|
||||
from_agent=sender,
|
||||
to_agents=[recipient],
|
||||
subject=f"good-{unique}",
|
||||
body="body",
|
||||
requires_ack=True,
|
||||
expires_at=datetime.now(UTC) - timedelta(minutes=5),
|
||||
)
|
||||
bad = NotificationTable(
|
||||
type=NotificationType.ALERT,
|
||||
priority=NotificationPriority.HIGH,
|
||||
from_agent=sender,
|
||||
to_agents=[recipient],
|
||||
subject=f"bad-{unique}",
|
||||
body="body",
|
||||
requires_ack=True,
|
||||
expires_at=datetime.now(UTC) - timedelta(minutes=5),
|
||||
)
|
||||
db_session.add_all([good, bad])
|
||||
await db_session.flush()
|
||||
good_id = cast("UUID", good.id)
|
||||
bad_id = cast("UUID", bad.id)
|
||||
|
||||
# This test only needs the escalation chain to resolve; who it resolves
|
||||
# to doesn't depend on the recipient slug (real `ESCALATION_CHAIN` keys
|
||||
# are fixed strings we can't safely reuse across tests without colliding
|
||||
# on `agents.slug`'s uniqueness once the fix's per-row commit makes these
|
||||
# rows durable rather than teardown-rolled-back).
|
||||
monkeypatch.setattr(
|
||||
"roboco.services.notification_delivery.get_escalation_target",
|
||||
lambda _slug: f"tgt-{unique}",
|
||||
)
|
||||
|
||||
# Fault-inject in `_re_escalate_unacked`, not `_maybe_reescalate` itself:
|
||||
# the sweep has no ORDER BY, so whichever row it reaches first must not
|
||||
# matter. `_maybe_reescalate`'s own CAS-claim commit runs BEFORE this
|
||||
# call, so by the time "bad" raises, that commit has already flushed
|
||||
# whatever was pending — including the initial insert of BOTH rows,
|
||||
# which were only `flush()`-ed (not committed) before this sweep call.
|
||||
# Raising any earlier (inside `_maybe_reescalate` itself, before its own
|
||||
# commit) would let "bad" processed first roll back "good"'s still-
|
||||
# uncommitted insert too — an order dependency, not a real assertion.
|
||||
orig_re_escalate_unacked = NotificationDeliveryService._re_escalate_unacked
|
||||
|
||||
async def _re_escalate_unacked_one_fails(
|
||||
self: NotificationDeliveryService, n: NotificationTable
|
||||
) -> int:
|
||||
if n.id == bad_id:
|
||||
raise RuntimeError("simulated processing failure")
|
||||
return await orig_re_escalate_unacked(self, n)
|
||||
|
||||
monkeypatch.setattr(
|
||||
NotificationDeliveryService,
|
||||
"_re_escalate_unacked",
|
||||
_re_escalate_unacked_one_fails,
|
||||
)
|
||||
|
||||
deliv = get_notification_delivery_service(db_session)
|
||||
count = await deliv.sweep_expired_notifications()
|
||||
# Both new rows still counted as stale + unacked this tick, on top of
|
||||
# whatever earlier tests in this session-scoped DB already committed.
|
||||
assert count == baseline + 2
|
||||
|
||||
# Verify durability from a SEPARATE connection bound to the same test
|
||||
# DB — proves the good row's commit really reached Postgres, not just
|
||||
# this session's own still-mutable view.
|
||||
verify_engine = create_async_engine(_test_database_url, future=True)
|
||||
try:
|
||||
verify_factory = async_sessionmaker(bind=verify_engine, class_=AsyncSession)
|
||||
async with verify_factory() as verify_session:
|
||||
good_row = (
|
||||
await verify_session.execute(
|
||||
select(NotificationTable).where(NotificationTable.id == good_id)
|
||||
)
|
||||
).scalar_one()
|
||||
assert good_row.reescalation_count == 1
|
||||
assert good_row.reescalation_delivered_count == 1
|
||||
|
||||
escalated = (
|
||||
(
|
||||
await verify_session.execute(
|
||||
select(NotificationTable).where(
|
||||
NotificationTable.to_agents.contains([target])
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
finally:
|
||||
await verify_engine.dispose()
|
||||
|
||||
assert any(f"good-{unique}" in (e.subject or "") for e in escalated)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_one_row_failure_does_not_block_the_other_rows_processing(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
_test_database_url: str,
|
||||
_committed_notification_senders: list[UUID],
|
||||
) -> None:
|
||||
"""Before the fix, a root `rollback()` following one row's exception
|
||||
expired every object still held from the earlier SELECT, so whichever
|
||||
row `unacked` iterated to next raised `MissingGreenlet` on its own
|
||||
attribute access (an async lazy-refresh attempted in a sync/greenlet
|
||||
context) — aborting the whole tick instead of just skipping the bad row.
|
||||
Snapshotting ids and re-fetching each row via `session.get` (async-safe
|
||||
even post-expiry) makes every row's processing independent of any
|
||||
earlier row's failure."""
|
||||
unique = uuid4().hex[:8]
|
||||
sender = await _seed_agent(db_session, role=AgentRole.DEVELOPER, slug=f"s-{unique}")
|
||||
good_recipient = await _seed_agent(
|
||||
db_session, role=AgentRole.QA, slug=f"gr-{unique}"
|
||||
)
|
||||
bad_recipient = await _seed_agent(
|
||||
db_session, role=AgentRole.QA, slug=f"br-{unique}"
|
||||
)
|
||||
_committed_notification_senders.append(sender)
|
||||
baseline = await _stale_unacked_count(db_session)
|
||||
|
||||
good = NotificationTable(
|
||||
type=NotificationType.ALERT,
|
||||
priority=NotificationPriority.HIGH,
|
||||
from_agent=sender,
|
||||
to_agents=[good_recipient],
|
||||
subject=f"good-{unique}",
|
||||
body="body",
|
||||
requires_ack=True,
|
||||
expires_at=datetime.now(UTC) - timedelta(minutes=5),
|
||||
)
|
||||
bad = NotificationTable(
|
||||
type=NotificationType.ALERT,
|
||||
priority=NotificationPriority.HIGH,
|
||||
from_agent=sender,
|
||||
to_agents=[bad_recipient],
|
||||
subject=f"bad-{unique}",
|
||||
body="body",
|
||||
requires_ack=True,
|
||||
expires_at=datetime.now(UTC) - timedelta(minutes=5),
|
||||
)
|
||||
db_session.add_all([good, bad])
|
||||
await db_session.flush()
|
||||
good_id = cast("UUID", good.id)
|
||||
bad_id = cast("UUID", bad.id)
|
||||
|
||||
# No resolvable chain for either row — isolates this test to the
|
||||
# row-isolation property alone (delivery-path coverage is the other test).
|
||||
monkeypatch.setattr(
|
||||
"roboco.services.notification_delivery.get_escalation_target",
|
||||
lambda _slug: None,
|
||||
)
|
||||
|
||||
orig_re_escalate_unacked = NotificationDeliveryService._re_escalate_unacked
|
||||
|
||||
async def _re_escalate_unacked_one_fails(
|
||||
self: NotificationDeliveryService, n: NotificationTable
|
||||
) -> int:
|
||||
if n.id == bad_id:
|
||||
raise RuntimeError("simulated delivery failure")
|
||||
return await orig_re_escalate_unacked(self, n)
|
||||
|
||||
monkeypatch.setattr(
|
||||
NotificationDeliveryService,
|
||||
"_re_escalate_unacked",
|
||||
_re_escalate_unacked_one_fails,
|
||||
)
|
||||
|
||||
deliv = get_notification_delivery_service(db_session)
|
||||
count = await deliv.sweep_expired_notifications()
|
||||
assert count == baseline + 2 # both new rows still stale + unacked
|
||||
|
||||
verify_engine = create_async_engine(_test_database_url, future=True)
|
||||
try:
|
||||
verify_factory = async_sessionmaker(bind=verify_engine, class_=AsyncSession)
|
||||
async with verify_factory() as verify_session:
|
||||
good_row = (
|
||||
await verify_session.execute(
|
||||
select(NotificationTable).where(NotificationTable.id == good_id)
|
||||
)
|
||||
).scalar_one()
|
||||
bad_row = (
|
||||
await verify_session.execute(
|
||||
select(NotificationTable).where(NotificationTable.id == bad_id)
|
||||
)
|
||||
).scalar_one()
|
||||
finally:
|
||||
await verify_engine.dispose()
|
||||
|
||||
# The good row was fully processed (attempt slot claimed + committed)
|
||||
# independent of whatever happened to the bad row.
|
||||
assert good_row.reescalation_count == 1
|
||||
# The bad row's CAS-claim commit (before the raise) is also durable —
|
||||
# the raise happens in `_re_escalate_unacked`, strictly after that commit.
|
||||
assert bad_row.reescalation_count == 1
|
||||
assert bad_row.reescalation_delivered_count == 0
|
||||
|
||||
@@ -35,6 +35,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -40,6 +40,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -41,6 +41,15 @@ def _over_cap_project() -> MagicMock:
|
||||
|
||||
|
||||
def _make_deps(task_svc: AsyncMock, **overrides: Any) -> ChoreographerDeps:
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
base: dict[str, Any] = {
|
||||
"task": task_svc,
|
||||
"work_session": AsyncMock(),
|
||||
|
||||
@@ -37,6 +37,15 @@ def _choreographer(
|
||||
repo.similar_memory = AsyncMock(return_value=similar_memory_out)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.agent_for.return_value = MagicMock(role="developer")
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
choreo = object.__new__(Choreographer)
|
||||
choreo._deps = MagicMock(evidence_repo=repo, task=task_svc)
|
||||
return choreo, repo
|
||||
|
||||
@@ -22,6 +22,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
@@ -215,6 +224,7 @@ async def test_i_documented_succeeds_and_transitions() -> None:
|
||||
task_svc.cell_pm_for_team.return_value = MagicMock(id=uuid4())
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.flush = AsyncMock()
|
||||
task_svc.session.refresh = AsyncMock()
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
@@ -252,6 +262,7 @@ def _doc_success_task_svc(task_id: Any, doc_id: Any) -> AsyncMock:
|
||||
task_svc.cell_pm_for_team.return_value = MagicMock(id=uuid4())
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.flush = AsyncMock()
|
||||
task_svc.session.refresh = AsyncMock()
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
@@ -352,6 +363,7 @@ async def test_i_documented_survives_handoff_failure() -> None:
|
||||
task_svc.cell_pm_for_team.return_value = MagicMock(id=uuid4())
|
||||
task_svc.session = MagicMock()
|
||||
task_svc.session.flush = AsyncMock()
|
||||
task_svc.session.refresh = AsyncMock()
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
|
||||
@@ -40,6 +40,15 @@ def _dev_agent_task_svc() -> tuple[AsyncMock, UUID]:
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
# Default: lane clear (no earlier incomplete sibling).
|
||||
task_svc.has_earlier_incomplete_code_sibling.return_value = False
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
return task_svc, uuid4()
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,18 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
# That same verified-stamp now runs inside its own savepoint (a mid-flush
|
||||
# failure must not poison the shared session) — an unconfigured
|
||||
# AsyncMock's begin_nested() call returns a raw unawaited coroutine,
|
||||
# which `async with` cannot use. Same shape as the execute default above;
|
||||
# a test's own explicit begin_nested config (e.g. submit_root's) is the
|
||||
# identical shape, so overwriting it here is a no-op for those tests.
|
||||
base["task"].session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
return ChoreographerDeps(**base)
|
||||
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ async def test_claim_review_returns_evidence_inline() -> None:
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.qa_claim.return_value = t_claimed
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
work_svc = AsyncMock()
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "+++ diff content"
|
||||
@@ -146,6 +147,7 @@ async def test_claim_review_marks_evidence_inspected() -> None:
|
||||
task_svc.list_in_progress_for_agent.return_value = []
|
||||
task_svc.list_paused_for_agent.return_value = []
|
||||
task_svc.qa_claim.return_value = t_claimed
|
||||
_stub_empty_ledger(task_svc.session)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = ""
|
||||
deps = _make_deps(task=task_svc, git=git_svc)
|
||||
|
||||
@@ -51,6 +51,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -31,6 +31,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for m in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -644,6 +644,15 @@ async def test_evidence_valid_task_returns_ok_with_pr_diff() -> None:
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = task_obj
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff --git a/foo.py b/foo.py\n+added line"
|
||||
workspace_svc = AsyncMock()
|
||||
|
||||
@@ -409,6 +409,15 @@ async def test_evidence_unassigned_task_allows_inspection() -> None:
|
||||
)
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = task_obj
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = "diff content"
|
||||
workspace_svc = AsyncMock()
|
||||
@@ -445,6 +454,15 @@ async def test_evidence_allows_dependency_inspection() -> None:
|
||||
task_svc = AsyncMock()
|
||||
task_svc.get.return_value = target
|
||||
task_svc.list_assigned_for_agent.return_value = [callers_task]
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
git_svc = AsyncMock()
|
||||
git_svc.diff.return_value = ""
|
||||
git_svc.list_changed_files.return_value = []
|
||||
|
||||
@@ -29,6 +29,15 @@ def _deps_for_evidence(
|
||||
workspace_svc: AsyncMock,
|
||||
evidence_repo: AsyncMock,
|
||||
) -> ContentActionsDeps:
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
return ContentActionsDeps(
|
||||
task=task_svc,
|
||||
git=git_svc,
|
||||
|
||||
@@ -38,6 +38,17 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings); a test that
|
||||
# needs real ledger data monkeypatches the findings module functions
|
||||
# directly (see below), so this default never masks that.
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -115,6 +115,18 @@ def _make_deps(agent_id: object, task_id: object) -> ChoreographerDeps:
|
||||
task_svc.agent_for.return_value = MagicMock(
|
||||
id=agent_id, role="developer", team="backend", slug="be-dev-1"
|
||||
)
|
||||
# VerbRunner uses task.session.begin_nested() as a savepoint context
|
||||
# manager — an unconfigured AsyncMock's `begin_nested()` call returns a
|
||||
# raw unawaited coroutine, which `async with` cannot use (real failure,
|
||||
# not just a warning: it was silently turning into a masking
|
||||
# "verb runner failed" invalid_state envelope instead of exercising the
|
||||
# real block path below).
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(
|
||||
__aenter__=AsyncMock(return_value=None),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
)
|
||||
evidence_repo = AsyncMock()
|
||||
for m in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -23,6 +23,14 @@ from roboco.services.gateway.choreographer import Choreographer, ChoreographerDe
|
||||
|
||||
|
||||
def _make_choreographer(*, task_service: AsyncMock, git: AsyncMock) -> Choreographer:
|
||||
# `_project_slug_for`/`ReviewFindingsRepository.list_for_task` both read
|
||||
# via session.execute — an unconfigured AsyncMock's awaited result is
|
||||
# itself an AsyncMock, so a plain sync `.scalars()`/`.scalar_one_or_none()`
|
||||
# call on it leaks an unawaited coroutine. A bare MagicMock's `.scalars()
|
||||
# .all()` already returns `[]` by default; these tests don't assert on
|
||||
# the resolved project/slug, so a default (truthy) `.scalar_one_or_none()`
|
||||
# is harmless too.
|
||||
task_service.session.execute = AsyncMock(return_value=MagicMock())
|
||||
return Choreographer(
|
||||
ChoreographerDeps(
|
||||
task=task_service,
|
||||
|
||||
@@ -33,6 +33,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -53,6 +53,15 @@ def test_pm_claim_needs_revision_works_for_code_typed_root() -> None:
|
||||
|
||||
|
||||
def _make_deps(task_svc: AsyncMock) -> ChoreographerDeps:
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = AsyncMock()
|
||||
for m in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Tests for ContentActions.notify_get — read-one-notification (marks read).
|
||||
|
||||
`notify_get` used to swallow ANY exception from
|
||||
`get_for_recipient_and_mark_read` into `Envelope.not_found` — including a DB
|
||||
error from the mark-read UPDATE (e.g. hitting `lock_timeout`), which poisoned
|
||||
the session for the rest of the transaction and surfaced later as an opaque
|
||||
`PendingRollbackError`, while also lying to the calling agent that an
|
||||
existing notification didn't exist. The fix narrows the catch to the two real
|
||||
domain outcomes (`NotFoundError`, `PermissionError`); anything else must
|
||||
propagate so the session actually rolls back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from roboco.services.base import NotFoundError
|
||||
from roboco.services.gateway.content_actions import ContentActions, ContentActionsDeps
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
|
||||
def _make_deps(**overrides: AsyncMock) -> ContentActionsDeps:
|
||||
task = overrides.get("task", AsyncMock())
|
||||
git = overrides.get("git", AsyncMock())
|
||||
a2a = overrides.get("a2a", AsyncMock())
|
||||
journal = overrides.get("journal", AsyncMock())
|
||||
workspace = overrides.get("workspace", AsyncMock())
|
||||
notifications = overrides.get("notifications", AsyncMock())
|
||||
notification_delivery = overrides.get("notification_delivery", AsyncMock())
|
||||
return ContentActionsDeps(
|
||||
task=task,
|
||||
git=git,
|
||||
a2a=a2a,
|
||||
journal=journal,
|
||||
workspace=workspace,
|
||||
notifications=notifications,
|
||||
notification_delivery=notification_delivery,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_get_not_found_error_maps_to_not_found() -> None:
|
||||
"""A genuinely missing notification -> Envelope.not_found."""
|
||||
notification_id = uuid4()
|
||||
notif_delivery = AsyncMock()
|
||||
notif_delivery.get_for_recipient_and_mark_read.side_effect = NotFoundError(
|
||||
resource_type="Notification", resource_id=str(notification_id)
|
||||
)
|
||||
deps = _make_deps(notification_delivery=notif_delivery)
|
||||
ca = ContentActions(deps)
|
||||
|
||||
env = await ca.notify_get(agent_id=uuid4(), notification_id=notification_id)
|
||||
body = env.as_dict()
|
||||
|
||||
assert body["error"] == "not_found"
|
||||
assert str(notification_id) in body["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_get_permission_error_maps_to_not_found() -> None:
|
||||
"""A recipient mismatch -> Envelope.not_found (never leaks a 403/details)."""
|
||||
notification_id = uuid4()
|
||||
notif_delivery = AsyncMock()
|
||||
notif_delivery.get_for_recipient_and_mark_read.side_effect = PermissionError(
|
||||
"view notification: not a recipient"
|
||||
)
|
||||
deps = _make_deps(notification_delivery=notif_delivery)
|
||||
ca = ContentActions(deps)
|
||||
|
||||
env = await ca.notify_get(agent_id=uuid4(), notification_id=notification_id)
|
||||
body = env.as_dict()
|
||||
|
||||
assert body["error"] == "not_found"
|
||||
assert str(notification_id) in body["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_get_db_error_propagates() -> None:
|
||||
"""A DB-shaped failure (e.g. the mark-read UPDATE hitting lock_timeout)
|
||||
must NOT be swallowed into not_found — it has to propagate so the
|
||||
session actually rolls back instead of silently poisoning the
|
||||
transaction for the caller's later commit."""
|
||||
notification_id = uuid4()
|
||||
notif_delivery = AsyncMock()
|
||||
notif_delivery.get_for_recipient_and_mark_read.side_effect = OperationalError(
|
||||
"UPDATE notifications ...", {}, Exception("lock timeout")
|
||||
)
|
||||
deps = _make_deps(notification_delivery=notif_delivery)
|
||||
ca = ContentActions(deps)
|
||||
|
||||
with pytest.raises(OperationalError):
|
||||
await ca.notify_get(agent_id=uuid4(), notification_id=notification_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notify_get_success_returns_notification_and_marks_read() -> None:
|
||||
"""Happy path: the resolved notification's fields land in evidence."""
|
||||
notification_id = uuid4()
|
||||
from_agent = uuid4()
|
||||
n = MagicMock()
|
||||
n.id = notification_id
|
||||
n.type = "alert"
|
||||
n.priority = "normal"
|
||||
n.subject = "subject line"
|
||||
n.body = "body text"
|
||||
n.requires_ack = False
|
||||
n.from_agent = from_agent
|
||||
|
||||
notif_delivery = AsyncMock()
|
||||
notif_delivery.get_for_recipient_and_mark_read.return_value = n
|
||||
deps = _make_deps(notification_delivery=notif_delivery)
|
||||
ca = ContentActions(deps)
|
||||
|
||||
env = await ca.notify_get(agent_id=uuid4(), notification_id=notification_id)
|
||||
body = env.as_dict()
|
||||
|
||||
assert body["error"] is None
|
||||
assert body["evidence"]["id"] == str(notification_id)
|
||||
assert body["evidence"]["subject"] == "subject line"
|
||||
assert body["evidence"]["from_agent"] == str(from_agent)
|
||||
notif_delivery.get_for_recipient_and_mark_read.assert_awaited_once()
|
||||
@@ -35,6 +35,15 @@ def _make_deps(**overrides: Any) -> ChoreographerDeps:
|
||||
"evidence_repo": AsyncMock(),
|
||||
}
|
||||
base.update(overrides)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
base["task"].session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
repo = base["evidence_repo"]
|
||||
for method in (
|
||||
"list_unread_a2a",
|
||||
|
||||
@@ -89,6 +89,15 @@ def _resubmit_root(
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
||||
)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
c = Choreographer(_make_deps(task=task_svc, git=AsyncMock()))
|
||||
# Real _project_slug_for would walk a mock session into a MagicMock slug; the
|
||||
# gate under test needs a real string slug + a controllable head SHA. Alias to
|
||||
|
||||
@@ -72,6 +72,15 @@ def _resubmit_cell(
|
||||
task_svc.session.begin_nested = MagicMock(
|
||||
return_value=MagicMock(__aenter__=AsyncMock(), __aexit__=AsyncMock())
|
||||
)
|
||||
# Findings-ledger reads (ReviewFindingsRepository.list_for_task) go
|
||||
# through session.execute — an unconfigured AsyncMock's awaited result
|
||||
# is itself an AsyncMock, so a plain sync `.scalars()` call on it leaks
|
||||
# an unawaited coroutine. Empty scalars result (no findings).
|
||||
task_svc.session.execute = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))
|
||||
)
|
||||
)
|
||||
c = Choreographer(_make_deps(task=task_svc, git=AsyncMock()))
|
||||
cc: Any = c
|
||||
cc._project_slug_for = AsyncMock(return_value="proj-slug")
|
||||
|
||||
@@ -111,10 +111,23 @@ def _session_returning(
|
||||
"""A session whose SELECT (the sweep's stale-notifications query) returns
|
||||
`notifications`; every re-escalation CAS UPDATE (`_claim_reescalation_slot`)
|
||||
reports 1 row affected — the claim wins — unless `claim_succeeds` is False,
|
||||
simulating a concurrent sweep tick that already claimed this row's slot."""
|
||||
simulating a concurrent sweep tick that already claimed this row's slot.
|
||||
|
||||
`commit`/`rollback` are awaitable no-ops (the per-row commit scope in
|
||||
`sweep_expired_notifications`/`_maybe_reescalate` awaits both); `get`
|
||||
is an awaitable lookup against `notifications` by id (the sweep loop
|
||||
re-fetches each row post-snapshot via `session.get` instead of
|
||||
iterating the ORM instances directly)."""
|
||||
session = MagicMock()
|
||||
session.add = MagicMock(side_effect=_assign_id_on_add)
|
||||
session.flush = AsyncMock()
|
||||
session.commit = AsyncMock()
|
||||
session.rollback = AsyncMock()
|
||||
|
||||
async def _get(_model: Any, ident: Any) -> MagicMock | None:
|
||||
return next((n for n in notifications if n.id == ident), None)
|
||||
|
||||
session.get = AsyncMock(side_effect=_get)
|
||||
|
||||
select_result = MagicMock()
|
||||
select_result.scalars.return_value.all.return_value = notifications
|
||||
|
||||
@@ -40,6 +40,23 @@ def _slug_row(slug: str) -> MagicMock:
|
||||
return MagicMock(scalar_one_or_none=MagicMock(return_value=slug))
|
||||
|
||||
|
||||
def _empty_result() -> MagicMock:
|
||||
"""A `session.execute(...)` result shaped for every sync accessor real
|
||||
code calls on it (`.scalar_one_or_none()`, `.one_or_none()`,
|
||||
`.scalars().all()`) — all "nothing found" defaults. A bare unconfigured
|
||||
`AsyncMock()` leaks an unawaited coroutine on each of those (its
|
||||
attributes are AsyncMock too, unlike MagicMock's), and a plain
|
||||
`MagicMock()`'s `.scalar_one_or_none()`/`.one_or_none()` would each
|
||||
default to a truthy MagicMock instead of the `None` real callers here
|
||||
(`_get_ceo_agent`, `MetricsService.get_task_metrics`) treat as "not
|
||||
found"."""
|
||||
result = MagicMock()
|
||||
result.scalar_one_or_none.return_value = None
|
||||
result.one_or_none.return_value = None
|
||||
result.scalars.return_value.all.return_value = []
|
||||
return result
|
||||
|
||||
|
||||
def _svc(execute: object) -> tuple[TaskService, MagicMock]:
|
||||
# Build the session as a local MagicMock and preset `execute` on it before
|
||||
# handing it to TaskService — assigning to `svc.session.execute` directly
|
||||
@@ -155,7 +172,7 @@ async def test_ceo_approve_removes_assignee_worktree_best_effort() -> None:
|
||||
@pytest.mark.asyncio
|
||||
async def test_ceo_approve_skips_worktree_cleanup_for_branchless_task() -> None:
|
||||
task = _build_task(status=TaskStatus.AWAITING_CEO_APPROVAL, branch_name=None)
|
||||
svc, _ = _svc(AsyncMock())
|
||||
svc, _ = _svc(AsyncMock(return_value=_empty_result()))
|
||||
_bind(svc, "get", AsyncMock(return_value=task))
|
||||
_bind(svc, "_validate_and_set_status", MagicMock())
|
||||
_bind(svc, "_close_work_session_for_task", AsyncMock())
|
||||
|
||||
Reference in New Issue
Block a user