mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
fix(notifications): exponential backoff + CAS claim for expired-unacked re-escalation (#652)
The sweep re-escalated every expired unacked ack-required notification on every ~60s tick, forever — the live incident: 3 fresh blocker escalations + Telegram DMs per minute from a static stale pile. Now each notification carries reescalation_count / last_reescalated_at / reescalation_delivered_count (migration 079): first fire at expiry, then doubling intervals from 1h capped at 24h, hard stop after ROBOCO_NOTIFICATION_MAX_REESCALATIONS (default 5) with one permanent log carrying attempts-vs-delivered so 'seen and ignored' is distinguishable from 'route never worked'. The due/wait/capped decision is a pure function in foundation/policy/communications.py. Per adversarial review, the attempt slot is claimed by compare-and-set (UPDATE ... WHERE reescalation_count = :n) BEFORE delivery — the previous draft leaned on the 60s dedup window, which never engages for BLOCKER_ESCALATION (_LOOP_PRONE_TYPES excludes it), so concurrent sweeps would have double-delivered. A lost claim skips delivery outright. Legacy rows read as count=0 and keep today's first-fire semantics. 61 tests incl. a two-session CAS race and a real alembic upgrade/downgrade round trip. Co-authored-by: Renn F <rennf93@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,64 @@
|
|||||||
|
"""Add notifications.reescalation_count/_delivered_count + last_reescalated_at.
|
||||||
|
|
||||||
|
`sweep_expired_notifications` used to re-escalate EVERY ack-required
|
||||||
|
notification past `expires_at` on every ~1min sweep tick, forever — a static
|
||||||
|
pile of stale notifications produced a fresh blocker_escalation row (+
|
||||||
|
Telegram DM) per row per tick. These three columns back a per-notification
|
||||||
|
exponential backoff (first re-escalation at expiry, then doubling from
|
||||||
|
`notification_reescalation_base_seconds`, capped at 24h, hard-stopped past
|
||||||
|
`notification_max_reescalations`): `reescalation_count` is the attempt
|
||||||
|
counter (claimed via compare-and-set even when delivery then fails),
|
||||||
|
`reescalation_delivered_count` is how many of those attempts actually
|
||||||
|
reached a recipient, `last_reescalated_at` anchors the backoff interval.
|
||||||
|
Additive and default-`0`/`NULL`: existing rows read as `count=0`, preserving
|
||||||
|
today's first-fire semantics.
|
||||||
|
|
||||||
|
Revision ID: 079_notification_backoff
|
||||||
|
Revises: 078_project_codegen_command
|
||||||
|
Create Date: 2026-07-22
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "079_notification_backoff"
|
||||||
|
down_revision = "078_project_codegen_command"
|
||||||
|
branch_labels: dict[str, str] | None = None
|
||||||
|
depends_on: dict[str, str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"notifications",
|
||||||
|
sa.Column(
|
||||||
|
"reescalation_count",
|
||||||
|
sa.Integer(),
|
||||||
|
nullable=False,
|
||||||
|
server_default="0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"notifications",
|
||||||
|
sa.Column(
|
||||||
|
"reescalation_delivered_count",
|
||||||
|
sa.Integer(),
|
||||||
|
nullable=False,
|
||||||
|
server_default="0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"notifications",
|
||||||
|
sa.Column(
|
||||||
|
"last_reescalated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("notifications", "last_reescalated_at")
|
||||||
|
op.drop_column("notifications", "reescalation_delivered_count")
|
||||||
|
op.drop_column("notifications", "reescalation_count")
|
||||||
@@ -302,6 +302,27 @@ class Settings(BaseSettings):
|
|||||||
"(legacy: expires_at stays NULL, notifications never expire)."
|
"(legacy: expires_at stays NULL, notifications never expire)."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
notification_reescalation_base_seconds: int = Field(
|
||||||
|
default=3600,
|
||||||
|
ge=1,
|
||||||
|
description=(
|
||||||
|
"Base interval for the per-notification re-escalation backoff: "
|
||||||
|
"the first re-escalation fires at expiry, each one after that "
|
||||||
|
"doubles the wait from this base (1h, 2h, 4h, 8h, ...) capped at "
|
||||||
|
"24h between attempts. Without this a static pile of expired, "
|
||||||
|
"still-unacked notifications re-escalates every sweep tick "
|
||||||
|
"(~1min) forever."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
notification_max_reescalations: int = Field(
|
||||||
|
default=5,
|
||||||
|
ge=1,
|
||||||
|
description=(
|
||||||
|
"Hard cap on re-escalations per notification. Past this many "
|
||||||
|
"attempts a still-unacked notification is logged once as "
|
||||||
|
"permanently-unacked and never re-escalated again."
|
||||||
|
),
|
||||||
|
)
|
||||||
audit_interval_seconds: int = Field(
|
audit_interval_seconds: int = Field(
|
||||||
default=21600,
|
default=21600,
|
||||||
ge=0,
|
ge=0,
|
||||||
|
|||||||
@@ -1041,6 +1041,24 @@ class NotificationTable(Base):
|
|||||||
DateTime(timezone=True), nullable=True
|
DateTime(timezone=True), nullable=True
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Re-escalation backoff (sweep_expired_notifications): how many times this
|
||||||
|
# row has been re-escalated past expiry, and when the last one fired —
|
||||||
|
# drives the exponential schedule so a static pile of stale rows doesn't
|
||||||
|
# re-fire every sweep tick forever. `reescalation_count` is the attempt
|
||||||
|
# counter (bumped by a compare-and-set claim even when delivery then
|
||||||
|
# fails); `reescalation_delivered_count` is how many of those attempts
|
||||||
|
# actually reached a recipient — the two can diverge (a broken escalation
|
||||||
|
# chain burns attempts with zero deliveries).
|
||||||
|
reescalation_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=0, server_default="0", nullable=False
|
||||||
|
)
|
||||||
|
reescalation_delivered_count: Mapped[int] = mapped_column(
|
||||||
|
Integer, default=0, server_default="0", nullable=False
|
||||||
|
)
|
||||||
|
last_reescalated_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
# Timestamps
|
# Timestamps
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), default=lambda: datetime.now(UTC), nullable=False
|
DateTime(timezone=True), default=lambda: datetime.now(UTC), nullable=False
|
||||||
|
|||||||
@@ -6,10 +6,16 @@ Single source of truth for:
|
|||||||
- NotificationType -> requires_ack mapping (replaces 7 hand-set callsites in
|
- NotificationType -> requires_ack mapping (replaces 7 hand-set callsites in
|
||||||
services/notification_delivery.py)
|
services/notification_delivery.py)
|
||||||
- Priority enum (re-exports models.base.NotificationPriority for SQLAlchemy compat)
|
- Priority enum (re-exports models.base.NotificationPriority for SQLAlchemy compat)
|
||||||
|
- Re-escalation backoff schedule (pure; consumed by
|
||||||
|
services/notification_delivery.py's sweep)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
from roboco.foundation.identity import Role
|
from roboco.foundation.identity import Role
|
||||||
from roboco.models.base import NotificationPriority, NotificationType
|
from roboco.models.base import NotificationPriority, NotificationType
|
||||||
|
|
||||||
@@ -94,3 +100,51 @@ ACK_REQUIRED_BY_TYPE: dict[NotificationType, bool] = {
|
|||||||
NotificationType.MENTION: False, # chat @mention, no ack
|
NotificationType.MENTION: False, # chat @mention, no ack
|
||||||
NotificationType.A2A_REQUEST: False, # request/reply lives at message layer
|
NotificationType.A2A_REQUEST: False, # request/reply lives at message layer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Doubling-schedule ceiling: no re-escalation ever waits longer than this
|
||||||
|
# between attempts, however high `notification_max_reescalations` is set.
|
||||||
|
REESCALATION_INTERVAL_CAP_SECONDS = 24 * 3600
|
||||||
|
|
||||||
|
ReescalationDecision = Literal["due", "wait", "capped"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReescalationPolicy:
|
||||||
|
"""The two `settings.notification_*` knobs the backoff schedule reads,
|
||||||
|
bundled so `reescalation_decision` stays under the 5-arg lint ceiling —
|
||||||
|
they always travel together (both come straight from `settings`),
|
||||||
|
unlike the per-row facts (`now`/`expires_at`/`count`/`last_reescalated_at`)."""
|
||||||
|
|
||||||
|
base_seconds: int
|
||||||
|
max_reescalations: int
|
||||||
|
|
||||||
|
|
||||||
|
def reescalation_decision(
|
||||||
|
*,
|
||||||
|
now: datetime,
|
||||||
|
expires_at: datetime,
|
||||||
|
count: int,
|
||||||
|
last_reescalated_at: datetime | None,
|
||||||
|
policy: ReescalationPolicy,
|
||||||
|
) -> ReescalationDecision:
|
||||||
|
"""Pure per-notification re-escalation backoff decision.
|
||||||
|
|
||||||
|
Schedule: the first re-escalation (``count == 0``) is due at
|
||||||
|
``expires_at`` itself. Each one after that doubles the wait from
|
||||||
|
``policy.base_seconds`` (1h, 2h, 4h, 8h, ...) measured from
|
||||||
|
``last_reescalated_at``, capped at ``REESCALATION_INTERVAL_CAP_SECONDS``.
|
||||||
|
Past ``policy.max_reescalations``, always "capped" — the caller must
|
||||||
|
never act on it again. A legacy row with no backoff state reads as
|
||||||
|
``count=0``, preserving the original first-fire-at-expiry behaviour.
|
||||||
|
"""
|
||||||
|
if count >= policy.max_reescalations:
|
||||||
|
return "capped"
|
||||||
|
if count == 0:
|
||||||
|
due_at = expires_at
|
||||||
|
else:
|
||||||
|
interval = min(
|
||||||
|
policy.base_seconds * (2 ** (count - 1)), REESCALATION_INTERVAL_CAP_SECONDS
|
||||||
|
)
|
||||||
|
due_at = (last_reescalated_at or expires_at) + timedelta(seconds=interval)
|
||||||
|
return "due" if now >= due_at else "wait"
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
|
|||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
import structlog
|
import structlog
|
||||||
from sqlalchemy import and_, event, select
|
from sqlalchemy import CursorResult, and_, event, select, update
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from roboco.agents_config import (
|
from roboco.agents_config import (
|
||||||
@@ -27,9 +27,14 @@ from roboco.agents_config import (
|
|||||||
get_pm_for_agent,
|
get_pm_for_agent,
|
||||||
get_pm_for_team,
|
get_pm_for_team,
|
||||||
)
|
)
|
||||||
|
from roboco.config import settings
|
||||||
from roboco.db.tables import AgentTable, NotificationTable, TaskTable
|
from roboco.db.tables import AgentTable, NotificationTable, TaskTable
|
||||||
from roboco.events import Event, EventType, get_event_bus
|
from roboco.events import Event, EventType, get_event_bus
|
||||||
from roboco.foundation.policy.communications import ACK_REQUIRED_BY_TYPE
|
from roboco.foundation.policy.communications import (
|
||||||
|
ACK_REQUIRED_BY_TYPE,
|
||||||
|
ReescalationPolicy,
|
||||||
|
reescalation_decision,
|
||||||
|
)
|
||||||
from roboco.models.base import AgentRole, NotificationPriority, NotificationType
|
from roboco.models.base import AgentRole, NotificationPriority, NotificationType
|
||||||
from roboco.services.base import BaseService, NotFoundError
|
from roboco.services.base import BaseService, NotFoundError
|
||||||
from roboco.services.notification_dedup import (
|
from roboco.services.notification_dedup import (
|
||||||
@@ -338,20 +343,45 @@ class NotificationDeliveryService(BaseService):
|
|||||||
recipient_count=len(n.to_agents or []),
|
recipient_count=len(n.to_agents or []),
|
||||||
ack_count=len(n.acked_by or []),
|
ack_count=len(n.acked_by or []),
|
||||||
expired_at=n.expires_at.isoformat() if n.expires_at else None,
|
expired_at=n.expires_at.isoformat() if n.expires_at else None,
|
||||||
|
reescalation_count=n.reescalation_count,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _log_permanently_unacked(self, n: NotificationTable) -> None:
|
||||||
|
"""One-time terminal log: `n` hit the re-escalation cap and will never
|
||||||
|
be re-escalated again (fires exactly once, on the tick of its last
|
||||||
|
permitted re-escalation). Carries both totals so "seen and ignored"
|
||||||
|
(delivered > 0, recipients just never acked) reads distinctly from
|
||||||
|
"route never worked" (delivered == 0 despite every attempt — a
|
||||||
|
broken escalation chain, e.g. no configured up-role)."""
|
||||||
|
self.log.warning(
|
||||||
|
"Notification permanently unacked — re-escalation cap reached",
|
||||||
|
notification_id=str(n.id),
|
||||||
|
type=n.type.value if n.type else None,
|
||||||
|
priority=n.priority.value if n.priority else None,
|
||||||
|
recipient_count=len(n.to_agents or []),
|
||||||
|
ack_count=len(n.acked_by or []),
|
||||||
|
reescalation_count=n.reescalation_count,
|
||||||
|
reescalation_delivered_count=n.reescalation_delivered_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def sweep_expired_notifications(self) -> int:
|
async def sweep_expired_notifications(self) -> int:
|
||||||
"""Re-escalate then log ack-required notifications past `expires_at`.
|
"""Re-escalate (per a backoff schedule) then log ack-required
|
||||||
|
notifications past `expires_at`.
|
||||||
|
|
||||||
`NotificationTable.expires_at` existed but nothing acted on it. This
|
`NotificationTable.expires_at` existed but nothing acted on it. This
|
||||||
sweep surfaces notifications that have become stale. For an
|
sweep surfaces notifications that have become stale. For an
|
||||||
ack-required row still unacked past the threshold, the recipient's
|
ack-required row still unacked past the threshold, the recipient's
|
||||||
up-role (the PM's PM, or the CEO) is re-notified BEFORE the row is
|
up-role (the PM's PM, or the CEO) is re-notified — but only when
|
||||||
logged as expired — so an inattentive PM can't both miss a blocker
|
`reescalation_decision` says it's due: the first re-escalation fires
|
||||||
and prevent anyone upstream from seeing it. Non-ack-required rows
|
immediately at expiry, each one after that backs off exponentially
|
||||||
and already-acked rows are not re-escalated. We log rather than
|
(`notification_reescalation_base_seconds`, doubling, capped at 24h),
|
||||||
auto-cancel because the notification is the record; rewriting
|
and past `notification_max_reescalations` the row is left alone for
|
||||||
status would be ambiguous. Returns the count of stale unacked items.
|
good. Without the schedule, a static pile of stale rows re-escalated
|
||||||
|
on every ~1min sweep tick forever. Non-ack-required rows and
|
||||||
|
already-acked rows are never re-escalated. We log rather than
|
||||||
|
auto-cancel because the notification is the record; rewriting status
|
||||||
|
would be ambiguous. Returns the count of stale unacked items (not just
|
||||||
|
the ones actually re-escalated this tick).
|
||||||
"""
|
"""
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
result = await self.session.execute(
|
result = await self.session.execute(
|
||||||
@@ -374,36 +404,109 @@ class NotificationDeliveryService(BaseService):
|
|||||||
if n.requires_ack and not self._notification_is_fully_acked(n)
|
if n.requires_ack and not self._notification_is_fully_acked(n)
|
||||||
]
|
]
|
||||||
for n in unacked:
|
for n in unacked:
|
||||||
await self._re_escalate_unacked(n)
|
await self._maybe_reescalate(n, now)
|
||||||
self._log_expired_notification(n)
|
|
||||||
return len(unacked)
|
return len(unacked)
|
||||||
|
|
||||||
async def _re_escalate_unacked(self, n: NotificationTable) -> None:
|
async def _maybe_reescalate(self, n: NotificationTable, now: datetime) -> None:
|
||||||
|
"""Re-escalate `n` only when its backoff schedule says it's due.
|
||||||
|
|
||||||
|
`_persist_and_deliver`'s 60s dedup guard does NOT backstop a
|
||||||
|
concurrent double-sweep here: `BLOCKER_ESCALATION` — the type every
|
||||||
|
re-escalation notification is created as — is not in
|
||||||
|
`_LOOP_PRONE_TYPES` (notification_dedup.py), so that guard returns
|
||||||
|
False unconditionally for it. The real guard is
|
||||||
|
`_claim_reescalation_slot`'s compare-and-set: it must succeed BEFORE
|
||||||
|
any delivery is attempted, so two sweep ticks racing the same row
|
||||||
|
can never both deliver.
|
||||||
|
"""
|
||||||
|
decision = reescalation_decision(
|
||||||
|
now=now,
|
||||||
|
expires_at=cast("datetime", n.expires_at),
|
||||||
|
count=n.reescalation_count,
|
||||||
|
last_reescalated_at=n.last_reescalated_at,
|
||||||
|
policy=ReescalationPolicy(
|
||||||
|
base_seconds=settings.notification_reescalation_base_seconds,
|
||||||
|
max_reescalations=settings.notification_max_reescalations,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if decision != "due":
|
||||||
|
return # "wait": not due yet; "capped": already logged + done
|
||||||
|
if not await self._claim_reescalation_slot(n, now):
|
||||||
|
return # another sweep tick already claimed this attempt
|
||||||
|
delivered = await self._re_escalate_unacked(n)
|
||||||
|
n.reescalation_delivered_count += delivered
|
||||||
|
if n.reescalation_count >= settings.notification_max_reescalations:
|
||||||
|
self._log_permanently_unacked(n)
|
||||||
|
else:
|
||||||
|
self._log_expired_notification(n)
|
||||||
|
|
||||||
|
async def _claim_reescalation_slot(
|
||||||
|
self, n: NotificationTable, now: datetime
|
||||||
|
) -> bool:
|
||||||
|
"""Compare-and-set claim on `n`'s attempt slot, BEFORE any delivery.
|
||||||
|
|
||||||
|
A guarded `UPDATE ... WHERE id = :id AND reescalation_count = :n`:
|
||||||
|
Postgres takes a row lock, so a concurrent claim against the same
|
||||||
|
`reescalation_count` value blocks until this one commits, then loses
|
||||||
|
(0 rows matched — the count already moved). Only the winner proceeds
|
||||||
|
to `_re_escalate_unacked`. The slot is consumed (count bumped) even
|
||||||
|
though delivery hasn't happened yet: a transient delivery failure
|
||||||
|
still burns an attempt, which is what stops a permanently-broken
|
||||||
|
escalation chain from looping forever rather than eventually capping.
|
||||||
|
"""
|
||||||
|
result = await self.session.execute(
|
||||||
|
update(NotificationTable)
|
||||||
|
.where(
|
||||||
|
NotificationTable.id == n.id,
|
||||||
|
NotificationTable.reescalation_count == n.reescalation_count,
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
reescalation_count=n.reescalation_count + 1, last_reescalated_at=now
|
||||||
|
)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
# UPDATE always yields a CursorResult (has `.rowcount`); `execute`'s
|
||||||
|
# declared return type is the generic `Result` supertype, so peel it.
|
||||||
|
claimed = cast("CursorResult[Any]", result).rowcount == 1
|
||||||
|
if claimed:
|
||||||
|
# Mirror the winning UPDATE onto the in-memory object so the rest
|
||||||
|
# of this tick (and the eventual `reescalation_delivered_count`
|
||||||
|
# flush) sees consistent state. SQLAlchemy will re-flush these
|
||||||
|
# same two columns with the caller's next commit — a harmless
|
||||||
|
# no-op re-write of the value we just committed, not a bug.
|
||||||
|
n.reescalation_count += 1
|
||||||
|
n.last_reescalated_at = now
|
||||||
|
return claimed
|
||||||
|
|
||||||
|
async def _re_escalate_unacked(self, n: NotificationTable) -> int:
|
||||||
"""Re-send an unacked ack-required notification to each non-acking
|
"""Re-send an unacked ack-required notification to each non-acking
|
||||||
recipient's up-role before expiry. Best-effort: a missing chain,
|
recipient's up-role. Best-effort: a missing chain or target is
|
||||||
target, or a dedup-suppressed re-fire is logged-and-skipped, never
|
logged-and-skipped, never raises. Returns how many recipients were
|
||||||
raises — the expiry log still fires. The loop-prone dedup guard in
|
actually re-notified — used to distinguish a broken escalation chain
|
||||||
`_persist_and_deliver` caps repeat re-escalations within the 60s
|
(0 delivered despite an attempt) from one that works but is ignored."""
|
||||||
window so a tight sweep loop can't flood the upstream role."""
|
|
||||||
acked = {str(a) for a in (n.acked_by or [])}
|
acked = {str(a) for a in (n.acked_by or [])}
|
||||||
|
delivered = 0
|
||||||
for recipient_id in n.to_agents or []:
|
for recipient_id in n.to_agents or []:
|
||||||
if str(recipient_id) in acked:
|
if str(recipient_id) in acked:
|
||||||
continue
|
continue
|
||||||
await self._re_escalate_recipient(n, cast("UUID", recipient_id))
|
if await self._re_escalate_recipient(n, cast("UUID", recipient_id)):
|
||||||
|
delivered += 1
|
||||||
|
return delivered
|
||||||
|
|
||||||
async def _re_escalate_recipient(
|
async def _re_escalate_recipient(
|
||||||
self, n: NotificationTable, recipient_id: UUID
|
self, n: NotificationTable, recipient_id: UUID
|
||||||
) -> None:
|
) -> bool:
|
||||||
"""Resolve one recipient's up-role and re-fire the escalation."""
|
"""Resolve one recipient's up-role and re-fire the escalation.
|
||||||
|
Returns True iff it was actually persisted+delivered."""
|
||||||
recipient = await self._get_agent_by_id(recipient_id)
|
recipient = await self._get_agent_by_id(recipient_id)
|
||||||
if not recipient or not recipient.slug:
|
if not recipient or not recipient.slug:
|
||||||
return
|
return False
|
||||||
target_slug = get_escalation_target(recipient.slug)
|
target_slug = get_escalation_target(recipient.slug)
|
||||||
if not target_slug:
|
if not target_slug:
|
||||||
return
|
return False
|
||||||
target = await self._get_agent_by_slug(target_slug)
|
target = await self._get_agent_by_slug(target_slug)
|
||||||
if not target:
|
if not target:
|
||||||
return
|
return False
|
||||||
notification = NotificationTable(
|
notification = NotificationTable(
|
||||||
type=NotificationType.BLOCKER_ESCALATION,
|
type=NotificationType.BLOCKER_ESCALATION,
|
||||||
priority=NotificationPriority.HIGH,
|
priority=NotificationPriority.HIGH,
|
||||||
@@ -422,7 +525,7 @@ class NotificationDeliveryService(BaseService):
|
|||||||
acked_by=[],
|
acked_by=[],
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
await self._persist_and_deliver(notification)
|
return await self._persist_and_deliver(notification)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log.warning(
|
self.log.warning(
|
||||||
"Re-escalation deliver failed",
|
"Re-escalation deliver failed",
|
||||||
@@ -430,6 +533,7 @@ class NotificationDeliveryService(BaseService):
|
|||||||
target_slug=target_slug,
|
target_slug=target_slug,
|
||||||
error=str(e),
|
error=str(e),
|
||||||
)
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
async def get_pending_for_agent(
|
async def get_pending_for_agent(
|
||||||
self,
|
self,
|
||||||
@@ -1219,8 +1323,18 @@ class NotificationDeliveryService(BaseService):
|
|||||||
"""Find the auditor agent (org-wide; earliest-created if many)."""
|
"""Find the auditor agent (org-wide; earliest-created if many)."""
|
||||||
return await get_agent_by_role(self.session, AgentRole.AUDITOR)
|
return await get_agent_by_role(self.session, AgentRole.AUDITOR)
|
||||||
|
|
||||||
async def _persist_and_deliver(self, notification: NotificationTable) -> None:
|
async def _persist_and_deliver(self, notification: NotificationTable) -> bool:
|
||||||
"""Add to session, flush (to get an id), deliver. Caller commits."""
|
"""Add to session, flush (to get an id), deliver. Caller commits.
|
||||||
|
|
||||||
|
Returns True iff actually persisted+delivered, False if suppressed by
|
||||||
|
the 60s dedup guard below. That guard only ever applies to
|
||||||
|
`_LOOP_PRONE_TYPES` (notification_dedup.py) — BLOCKER_ESCALATION,
|
||||||
|
the type re-escalations use, is NOT one of them, so for that path
|
||||||
|
this always returns True or raises; the real double-delivery guard
|
||||||
|
for re-escalations is the CAS claim in
|
||||||
|
`NotificationDeliveryService._claim_reescalation_slot`, upstream of
|
||||||
|
this call.
|
||||||
|
"""
|
||||||
# Re-fire guard (loop-prone types): this path skips the DB dedup, so
|
# Re-fire guard (loop-prone types): this path skips the DB dedup, so
|
||||||
# apply the same 60s Redis SET-NX window. Fail-open on Redis down.
|
# apply the same 60s Redis SET-NX window. Fail-open on Redis down.
|
||||||
# Casts peel the SA UUID column type-leak for the type checker.
|
# Casts peel the SA UUID column type-leak for the type checker.
|
||||||
@@ -1241,10 +1355,11 @@ class NotificationDeliveryService(BaseService):
|
|||||||
if notification.related_task_id is not None
|
if notification.related_task_id is not None
|
||||||
else None,
|
else None,
|
||||||
)
|
)
|
||||||
return
|
return False
|
||||||
self.session.add(notification)
|
self.session.add(notification)
|
||||||
await self.session.flush()
|
await self.session.flush()
|
||||||
await self.deliver(require_uuid(notification.id))
|
await self.deliver(require_uuid(notification.id))
|
||||||
|
return True
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# API-FACING LIST + CRUD (consumed by api/routes/notifications.py)
|
# API-FACING LIST + CRUD (consumed by api/routes/notifications.py)
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Re-escalation backoff columns on notifications (migration 079).
|
||||||
|
|
||||||
|
Migration 079 adds ``notifications.reescalation_count`` /
|
||||||
|
``.reescalation_delivered_count`` (integer, not null, default 0) and
|
||||||
|
``.last_reescalated_at`` (timestamptz, null). The real upgrade/downgrade
|
||||||
|
chain is verified separately against a throwaway Postgres; these assertions
|
||||||
|
guard the resulting schema shape and a value round-trip.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from roboco.db.tables import AgentTable, NotificationTable
|
||||||
|
from roboco.models import AgentRole, AgentStatus, NotificationPriority, NotificationType
|
||||||
|
from roboco.models.base import Team
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_notification(db_session: AsyncSession) -> NotificationTable:
|
||||||
|
sender = AgentTable(
|
||||||
|
id=uuid4(),
|
||||||
|
name="Dev",
|
||||||
|
slug=f"be-dev-{uuid4().hex[:8]}",
|
||||||
|
role=AgentRole.DEVELOPER,
|
||||||
|
team=Team.BACKEND,
|
||||||
|
status=AgentStatus.ACTIVE,
|
||||||
|
model_config={},
|
||||||
|
system_prompt="dev",
|
||||||
|
capabilities=[],
|
||||||
|
permissions={},
|
||||||
|
metrics={},
|
||||||
|
)
|
||||||
|
db_session.add(sender)
|
||||||
|
await db_session.flush()
|
||||||
|
notification = NotificationTable(
|
||||||
|
type=NotificationType.BLOCKER_ESCALATION,
|
||||||
|
priority=NotificationPriority.HIGH,
|
||||||
|
from_agent=sender.id,
|
||||||
|
to_agents=[sender.id],
|
||||||
|
subject="stale",
|
||||||
|
body="body",
|
||||||
|
requires_ack=True,
|
||||||
|
)
|
||||||
|
db_session.add(notification)
|
||||||
|
await db_session.flush()
|
||||||
|
return notification
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reescalation_backoff_columns_default(db_session: AsyncSession) -> None:
|
||||||
|
notification = await _seed_notification(db_session)
|
||||||
|
assert notification.reescalation_count == 0
|
||||||
|
assert notification.reescalation_delivered_count == 0
|
||||||
|
assert notification.last_reescalated_at is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reescalation_backoff_columns_round_trip(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
) -> None:
|
||||||
|
notification = await _seed_notification(db_session)
|
||||||
|
stamped_at = datetime.now(UTC)
|
||||||
|
attempts, delivered = 3, 2
|
||||||
|
notification.reescalation_count = attempts
|
||||||
|
notification.reescalation_delivered_count = delivered
|
||||||
|
notification.last_reescalated_at = stamped_at
|
||||||
|
await db_session.flush()
|
||||||
|
|
||||||
|
row = (
|
||||||
|
await db_session.execute(
|
||||||
|
select(NotificationTable).where(NotificationTable.id == notification.id)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
assert row.reescalation_count == attempts
|
||||||
|
assert row.reescalation_delivered_count == delivered
|
||||||
|
assert row.last_reescalated_at == stamped_at
|
||||||
@@ -5,6 +5,17 @@ unacked must be re-escalated to the recipient's up-role (the PM's PM or
|
|||||||
the CEO) BEFORE the sweep logs/expiring it — not just logged-and-dropped.
|
the CEO) BEFORE the sweep logs/expiring it — not just logged-and-dropped.
|
||||||
Combined with H12 (Task 6), an inattentive PM can't both miss a blocker
|
Combined with H12 (Task 6), an inattentive PM can't both miss a blocker
|
||||||
and prevent main-pm from seeing it.
|
and prevent main-pm from seeing it.
|
||||||
|
|
||||||
|
Re-escalation backoff: a static pile of stale notifications used to
|
||||||
|
re-escalate on *every* sweep tick (~1min) forever. `reescalation_decision`
|
||||||
|
(pure, in `foundation/policy/communications.py`) gates each tick behind a
|
||||||
|
per-notification exponential schedule + a hard retry cap.
|
||||||
|
|
||||||
|
Double-delivery race: `_persist_and_deliver`'s 60s dedup guard is a no-op for
|
||||||
|
`BLOCKER_ESCALATION` (not in `_LOOP_PRONE_TYPES`), so it can't backstop two
|
||||||
|
concurrent sweep ticks racing the same stale row — a compare-and-set claim
|
||||||
|
(`_claim_reescalation_slot`) is the real guard, exercised below by racing two
|
||||||
|
service instances against the same row.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -15,8 +26,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||||||
from uuid import UUID, uuid4
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from roboco.config import settings
|
||||||
|
from roboco.foundation.policy.communications import (
|
||||||
|
ReescalationPolicy,
|
||||||
|
reescalation_decision,
|
||||||
|
)
|
||||||
from roboco.models import NotificationPriority, NotificationType
|
from roboco.models import NotificationPriority, NotificationType
|
||||||
from roboco.services.notification_delivery import NotificationDeliveryService
|
from roboco.services.notification_delivery import NotificationDeliveryService
|
||||||
|
from sqlalchemy import Update
|
||||||
|
|
||||||
|
|
||||||
def _stale_notification(
|
def _stale_notification(
|
||||||
@@ -24,7 +41,8 @@ def _stale_notification(
|
|||||||
requires_ack: bool = True,
|
requires_ack: bool = True,
|
||||||
acked: bool = False,
|
acked: bool = False,
|
||||||
recipient_id: UUID | None = None,
|
recipient_id: UUID | None = None,
|
||||||
from_agent_id: UUID | None = None,
|
reescalation_count: int = 0,
|
||||||
|
last_reescalated_at: datetime | None = None,
|
||||||
) -> MagicMock:
|
) -> MagicMock:
|
||||||
n = MagicMock()
|
n = MagicMock()
|
||||||
n.id = uuid4()
|
n.id = uuid4()
|
||||||
@@ -39,8 +57,11 @@ def _stale_notification(
|
|||||||
n.acked_by = [rid] if acked else []
|
n.acked_by = [rid] if acked else []
|
||||||
n.read_by = []
|
n.read_by = []
|
||||||
n.requires_ack = requires_ack
|
n.requires_ack = requires_ack
|
||||||
n.from_agent = from_agent_id or uuid4()
|
n.from_agent = uuid4()
|
||||||
n.related_task_id = uuid4()
|
n.related_task_id = uuid4()
|
||||||
|
n.reescalation_count = reescalation_count
|
||||||
|
n.reescalation_delivered_count = 0 # no test needs a nonzero starting value
|
||||||
|
n.last_reescalated_at = last_reescalated_at
|
||||||
return n
|
return n
|
||||||
|
|
||||||
|
|
||||||
@@ -74,14 +95,37 @@ def _svc_with_agents(
|
|||||||
return svc
|
return svc
|
||||||
|
|
||||||
|
|
||||||
def _session_returning(notifications: list[MagicMock]) -> MagicMock:
|
def _assign_id_on_add(obj: Any) -> None:
|
||||||
"""A session whose `execute(...).scalars().all()` returns `notifications`."""
|
"""`session.add` side effect: a real flush assigns the SQLAlchemy-default
|
||||||
|
id; this mock has no engine to do that, so stand in for it here — without
|
||||||
|
it `require_uuid(notification.id)` in `_persist_and_deliver` always raises
|
||||||
|
on the freshly-built re-escalation row, making every "delivered" outcome
|
||||||
|
in this suite look like a failure."""
|
||||||
|
if getattr(obj, "id", None) is None:
|
||||||
|
obj.id = uuid4()
|
||||||
|
|
||||||
|
|
||||||
|
def _session_returning(
|
||||||
|
notifications: list[MagicMock], *, claim_succeeds: bool = True
|
||||||
|
) -> MagicMock:
|
||||||
|
"""A session whose SELECT (the sweep's stale-notifications query) returns
|
||||||
|
`notifications`; every re-escalation CAS UPDATE (`_claim_reescalation_slot`)
|
||||||
|
reports 1 row affected — the claim wins — unless `claim_succeeds` is False,
|
||||||
|
simulating a concurrent sweep tick that already claimed this row's slot."""
|
||||||
session = MagicMock()
|
session = MagicMock()
|
||||||
session.add = MagicMock()
|
session.add = MagicMock(side_effect=_assign_id_on_add)
|
||||||
session.flush = AsyncMock()
|
session.flush = AsyncMock()
|
||||||
result = MagicMock()
|
|
||||||
result.scalars.return_value.all.return_value = notifications
|
select_result = MagicMock()
|
||||||
session.execute = AsyncMock(return_value=result)
|
select_result.scalars.return_value.all.return_value = notifications
|
||||||
|
|
||||||
|
update_result = MagicMock()
|
||||||
|
update_result.rowcount = 1 if claim_succeeds else 0
|
||||||
|
|
||||||
|
async def _execute(statement: Any, *_args: Any, **_kwargs: Any) -> MagicMock:
|
||||||
|
return update_result if isinstance(statement, Update) else select_result
|
||||||
|
|
||||||
|
session.execute = AsyncMock(side_effect=_execute)
|
||||||
return session
|
return session
|
||||||
|
|
||||||
|
|
||||||
@@ -119,6 +163,7 @@ async def test_sweep_re_escalates_stale_unacked_ack_required() -> None:
|
|||||||
assert re_escalated.type == NotificationType.BLOCKER_ESCALATION
|
assert re_escalated.type == NotificationType.BLOCKER_ESCALATION
|
||||||
assert re_escalated.requires_ack is True
|
assert re_escalated.requires_ack is True
|
||||||
assert "Re-escalation" in re_escalated.subject
|
assert "Re-escalation" in re_escalated.subject
|
||||||
|
assert notif.reescalation_delivered_count == 1 # the attempt was delivered
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -182,7 +227,11 @@ async def test_sweep_does_not_re_escalate_non_ack_required() -> None:
|
|||||||
async def test_sweep_skips_re_escalation_when_no_chain_target() -> None:
|
async def test_sweep_skips_re_escalation_when_no_chain_target() -> None:
|
||||||
"""Recipient with no configured escalation target → no re-escalation, but
|
"""Recipient with no configured escalation target → no re-escalation, but
|
||||||
the stale unacked count still surfaces (best-effort: missing chain is
|
the stale unacked count still surfaces (best-effort: missing chain is
|
||||||
logged-and-skipped, never raises)."""
|
logged-and-skipped, never raises). The attempt slot is still consumed
|
||||||
|
(reescalation_count bumps) even though nothing was delivered — a broken
|
||||||
|
chain burns attempts rather than looping forever; delivered stays 0,
|
||||||
|
which is exactly the "route never worked" signal `_log_permanently_unacked`
|
||||||
|
now carries."""
|
||||||
recipient = _agent("ghost-role")
|
recipient = _agent("ghost-role")
|
||||||
notif = _stale_notification(
|
notif = _stale_notification(
|
||||||
requires_ack=True, acked=False, recipient_id=recipient.id
|
requires_ack=True, acked=False, recipient_id=recipient.id
|
||||||
@@ -205,12 +254,164 @@ async def test_sweep_skips_re_escalation_when_no_chain_target() -> None:
|
|||||||
|
|
||||||
assert count == 1 # still stale + unacked
|
assert count == 1 # still stale + unacked
|
||||||
session.add.assert_not_called()
|
session.add.assert_not_called()
|
||||||
|
assert notif.reescalation_count == 1 # attempt slot consumed regardless
|
||||||
|
assert notif.reescalation_delivered_count == 0 # ...but nothing delivered
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_sweep_dedup_suppresses_repeat_re_escalation() -> None:
|
async def test_sweep_cas_claim_prevents_double_delivery_race() -> None:
|
||||||
"""A repeat sweep within the dedup window does not re-fire the same
|
"""Two service instances (simulating two concurrent sweep ticks) race the
|
||||||
re-escalation (loop-prone guard in `_persist_and_deliver`)."""
|
same stale row. `_persist_and_deliver`'s 60s dedup guard cannot arbitrate
|
||||||
|
this — BLOCKER_ESCALATION isn't a `_LOOP_PRONE_TYPES` member, so it's a
|
||||||
|
no-op for this path. The CAS claim in `_claim_reescalation_slot` is what
|
||||||
|
actually decides it: exactly one instance wins the guarded UPDATE and
|
||||||
|
delivers; the loser (0 rows updated) skips delivery entirely, without
|
||||||
|
raising."""
|
||||||
|
recipient = _agent("be-pm")
|
||||||
|
target = _agent("main-pm")
|
||||||
|
notif = _stale_notification(
|
||||||
|
requires_ack=True, acked=False, recipient_id=recipient.id
|
||||||
|
)
|
||||||
|
|
||||||
|
winner_session = _session_returning([notif], claim_succeeds=True)
|
||||||
|
loser_session = _session_returning([notif], claim_succeeds=False)
|
||||||
|
winner = _svc_with_agents(
|
||||||
|
winner_session, recipient=recipient, escalation_target=target
|
||||||
|
)
|
||||||
|
loser = _svc_with_agents(
|
||||||
|
loser_session, recipient=recipient, escalation_target=target
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"roboco.services.notification_delivery.all_recipients_recently_notified",
|
||||||
|
AsyncMock(return_value=False),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"roboco.services.notification_delivery.get_escalation_target",
|
||||||
|
return_value="main-pm",
|
||||||
|
),
|
||||||
|
):
|
||||||
|
winner_count = await winner.sweep_expired_notifications()
|
||||||
|
loser_count = await loser.sweep_expired_notifications()
|
||||||
|
|
||||||
|
assert winner_count == 1
|
||||||
|
assert loser_count == 1 # still stale + unacked from the loser's own view
|
||||||
|
assert winner_session.add.call_count == 1 # won the claim, delivered
|
||||||
|
loser_session.add.assert_not_called() # lost the claim, never touched delivery
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# reescalation_decision — pure schedule math
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
_DEFAULT_POLICY = ReescalationPolicy(base_seconds=3600, max_reescalations=5)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reescalation_decision_first_fire_due_at_expiry() -> None:
|
||||||
|
"""count=0 (including a legacy row with no backoff state) is due the
|
||||||
|
instant `now` reaches `expires_at` — preserves the original semantics."""
|
||||||
|
expires_at = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
assert (
|
||||||
|
reescalation_decision(
|
||||||
|
now=expires_at,
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=0,
|
||||||
|
last_reescalated_at=None,
|
||||||
|
policy=_DEFAULT_POLICY,
|
||||||
|
)
|
||||||
|
== "due"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reescalation_decision_first_fire_not_due_before_expiry() -> None:
|
||||||
|
expires_at = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
assert (
|
||||||
|
reescalation_decision(
|
||||||
|
now=expires_at - timedelta(seconds=1),
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=0,
|
||||||
|
last_reescalated_at=None,
|
||||||
|
policy=_DEFAULT_POLICY,
|
||||||
|
)
|
||||||
|
== "wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reescalation_decision_backoff_doubles() -> None:
|
||||||
|
"""count=2 waits 2*base (2h at the default base) from the last fire."""
|
||||||
|
last = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
expires_at = last - timedelta(hours=3)
|
||||||
|
not_yet = reescalation_decision(
|
||||||
|
now=last + timedelta(hours=2) - timedelta(seconds=1),
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=2,
|
||||||
|
last_reescalated_at=last,
|
||||||
|
policy=_DEFAULT_POLICY,
|
||||||
|
)
|
||||||
|
due = reescalation_decision(
|
||||||
|
now=last + timedelta(hours=2),
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=2,
|
||||||
|
last_reescalated_at=last,
|
||||||
|
policy=_DEFAULT_POLICY,
|
||||||
|
)
|
||||||
|
assert not_yet == "wait"
|
||||||
|
assert due == "due"
|
||||||
|
|
||||||
|
|
||||||
|
def test_reescalation_decision_interval_capped_at_24h() -> None:
|
||||||
|
"""However high `count` climbs (a raised max_reescalations), the wait
|
||||||
|
between attempts never exceeds 24h."""
|
||||||
|
last = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
expires_at = last - timedelta(days=1)
|
||||||
|
policy = ReescalationPolicy(base_seconds=3600, max_reescalations=20)
|
||||||
|
# Uncapped this would be base*2**8 = 256h; capped it's 24h.
|
||||||
|
not_yet = reescalation_decision(
|
||||||
|
now=last + timedelta(hours=24) - timedelta(seconds=1),
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=9,
|
||||||
|
last_reescalated_at=last,
|
||||||
|
policy=policy,
|
||||||
|
)
|
||||||
|
due = reescalation_decision(
|
||||||
|
now=last + timedelta(hours=24),
|
||||||
|
expires_at=expires_at,
|
||||||
|
count=9,
|
||||||
|
last_reescalated_at=last,
|
||||||
|
policy=policy,
|
||||||
|
)
|
||||||
|
assert not_yet == "wait"
|
||||||
|
assert due == "due"
|
||||||
|
|
||||||
|
|
||||||
|
def test_reescalation_decision_capped_past_max_regardless_of_timing() -> None:
|
||||||
|
"""count >= max_reescalations is always "capped", even if the schedule
|
||||||
|
math would otherwise say a re-escalation is overdue."""
|
||||||
|
last = datetime(2026, 1, 1, tzinfo=UTC)
|
||||||
|
assert (
|
||||||
|
reescalation_decision(
|
||||||
|
now=last + timedelta(days=365),
|
||||||
|
expires_at=last - timedelta(hours=1),
|
||||||
|
count=5,
|
||||||
|
last_reescalated_at=last,
|
||||||
|
policy=_DEFAULT_POLICY,
|
||||||
|
)
|
||||||
|
== "capped"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# sweep_expired_notifications — backoff integration
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sweep_backoff_does_not_refire_within_the_interval() -> None:
|
||||||
|
"""A row re-escalates once, then a same-tick-ish second sweep (interval
|
||||||
|
not elapsed) does not re-escalate again — and its schedule state (count,
|
||||||
|
last_reescalated_at) is stamped on the notification after the first."""
|
||||||
recipient = _agent("be-pm")
|
recipient = _agent("be-pm")
|
||||||
target = _agent("main-pm")
|
target = _agent("main-pm")
|
||||||
notif = _stale_notification(
|
notif = _stale_notification(
|
||||||
@@ -223,7 +424,47 @@ async def test_sweep_dedup_suppresses_repeat_re_escalation() -> None:
|
|||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"roboco.services.notification_delivery.all_recipients_recently_notified",
|
"roboco.services.notification_delivery.all_recipients_recently_notified",
|
||||||
AsyncMock(return_value=True),
|
AsyncMock(return_value=False),
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"roboco.services.notification_delivery.get_escalation_target",
|
||||||
|
return_value="main-pm",
|
||||||
|
),
|
||||||
|
):
|
||||||
|
first = await svc.sweep_expired_notifications()
|
||||||
|
assert notif.reescalation_count == 1
|
||||||
|
assert notif.last_reescalated_at is not None
|
||||||
|
assert session.add.call_count == 1
|
||||||
|
|
||||||
|
second = await svc.sweep_expired_notifications()
|
||||||
|
|
||||||
|
assert first == 1
|
||||||
|
assert second == 1 # still stale + unacked
|
||||||
|
assert session.add.call_count == 1 # no second re-escalation this soon
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_sweep_capped_row_never_re_escalates_again() -> None:
|
||||||
|
"""A row already at the retry cap is skipped forever — no re-escalation,
|
||||||
|
no repeat 'permanently unacked' log — but still counts as stale+unacked."""
|
||||||
|
recipient = _agent("be-pm")
|
||||||
|
target = _agent("main-pm")
|
||||||
|
capped_count = settings.notification_max_reescalations
|
||||||
|
notif = _stale_notification(
|
||||||
|
requires_ack=True,
|
||||||
|
acked=False,
|
||||||
|
recipient_id=recipient.id,
|
||||||
|
reescalation_count=capped_count,
|
||||||
|
last_reescalated_at=datetime.now(UTC) - timedelta(days=1),
|
||||||
|
)
|
||||||
|
|
||||||
|
session = _session_returning([notif])
|
||||||
|
svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"roboco.services.notification_delivery.all_recipients_recently_notified",
|
||||||
|
AsyncMock(return_value=False),
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"roboco.services.notification_delivery.get_escalation_target",
|
"roboco.services.notification_delivery.get_escalation_target",
|
||||||
@@ -232,5 +473,6 @@ async def test_sweep_dedup_suppresses_repeat_re_escalation() -> None:
|
|||||||
):
|
):
|
||||||
count = await svc.sweep_expired_notifications()
|
count = await svc.sweep_expired_notifications()
|
||||||
|
|
||||||
assert count == 1 # stale + unacked still reported
|
assert count == 1 # still stale + unacked
|
||||||
session.add.assert_not_called() # dedup suppressed the re-escalation row
|
session.add.assert_not_called()
|
||||||
|
assert notif.reescalation_count == capped_count # untouched — no further attempts
|
||||||
|
|||||||
Reference in New Issue
Block a user