roboco/tests/unit/services/test_notification_delivery.py

"""NotificationDeliveryService.sweep_expired_notifications re-escalation.

L26: an ack-required notification past its `expires_at` that is still
unacked must be re-escalated to the recipient's up-role (the PM's PM or
the CEO) BEFORE the sweep logs/expiring it — not just logged-and-dropped.
Combined with H12 (Task 6), an inattentive PM can't both miss a blocker
and prevent main-pm from seeing it.

Re-escalation backoff: a static pile of stale notifications used to
re-escalate on *every* sweep tick (~1min) forever. `reescalation_decision`
(pure, in `foundation/policy/communications.py`) gates each tick behind a
per-notification exponential schedule + a hard retry cap.

Double-delivery race: `_persist_and_deliver`'s 60s dedup guard is a no-op for
`BLOCKER_ESCALATION` (not in `_LOOP_PRONE_TYPES`), so it can't backstop two
concurrent sweep ticks racing the same stale row — a compare-and-set claim
(`_claim_reescalation_slot`) is the real guard, exercised below by racing two
service instances against the same row.
"""

from __future__ import annotations

from datetime import UTC, datetime, timedelta
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import UUID, uuid4

import pytest
from roboco.config import settings
from roboco.foundation.policy.communications import (
    ReescalationPolicy,
    reescalation_decision,
)
from roboco.models import NotificationPriority, NotificationType
from roboco.services.notification_delivery import NotificationDeliveryService
from sqlalchemy import Update


def _stale_notification(
    *,
    requires_ack: bool = True,
    acked: bool = False,
    recipient_id: UUID | None = None,
    reescalation_count: int = 0,
    last_reescalated_at: datetime | None = None,
) -> MagicMock:
    n = MagicMock()
    n.id = uuid4()
    n.type = NotificationType.BLOCKER_ESCALATION
    n.priority = NotificationPriority.HIGH
    n.subject = "Blocked: task X"
    n.body = "body"
    n.expires_at = datetime.now(UTC) - timedelta(minutes=5)
    n.timestamp = datetime.now(UTC) - timedelta(minutes=30)
    rid = recipient_id or uuid4()
    n.to_agents = [rid]
    n.acked_by = [rid] if acked else []
    n.read_by = []
    n.requires_ack = requires_ack
    n.from_agent = uuid4()
    n.related_task_id = uuid4()
    n.reescalation_count = reescalation_count
    n.reescalation_delivered_count = 0  # no test needs a nonzero starting value
    n.last_reescalated_at = last_reescalated_at
    return n


def _agent(slug: str, agent_id: UUID | None = None) -> MagicMock:
    a = MagicMock()
    a.id = agent_id or uuid4()
    a.slug = slug
    return a


def _svc_with_agents(
    session: MagicMock,
    *,
    recipient: MagicMock,
    escalation_target: MagicMock | None,
) -> Any:
    """Build a service with agent lookups stubbed.

    `_get_agent_by_id` resolves the unacked recipient; `_get_agent_by_slug`
    resolves the escalation target. `deliver` is stubbed so the re-escalation
    row flush + deliver path runs without a real DB.
    """
    svc = NotificationDeliveryService(session)
    cc: Any = svc
    cc._get_agent_by_id = AsyncMock(return_value=recipient)
    if escalation_target is not None:
        cc._get_agent_by_slug = AsyncMock(return_value=escalation_target)
    else:
        cc._get_agent_by_slug = AsyncMock(return_value=None)
    cc.deliver = AsyncMock(return_value=True)
    return svc


def _assign_id_on_add(obj: Any) -> None:
    """`session.add` side effect: a real flush assigns the SQLAlchemy-default
    id; this mock has no engine to do that, so stand in for it here — without
    it `require_uuid(notification.id)` in `_persist_and_deliver` always raises
    on the freshly-built re-escalation row, making every "delivered" outcome
    in this suite look like a failure."""
    if getattr(obj, "id", None) is None:
        obj.id = uuid4()


def _session_returning(
    notifications: list[MagicMock], *, claim_succeeds: bool = True
) -> MagicMock:
    """A session whose SELECT (the sweep's stale-notifications query) returns
    `notifications`; every re-escalation CAS UPDATE (`_claim_reescalation_slot`)
    reports 1 row affected — the claim wins — unless `claim_succeeds` is False,
    simulating a concurrent sweep tick that already claimed this row's slot.

    `commit`/`rollback` are awaitable no-ops (the per-row commit scope in
    `sweep_expired_notifications`/`_maybe_reescalate` awaits both); `get`
    is an awaitable lookup against `notifications` by id (the sweep loop
    re-fetches each row post-snapshot via `session.get` instead of
    iterating the ORM instances directly)."""
    session = MagicMock()
    session.add = MagicMock(side_effect=_assign_id_on_add)
    session.flush = AsyncMock()
    session.commit = AsyncMock()
    session.rollback = AsyncMock()

    async def _get(_model: Any, ident: Any) -> MagicMock | None:
        return next((n for n in notifications if n.id == ident), None)

    session.get = AsyncMock(side_effect=_get)

    select_result = MagicMock()
    select_result.scalars.return_value.all.return_value = notifications

    update_result = MagicMock()
    update_result.rowcount = 1 if claim_succeeds else 0

    async def _execute(statement: Any, *_args: Any, **_kwargs: Any) -> MagicMock:
        return update_result if isinstance(statement, Update) else select_result

    session.execute = AsyncMock(side_effect=_execute)
    return session


@pytest.mark.asyncio
async def test_sweep_re_escalates_stale_unacked_ack_required() -> None:
    """Ack-required + past threshold + unacked → a re-escalation notification
    is persisted to the recipient's escalation target before the expiry log."""
    recipient = _agent("be-pm")
    target = _agent("main-pm")
    notif = _stale_notification(
        requires_ack=True, acked=False, recipient_id=recipient.id
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="main-pm",
        ),
    ):
        count = await svc.sweep_expired_notifications()

    assert count == 1
    # A re-escalation notification was added, addressed to the escalation target.
    added = [c for c in session.add.call_args_list if c.args]
    assert added, "expected a re-escalation row to be added to the session"
    re_escalated = added[0].args[0]
    assert re_escalated.to_agents == [target.id]
    assert re_escalated.type == NotificationType.BLOCKER_ESCALATION
    assert re_escalated.requires_ack is True
    assert "Re-escalation" in re_escalated.subject
    assert notif.reescalation_delivered_count == 1  # the attempt was delivered


@pytest.mark.asyncio
async def test_sweep_does_not_re_escalate_already_acked() -> None:
    """Ack-required + past threshold + fully acked → no re-escalation, count 0."""
    recipient = _agent("be-pm")
    target = _agent("main-pm")
    notif = _stale_notification(
        requires_ack=True, acked=True, recipient_id=recipient.id
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="main-pm",
        ),
    ):
        count = await svc.sweep_expired_notifications()

    assert count == 0
    session.add.assert_not_called()


@pytest.mark.asyncio
async def test_sweep_does_not_re_escalate_non_ack_required() -> None:
    """Non-ack-required + past threshold → no re-escalation (preserve existing
    behaviour: only ack-required-still-unacked rows re-escalate)."""
    recipient = _agent("be-dev-1")
    target = _agent("be-pm")
    notif = _stale_notification(
        requires_ack=False, acked=False, recipient_id=recipient.id
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="be-pm",
        ),
    ):
        count = await svc.sweep_expired_notifications()

    assert count == 0
    session.add.assert_not_called()


@pytest.mark.asyncio
async def test_sweep_skips_re_escalation_when_no_chain_target() -> None:
    """Recipient with no configured escalation target → no re-escalation, but
    the stale unacked count still surfaces (best-effort: missing chain is
    logged-and-skipped, never raises). The attempt slot is still consumed
    (reescalation_count bumps) even though nothing was delivered — a broken
    chain burns attempts rather than looping forever; delivered stays 0,
    which is exactly the "route never worked" signal `_log_permanently_unacked`
    now carries."""
    recipient = _agent("ghost-role")
    notif = _stale_notification(
        requires_ack=True, acked=False, recipient_id=recipient.id
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=None)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value=None,
        ),
    ):
        count = await svc.sweep_expired_notifications()

    assert count == 1  # still stale + unacked
    session.add.assert_not_called()
    assert notif.reescalation_count == 1  # attempt slot consumed regardless
    assert notif.reescalation_delivered_count == 0  # ...but nothing delivered


@pytest.mark.asyncio
async def test_sweep_cas_claim_prevents_double_delivery_race() -> None:
    """Two service instances (simulating two concurrent sweep ticks) race the
    same stale row. `_persist_and_deliver`'s 60s dedup guard cannot arbitrate
    this — BLOCKER_ESCALATION isn't a `_LOOP_PRONE_TYPES` member, so it's a
    no-op for this path. The CAS claim in `_claim_reescalation_slot` is what
    actually decides it: exactly one instance wins the guarded UPDATE and
    delivers; the loser (0 rows updated) skips delivery entirely, without
    raising."""
    recipient = _agent("be-pm")
    target = _agent("main-pm")
    notif = _stale_notification(
        requires_ack=True, acked=False, recipient_id=recipient.id
    )

    winner_session = _session_returning([notif], claim_succeeds=True)
    loser_session = _session_returning([notif], claim_succeeds=False)
    winner = _svc_with_agents(
        winner_session, recipient=recipient, escalation_target=target
    )
    loser = _svc_with_agents(
        loser_session, recipient=recipient, escalation_target=target
    )

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="main-pm",
        ),
    ):
        winner_count = await winner.sweep_expired_notifications()
        loser_count = await loser.sweep_expired_notifications()

    assert winner_count == 1
    assert loser_count == 1  # still stale + unacked from the loser's own view
    assert winner_session.add.call_count == 1  # won the claim, delivered
    loser_session.add.assert_not_called()  # lost the claim, never touched delivery


# =============================================================================
# reescalation_decision — pure schedule math
# =============================================================================


_DEFAULT_POLICY = ReescalationPolicy(base_seconds=3600, max_reescalations=5)


def test_reescalation_decision_first_fire_due_at_expiry() -> None:
    """count=0 (including a legacy row with no backoff state) is due the
    instant `now` reaches `expires_at` — preserves the original semantics."""
    expires_at = datetime(2026, 1, 1, tzinfo=UTC)
    assert (
        reescalation_decision(
            now=expires_at,
            expires_at=expires_at,
            count=0,
            last_reescalated_at=None,
            policy=_DEFAULT_POLICY,
        )
        == "due"
    )


def test_reescalation_decision_first_fire_not_due_before_expiry() -> None:
    expires_at = datetime(2026, 1, 1, tzinfo=UTC)
    assert (
        reescalation_decision(
            now=expires_at - timedelta(seconds=1),
            expires_at=expires_at,
            count=0,
            last_reescalated_at=None,
            policy=_DEFAULT_POLICY,
        )
        == "wait"
    )


def test_reescalation_decision_backoff_doubles() -> None:
    """count=2 waits 2*base (2h at the default base) from the last fire."""
    last = datetime(2026, 1, 1, tzinfo=UTC)
    expires_at = last - timedelta(hours=3)
    not_yet = reescalation_decision(
        now=last + timedelta(hours=2) - timedelta(seconds=1),
        expires_at=expires_at,
        count=2,
        last_reescalated_at=last,
        policy=_DEFAULT_POLICY,
    )
    due = reescalation_decision(
        now=last + timedelta(hours=2),
        expires_at=expires_at,
        count=2,
        last_reescalated_at=last,
        policy=_DEFAULT_POLICY,
    )
    assert not_yet == "wait"
    assert due == "due"


def test_reescalation_decision_interval_capped_at_24h() -> None:
    """However high `count` climbs (a raised max_reescalations), the wait
    between attempts never exceeds 24h."""
    last = datetime(2026, 1, 1, tzinfo=UTC)
    expires_at = last - timedelta(days=1)
    policy = ReescalationPolicy(base_seconds=3600, max_reescalations=20)
    # Uncapped this would be base*2**8 = 256h; capped it's 24h.
    not_yet = reescalation_decision(
        now=last + timedelta(hours=24) - timedelta(seconds=1),
        expires_at=expires_at,
        count=9,
        last_reescalated_at=last,
        policy=policy,
    )
    due = reescalation_decision(
        now=last + timedelta(hours=24),
        expires_at=expires_at,
        count=9,
        last_reescalated_at=last,
        policy=policy,
    )
    assert not_yet == "wait"
    assert due == "due"


def test_reescalation_decision_capped_past_max_regardless_of_timing() -> None:
    """count >= max_reescalations is always "capped", even if the schedule
    math would otherwise say a re-escalation is overdue."""
    last = datetime(2026, 1, 1, tzinfo=UTC)
    assert (
        reescalation_decision(
            now=last + timedelta(days=365),
            expires_at=last - timedelta(hours=1),
            count=5,
            last_reescalated_at=last,
            policy=_DEFAULT_POLICY,
        )
        == "capped"
    )


# =============================================================================
# sweep_expired_notifications — backoff integration
# =============================================================================


@pytest.mark.asyncio
async def test_sweep_backoff_does_not_refire_within_the_interval() -> None:
    """A row re-escalates once, then a same-tick-ish second sweep (interval
    not elapsed) does not re-escalate again — and its schedule state (count,
    last_reescalated_at) is stamped on the notification after the first."""
    recipient = _agent("be-pm")
    target = _agent("main-pm")
    notif = _stale_notification(
        requires_ack=True, acked=False, recipient_id=recipient.id
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="main-pm",
        ),
    ):
        first = await svc.sweep_expired_notifications()
        assert notif.reescalation_count == 1
        assert notif.last_reescalated_at is not None
        assert session.add.call_count == 1

        second = await svc.sweep_expired_notifications()

    assert first == 1
    assert second == 1  # still stale + unacked
    assert session.add.call_count == 1  # no second re-escalation this soon


@pytest.mark.asyncio
async def test_sweep_capped_row_never_re_escalates_again() -> None:
    """A row already at the retry cap is skipped forever — no re-escalation,
    no repeat 'permanently unacked' log — but still counts as stale+unacked."""
    recipient = _agent("be-pm")
    target = _agent("main-pm")
    capped_count = settings.notification_max_reescalations
    notif = _stale_notification(
        requires_ack=True,
        acked=False,
        recipient_id=recipient.id,
        reescalation_count=capped_count,
        last_reescalated_at=datetime.now(UTC) - timedelta(days=1),
    )

    session = _session_returning([notif])
    svc = _svc_with_agents(session, recipient=recipient, escalation_target=target)

    with (
        patch(
            "roboco.services.notification_delivery.all_recipients_recently_notified",
            AsyncMock(return_value=False),
        ),
        patch(
            "roboco.services.notification_delivery.get_escalation_target",
            return_value="main-pm",
        ),
    ):
        count = await svc.sweep_expired_notifications()

    assert count == 1  # still stale + unacked
    session.add.assert_not_called()
    assert notif.reescalation_count == capped_count  # untouched — no further attempts