roboco/tests/unit/services/test_release_proposal_orphan_sweep.py

"""A restart mid-execute kills ``_run_approve_background`` but the Redis release
mutex (TTL 3000s) persists with no heartbeat, so a CEO retry gets
``already_in_progress`` for up to 50 min. ``sweep_orphan_release_locks`` scans
the ``roboco:release_proposal:*`` keyspace at orchestrator start and deletes any
whose task_id isn't in the in-flight registry — after a restart that registry is
empty, so every surviving key is an orphan.
"""

from __future__ import annotations

from typing import TYPE_CHECKING, cast
from unittest.mock import patch
from uuid import uuid4

import pytest

if TYPE_CHECKING:
    import asyncio
from roboco.services import release_proposal as rp
from roboco.services.release_proposal import sweep_orphan_release_locks


class _KeysRedis:
    """Minimal fake backing ``keys`` / ``delete`` / ``set`` / ``aclose`` for the
    sweep (the existing ``_FakeRedis`` in the concurrency module has no
    ``keys``). Keys are stored as bytes to mirror redis-py's default
    ``decode_responses=False``."""

    def __init__(self) -> None:
        self._store: dict[bytes, bytes] = {}

    async def set(self, name: str, value: str) -> bool:
        self._store[name.encode()] = value.encode()
        return True

    async def keys(self, pattern: str) -> list[bytes]:
        prefix = pattern.rstrip("*").encode()
        return [k for k in self._store if k.startswith(prefix)]

    async def delete(self, *keys: bytes | str) -> int:
        n = 0
        for k in keys:
            kb = k.encode() if isinstance(k, str) else k
            if self._store.pop(kb, None) is not None:
                n += 1
        return n

    async def get(self, name: str) -> bytes | None:
        return self._store.get(name.encode())

    async def aclose(self) -> None:
        return None


@pytest.mark.asyncio
async def test_sweep_deletes_orphan_release_locks() -> None:
    """A stale lock from a pre-restart approve (no longer in the in-flight
    registry) is deleted; a key whose task_id IS in flight is preserved."""

    fake = _KeysRedis()
    orphan_key = f"{rp._RELEASE_LOCK_PREFIX}{uuid4()}"
    await fake.set(orphan_key, "deadtoken")

    # An in-flight approve — its lock must survive the sweep.
    in_flight_id = uuid4()
    in_flight_key = f"{rp._RELEASE_LOCK_PREFIX}{in_flight_id}"
    await fake.set(in_flight_key, "livetoken")

    rp._INFLIGHT_APPROVES.clear()
    rp._INFLIGHT_APPROVES[in_flight_id] = cast("asyncio.Task[None]", object())

    with patch("roboco.services.release_proposal.redis.from_url", return_value=fake):
        await rp.sweep_orphan_release_locks()

    assert await fake.get(orphan_key) is None  # orphan deleted
    assert await fake.get(in_flight_key) is not None  # in-flight preserved
    rp._INFLIGHT_APPROVES.clear()


@pytest.mark.asyncio
async def test_sweep_ignores_non_uuid_keys() -> None:
    """A key under the prefix whose tail isn't a UUID is left alone (defensive
    against an unrelated key colliding with the prefix)."""

    fake = _KeysRedis()
    junk_key = f"{rp._RELEASE_LOCK_PREFIX}not-a-uuid"
    await fake.set(junk_key, "x")

    rp._INFLIGHT_APPROVES.clear()
    with patch("roboco.services.release_proposal.redis.from_url", return_value=fake):
        await rp.sweep_orphan_release_locks()

    assert await fake.get(junk_key) is not None


@pytest.mark.asyncio
async def test_sweep_redis_failure_does_not_raise() -> None:
    """A Redis outage at startup must not crash the orchestrator — the sweep is
    best-effort and logs a warning instead of raising."""

    class _BoomRedis:
        async def keys(self, _pattern: str) -> list[bytes]:
            raise ConnectionError("redis down")

        async def aclose(self) -> None:
            return None

    with patch(
        "roboco.services.release_proposal.redis.from_url", return_value=_BoomRedis()
    ):
        # Must not raise.
        await rp.sweep_orphan_release_locks()


@pytest.mark.asyncio
async def test_sweep_no_keys_is_noop() -> None:
    """Empty keyspace — sweep returns without touching anything."""

    fake = _KeysRedis()
    rp._INFLIGHT_APPROVES.clear()
    with patch("roboco.services.release_proposal.redis.from_url", return_value=fake):
        await rp.sweep_orphan_release_locks()
    assert await fake.keys(f"{rp._RELEASE_LOCK_PREFIX}*") == []


def test_sweep_is_module_level_callable() -> None:
    """``sweep_orphan_release_locks`` is a module-level function the
    orchestrator can import at startup (not bound to a service instance)."""

    assert callable(sweep_orphan_release_locks)


# ponytail: the in-flight registry values are asyncio.Task; for the sweep test
# only membership matters, so a plain object() stand-in avoids spinning a task.