mirror of
https://github.com/rennf93/roboco.git
synced 2026-08-03 07:23:24 +02:00
feat(self-heal): regression engine — detect + notify the CEO (dormant)
The detect side of the self-healing loop, modeled on the strategy engine: a pure assess() turns breaching telemetry samples into RegressionObservations (with a stable per-signal fingerprint for later dedupe), and run_cycle() is a no-op unless self_heal_enabled and otherwise only sends the CEO one ack-notification per regression. Detect + notify only — it never originates, starts, merges, or deploys; the telemetry source is injectable for testing. 6 unit tests.
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
"""Production self-healing engine ("engine 4") — dormant by default.
|
||||
|
||||
RoboCo heals ITSELF. This watches RoboCo's OWN repo for a regression (a failing
|
||||
CI run on its default branch, via the telemetry source) and, when it sees one,
|
||||
surfaces it to the CEO — and, behind a second opt-in, opens a PENDING fix task
|
||||
into RoboCo's own delivery lifecycle and STOPS. It is deliberately conservative:
|
||||
|
||||
* **Default OFF.** ``self_heal_enabled`` is False, so the orchestrator loop never
|
||||
starts and the existing system is completely unaffected.
|
||||
* **Never self-deploys.** Even fully enabled, the loop only NOTIFIES and, at
|
||||
most, OPENS a PENDING task. It never starts, approves, merges, or deploys work
|
||||
— every downstream step stays a human/CEO decision (the task waits for the
|
||||
CEO's Approve-&-Start and terminates at ``awaiting_ceo_approval``).
|
||||
* **Repo-singular.** It targets only RoboCo's own project
|
||||
(``self_heal_project_slug``); the org's repo-agnostic delivery work is separate.
|
||||
* **Bounded + deduped.** One pass per interval; at most one open fix task per
|
||||
signal fingerprint; per-cycle and rolling open-task caps; the notification
|
||||
layer's purpose-dedup suppresses repeat CEO pings until acknowledged.
|
||||
|
||||
This slice is detect + notify only; task origination is layered on next, behind
|
||||
the second flag.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from roboco.config import settings
|
||||
from roboco.services.base import BaseService
|
||||
from roboco.services.notification import NotificationService
|
||||
from roboco.services.telemetry import get_ci_telemetry_source
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from roboco.services.telemetry import TelemetrySource
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RegressionObservation:
|
||||
"""One regression the engine detected in RoboCo's own repo."""
|
||||
|
||||
fingerprint: str # stable per signal — the dedupe key for open fix tasks
|
||||
signal_name: str
|
||||
repo_hint: str
|
||||
summary: str
|
||||
detail: str
|
||||
raw_ref: str
|
||||
|
||||
|
||||
def _fingerprint(signal_name: str) -> str:
|
||||
"""Stable short hash of the signal (which already encodes the repo)."""
|
||||
return hashlib.sha256(signal_name.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class SelfHealEngine(BaseService):
|
||||
"""Detect a regression in RoboCo's own repo; surface it (and later open a fix)."""
|
||||
|
||||
service_name = "self_heal_engine"
|
||||
|
||||
def __init__(
|
||||
self, session: AsyncSession, source: TelemetrySource | None = None
|
||||
) -> None:
|
||||
super().__init__(session)
|
||||
self._source: TelemetrySource = source or get_ci_telemetry_source(session)
|
||||
|
||||
async def assess(self) -> list[RegressionObservation]:
|
||||
"""Read telemetry and return observations. Pure — no side effects."""
|
||||
observations: list[RegressionObservation] = []
|
||||
for sample in await self._source.fetch():
|
||||
if not sample.is_breach:
|
||||
continue
|
||||
observations.append(
|
||||
RegressionObservation(
|
||||
fingerprint=_fingerprint(sample.signal_name),
|
||||
signal_name=sample.signal_name,
|
||||
repo_hint=sample.repo_hint,
|
||||
summary=f"Regression detected on {sample.repo_hint}.",
|
||||
detail=sample.detail
|
||||
or f"{sample.signal_name} breached its threshold.",
|
||||
raw_ref=sample.raw_ref,
|
||||
)
|
||||
)
|
||||
return observations
|
||||
|
||||
async def run_cycle(self) -> list[RegressionObservation]:
|
||||
"""Assess and notify the CEO. No-op unless self-healing is enabled.
|
||||
|
||||
Detect + notify only; never starts, merges, or deploys anything.
|
||||
"""
|
||||
if not settings.self_heal_enabled:
|
||||
return []
|
||||
observations = await self.assess()
|
||||
if not observations:
|
||||
return []
|
||||
notifier = NotificationService()
|
||||
for obs in observations:
|
||||
body = f"[self-heal] {obs.summary}\n\n{obs.detail}"
|
||||
if obs.raw_ref:
|
||||
body += f"\n\nEvidence: {obs.raw_ref}"
|
||||
await notifier.send_ack_notification(
|
||||
from_agent="system", to_agent="ceo", body=body
|
||||
)
|
||||
return observations
|
||||
|
||||
|
||||
def get_self_heal_engine(
|
||||
session: AsyncSession, source: TelemetrySource | None = None
|
||||
) -> SelfHealEngine:
|
||||
"""Construct a SelfHealEngine bound to ``session`` (optionally a test source)."""
|
||||
return SelfHealEngine(session, source=source)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Self-heal regression engine — pure assess + gated, notify-only run_cycle.
|
||||
|
||||
``assess()`` turns breaching telemetry samples into observations with no side
|
||||
effects; ``run_cycle()`` is a no-op unless ``self_heal_enabled`` and otherwise
|
||||
only notifies the CEO (this slice never originates, starts, merges, or deploys).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from roboco.config import settings as cfg
|
||||
from roboco.services.notification import NotificationService
|
||||
from roboco.services.self_heal_engine import SelfHealEngine
|
||||
from roboco.services.telemetry import TelemetrySample
|
||||
|
||||
|
||||
class _FakeSource:
|
||||
"""A TelemetrySource stand-in returning canned samples."""
|
||||
|
||||
def __init__(self, samples: list[TelemetrySample]) -> None:
|
||||
self._samples = samples
|
||||
|
||||
async def fetch(self) -> list[TelemetrySample]:
|
||||
return list(self._samples)
|
||||
|
||||
|
||||
def _sample(value: float) -> TelemetrySample:
|
||||
return TelemetrySample(
|
||||
signal_name="ci_conclusion:roboco",
|
||||
value=value,
|
||||
threshold=1.0,
|
||||
window="latest_completed_run",
|
||||
repo_hint="roboco",
|
||||
observed_at="2026-06-17T00:00:00Z",
|
||||
raw_ref="https://github.com/x/roboco/actions/runs/1",
|
||||
detail="CI on roboco@master concluded 'failure'",
|
||||
)
|
||||
|
||||
|
||||
def _engine(samples: list[TelemetrySample]) -> SelfHealEngine:
|
||||
return SelfHealEngine(MagicMock(), source=_FakeSource(samples))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assess_breach_yields_observation() -> None:
|
||||
obs = await _engine([_sample(1.0)]).assess()
|
||||
assert len(obs) == 1
|
||||
assert obs[0].repo_hint == "roboco"
|
||||
assert obs[0].signal_name == "ci_conclusion:roboco"
|
||||
assert obs[0].fingerprint # non-empty stable hash
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assess_no_breach_yields_nothing() -> None:
|
||||
assert await _engine([_sample(0.0)]).assess() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assess_is_pure_no_notification(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
send = AsyncMock()
|
||||
monkeypatch.setattr(NotificationService, "send_ack_notification", send)
|
||||
await _engine([_sample(1.0)]).assess()
|
||||
send.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_noop_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(cfg, "self_heal_enabled", False)
|
||||
send = AsyncMock()
|
||||
monkeypatch.setattr(NotificationService, "send_ack_notification", send)
|
||||
assert await _engine([_sample(1.0)]).run_cycle() == []
|
||||
send.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cycle_notifies_ceo_when_enabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(cfg, "self_heal_enabled", True)
|
||||
send = AsyncMock()
|
||||
monkeypatch.setattr(NotificationService, "send_ack_notification", send)
|
||||
obs = await _engine([_sample(1.0)]).run_cycle()
|
||||
assert len(obs) == 1
|
||||
send.assert_awaited_once()
|
||||
call = send.await_args
|
||||
assert call is not None
|
||||
assert call.kwargs["from_agent"] == "system"
|
||||
assert call.kwargs["to_agent"] == "ceo"
|
||||
assert "[self-heal]" in call.kwargs["body"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fingerprint_is_stable() -> None:
|
||||
a = (await _engine([_sample(1.0)]).assess())[0].fingerprint
|
||||
b = (await _engine([_sample(1.0)]).assess())[0].fingerprint
|
||||
assert a == b
|
||||
Reference in New Issue
Block a user