Merge pull request #22 from 12vectors/task/11-failed-agent-runs-must-be-visible
11 — A failed agent run must leave a visible trace on the card
This commit is contained in:
@@ -280,10 +280,35 @@ reassignment. An unclaimed card claims itself on launch.
|
||||
2. The agent works in the worktree: implements, tests, commits. Its hook
|
||||
events stream to the board like any session.
|
||||
3. On clean exit with commits on the branch the board moves the card to
|
||||
`review/`; on failure it stays in `in-progress/` and the exit is narrated
|
||||
in the ticker. A clean exit that committed *nothing* also stays in
|
||||
`review/`; on failure it stays in `in-progress/` and the card wears the
|
||||
failure (below). A clean exit that committed *nothing* also stays in
|
||||
`in-progress/` and is called out loudly — an empty branch reaching
|
||||
review/ is how a broken launch hides. Stdout is kept in `.agent/logs/`.
|
||||
review/ is how a broken launch hides. Stdout is kept in
|
||||
`local/state/agent/logs/`.
|
||||
|
||||
### A run that died
|
||||
|
||||
An agent that exits non-zero is the one outcome a person must not miss, so
|
||||
it is a **state the card wears**, not an event that scrolls past. The run's
|
||||
record keeps the exit code, when it ended, and the cleaned tail of its log
|
||||
— the excerpt, which for an API outage is the whole story ("API Error:
|
||||
500 …") and which a launch that died before the agent ever spoke still
|
||||
answers honestly. From that the board does three things: the card takes the
|
||||
`--alarm` border and a `run failed` pill, with the excerpt on hover and in
|
||||
full in the card sheet; a toast fires, because failures are rare and
|
||||
actionable; and the ticker keeps its line, now naming what the log ended on
|
||||
rather than pointing vaguely at a file. Every headless kind lands here —
|
||||
work, act-pr, PR review, relevance check — and a card that is not in
|
||||
in-progress wears it just the same.
|
||||
|
||||
The state is scoped to the run and the stage: the next launch supersedes it
|
||||
(the card reads its most recent run), and moving the card to another stage
|
||||
drops it, since the failure was about the work in the stage it died in.
|
||||
Nothing retries by itself — a dead run is a human decision point, and an
|
||||
outage would make auto-retry a thundering herd — but the way is cleared for
|
||||
the human: a failed run that committed nothing has its worktree and empty
|
||||
branch removed, exactly as a decline does, so **▸ start work** is one click
|
||||
again. A failed run *with* commits keeps its worktree; there is work in it.
|
||||
|
||||
## Pull requests
|
||||
|
||||
|
||||
+93
-4
@@ -84,6 +84,10 @@ def _agent_public(record: dict) -> dict:
|
||||
# The model the launch was actually given; None = inherited the
|
||||
# vendor's own default. Honesty for the Sessions/Focus views.
|
||||
public["model"] = record.get("model")
|
||||
public["ended"] = record.get("ended")
|
||||
# The outcome a failed run leaves behind, for the card to wear (see
|
||||
# _record_failure). None on every run that did not die.
|
||||
public["failure"] = record.get("failure")
|
||||
return public
|
||||
|
||||
|
||||
@@ -368,6 +372,78 @@ def _discard_untouched_worktree(record: dict) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _failure_excerpt(log_path: str | None, lines: int = 6, cap: int = 600) -> str:
|
||||
"""The tail of a dead run's log, cleaned — usually the whole story
|
||||
("API Error: 500 …"). A launch that died before the agent ever spoke
|
||||
leaves a line or two, or nothing at all; say which rather than showing
|
||||
an empty card."""
|
||||
text = ""
|
||||
if log_path:
|
||||
try:
|
||||
text = Path(log_path).read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
text = ""
|
||||
kept = [line.rstrip() for line in _clean_log(text, cap=8000).splitlines()
|
||||
if line.strip()]
|
||||
if not kept:
|
||||
return "no output — the run died before the agent said anything"
|
||||
return "\n".join(kept[-lines:])[-cap:]
|
||||
|
||||
|
||||
def _headline(excerpt: str, cap: int = 120) -> str:
|
||||
"""One line of an excerpt for a ticker line or a toast: the last one,
|
||||
which is where a dying process says why."""
|
||||
lines = [line for line in (excerpt or "").splitlines() if line.strip()]
|
||||
return (lines[-1].strip()[:cap] if lines else "no output")
|
||||
|
||||
|
||||
def _why(record: dict) -> str:
|
||||
"""What a dead run's log ended on, for the ticker line that records it."""
|
||||
return _headline((record.get("failure") or {}).get("excerpt", ""))
|
||||
|
||||
|
||||
def _record_failure(record: dict, rc: int) -> dict:
|
||||
"""A dead run is a state its card wears, not an event that scrolls by.
|
||||
|
||||
Every headless kind lands here, launches that died before the agent
|
||||
spoke included: the outcome goes onto the run's record — exit code,
|
||||
when it ended, the log's cleaned tail, and the stage the card was in —
|
||||
so the board can show it, and a toast says it once to whoever is
|
||||
looking. The stage is part of the state because the state is about
|
||||
work in that stage: carried into review/ it would libel the next run.
|
||||
"""
|
||||
failure = {
|
||||
"rc": rc,
|
||||
"ended": record.get("ended") or time.time(),
|
||||
"excerpt": _failure_excerpt(record.get("log")),
|
||||
"stage": find_stage_of(record["task"]) or record.get("origin"),
|
||||
"log": record.get("log"),
|
||||
"mode": record.get("mode", "work"),
|
||||
}
|
||||
with state.LOCK:
|
||||
record["failure"] = failure
|
||||
name = record.get("name") or "the agent"
|
||||
state.broadcast({
|
||||
"type": "toast", "error": True,
|
||||
"message": f"{name} failed on {record['task']} (rc={rc}) — "
|
||||
f"{_headline(failure['excerpt'])}",
|
||||
})
|
||||
return failure
|
||||
|
||||
|
||||
def forget_failure(filename: str) -> bool:
|
||||
"""Drop a card's failed-run state. Called when the card moves stage:
|
||||
the failure belonged to the work in the stage it died in, and no card
|
||||
should arrive somewhere new already wearing an alarm. A relaunch needs
|
||||
no call — the newer run is what the card reads."""
|
||||
cleared = False
|
||||
with state.LOCK:
|
||||
for record in state.AGENTS.values():
|
||||
if record["task"] == filename and record.pop("failure", None):
|
||||
cleared = True
|
||||
return cleared
|
||||
|
||||
|
||||
def _finish(agent_id: str, proc: subprocess.Popen, log_file) -> tuple[dict, bool, int]:
|
||||
rc = proc.wait()
|
||||
log_file.close()
|
||||
@@ -376,6 +452,10 @@ def _finish(agent_id: str, proc: subprocess.Popen, log_file) -> tuple[dict, bool
|
||||
stopped = record["status"] == "stopped"
|
||||
record["status"] = "stopped" if stopped else ("done" if rc == 0 else "failed")
|
||||
record["rc"] = rc
|
||||
record["ended"] = time.time()
|
||||
failed = record["status"] == "failed"
|
||||
if failed:
|
||||
_record_failure(record, rc)
|
||||
return record, stopped, rc
|
||||
|
||||
|
||||
@@ -422,7 +502,14 @@ def _reap_agent(agent_id: str, proc: subprocess.Popen, log_file) -> None:
|
||||
elif stopped:
|
||||
summary = f"{name} was held on {filename} — nothing is lost"
|
||||
else:
|
||||
summary = f"{name} exited on {filename} rc={rc} — see its log"
|
||||
# The card now wears the failure; the ticker keeps the record of it
|
||||
# and names what the log's tail said. A run that committed nothing
|
||||
# also leaves nothing worth keeping, so the worktree goes and
|
||||
# ▸ start work is one click again — same reasoning as a decline.
|
||||
cleaned = _discard_untouched_worktree(record)
|
||||
summary = (f"{name} exited on {filename} rc={rc} — {_why(record)}"
|
||||
+ (" (worktree cleared — relaunch when you have read it)" if cleaned
|
||||
else f" (worktree {record['worktree']} kept: it has commits)"))
|
||||
state.record_board_event({"kind": "agent", "actor": "agent", "file": filename,
|
||||
"summary": summary})
|
||||
state.broadcast({"type": "agents"})
|
||||
@@ -534,7 +621,7 @@ def _reap_pr_fix(agent_id: str, proc: subprocess.Popen, log_file) -> None:
|
||||
elif stopped:
|
||||
summary = f"{name} was held while acting on {filename}'s PR"
|
||||
else:
|
||||
summary = f"{name} failed acting on {filename}'s PR (rc={rc}) — see its log"
|
||||
summary = f"{name} failed acting on {filename}'s PR (rc={rc}) — {_why(record)}"
|
||||
state.record_board_event({"kind": "agent", "actor": "agent", "file": filename,
|
||||
"summary": summary})
|
||||
state.broadcast({"type": "board"})
|
||||
@@ -561,7 +648,9 @@ def _reap_pr_review(agent_id: str, proc: subprocess.Popen, log_file) -> None:
|
||||
|
||||
if stopped:
|
||||
summary = f"{name}'s PR review of {filename} was held"
|
||||
elif rc != 0 or verdict is None:
|
||||
elif rc != 0:
|
||||
summary = f"{name}'s PR review of {filename} died (rc={rc}) — {_why(record)}"
|
||||
elif verdict is None:
|
||||
summary = f"{name}'s PR review of {filename} ended without a verdict — see its log"
|
||||
else:
|
||||
word = "approved it" if verdict == "APPROVE" else "asked for changes"
|
||||
@@ -592,7 +681,7 @@ def _reap_review(agent_id: str, proc: subprocess.Popen, log_file) -> None:
|
||||
if stopped:
|
||||
summary = f"{name}'s check of {filename} was held"
|
||||
elif rc != 0:
|
||||
summary = f"{name}'s check of {filename} exited rc={rc} — see its log"
|
||||
summary = f"{name}'s check of {filename} exited rc={rc} — {_why(record)}"
|
||||
else:
|
||||
summary = f"{name} on {filename}: {verdict[:140] if verdict else 'report appended to the task'}"
|
||||
state.record_board_event({"kind": "agent", "actor": "agent", "file": filename,
|
||||
|
||||
+59
-2
@@ -158,6 +158,9 @@
|
||||
/* PR verdicts: pine when it settled, terracotta when it snagged */
|
||||
.card.verdict-good{border-color:color-mix(in oklab, var(--calm) 55%, var(--border))}
|
||||
.card.verdict-bad{border-color:color-mix(in oklab, var(--alarm) 55%, var(--border))}
|
||||
/* the last run on this card died: the same terracotta, worn until the
|
||||
next launch replaces it or the card moves stage */
|
||||
.card.run-failed{border-color:color-mix(in oklab, var(--alarm) 55%, var(--border))}
|
||||
/* tool chips: destinations, not statuses — they live in the card's footer,
|
||||
never squeezed into the author row */
|
||||
.chiprow{
|
||||
@@ -437,6 +440,8 @@
|
||||
padding:20px 20px 20px 6px;
|
||||
}
|
||||
#drawer .dhead{display:flex;align-items:center;gap:10px}
|
||||
/* the dead run's excerpt, above the task itself: machine output, bounded */
|
||||
#drawer .well.bad pre{margin:6px 0 4px;white-space:pre-wrap;max-height:220px;overflow-y:auto;color:var(--text)}
|
||||
#drawer .dbody{font-size:13px;line-height:1.6}
|
||||
#drawer .dbody h1{font-size:19px;line-height:1.3;font-weight:600;letter-spacing:-.01em;margin:0 0 4px;text-wrap:pretty}
|
||||
#drawer .dbody h2{font-size:14px;margin:18px 0 6px}
|
||||
@@ -701,6 +706,30 @@ function agentFor(sid) { return (S.state?.agents || []).find(a => a.session ===
|
||||
function agentOnTask(file) {
|
||||
return (S.state?.agents || []).find(a => a.task === file && a.status === 'running');
|
||||
}
|
||||
/* The most recent run on a card. Records outlive their processes, so the
|
||||
latest launch is a max-by-start question, not a find. */
|
||||
function lastRunOn(file) {
|
||||
return (S.state?.agents || []).reduce(
|
||||
(best, a) => (a.task === file && (!best || a.started > best.started) ? a : best), null);
|
||||
}
|
||||
/* A dead run is a state the card wears: alarm border, `run failed` pill and
|
||||
the log's tail, until the next launch replaces it (a newer run is the
|
||||
latest one) or the card moves stage (the server drops the state, and the
|
||||
stage stamp keeps the card honest in the seconds before the watcher
|
||||
notices). Every headless kind counts — work, act-pr, PR review, relevance. */
|
||||
function failedRun(task) {
|
||||
const last = lastRunOn(task.file);
|
||||
const failure = last && last.status === 'failed' ? last.failure : null;
|
||||
return failure && failure.stage === task.stage ? failure : null;
|
||||
}
|
||||
/* The line a run died on: the excerpt's last, which is where a dying
|
||||
process says why ("API Error: 500 …"). Bounded, so one enormous line of
|
||||
machine output cannot grow the card — the whole excerpt is a hover away. */
|
||||
function whyFailed(failure) {
|
||||
const lines = (failure.excerpt || '').split('\n').filter(l => l.trim());
|
||||
const why = lines.length ? lines[lines.length - 1].trim() : 'no output';
|
||||
return why.length > 160 ? why.slice(0, 160) + '…' : why;
|
||||
}
|
||||
function sessionMeta(sid) { return (S.state?.sessions || []).find(m => m.id === sid); }
|
||||
function isLiveSession(m) {
|
||||
const fresh = (Date.now() / 1000 - (m.last || m.started || 0)) < 900;
|
||||
@@ -888,10 +917,12 @@ function cardFor(task) {
|
||||
const agent = agentOnTask(task.file);
|
||||
const working = agent && agent.mode !== 'review';
|
||||
const verdict = task.stage === 'review' ? prVerdict(task) : null;
|
||||
const failure = failedRun(task);
|
||||
el.className = 'card'
|
||||
+ (S.selected && S.selected.file === task.file ? ' selected' : '')
|
||||
+ (working ? ' running' : '')
|
||||
+ (verdict === 'green' ? ' verdict-good' : verdict === 'red' ? ' verdict-bad' : '')
|
||||
+ (failure ? ' run-failed' : '')
|
||||
+ (task.stage === 'done' ? ' done-dim' : '');
|
||||
el.draggable = true;
|
||||
|
||||
@@ -906,6 +937,13 @@ function cardFor(task) {
|
||||
pill = { text: 'changes asked', tint: 'var(--alarm)', bg: mix('var(--alarm)', 16) };
|
||||
tint = 'var(--alarm)';
|
||||
}
|
||||
if (failure) {
|
||||
// the newest thing that happened here, and the only actionable one:
|
||||
// it outranks a PR verdict from before the run died
|
||||
pill = { text: 'run failed', tint: 'var(--alarm)', bg: mix('var(--alarm)', 16),
|
||||
title: failure.excerpt };
|
||||
tint = 'var(--alarm)';
|
||||
}
|
||||
const high = (task.priority || '').toLowerCase() === 'high';
|
||||
const top = [
|
||||
`<span class="mark${working ? ' breathing' : ''}" style="background:${tint}"></span>`,
|
||||
@@ -913,7 +951,8 @@ function cardFor(task) {
|
||||
'<span class="spacer"></span>',
|
||||
high ? '<span class="high">HIGH</span>' : '',
|
||||
task.statusMismatch ? `<span class="pill drift" title="File says ${esc(task.declaredStatus)}">drift</span>` : '',
|
||||
`<span class="pill status" style="background:${pill.bg};color:${pill.tint}">${pill.text}</span>`,
|
||||
`<span class="pill status" style="background:${pill.bg};color:${pill.tint}"` +
|
||||
`${pill.title ? ` title="${esc(pill.title)}"` : ''}>${pill.text}</span>`,
|
||||
];
|
||||
|
||||
// two actions per state, max — whatever you'd actually do without opening the card
|
||||
@@ -1010,6 +1049,12 @@ function cardFor(task) {
|
||||
}
|
||||
} else if (agent) {
|
||||
liveLine = `<div class="well"><span class="lead">·</span><span class="wbody">warming up<span class="caret">▌</span></span></div>`;
|
||||
} else if (failure) {
|
||||
// "API Error: 500" one hover away instead of buried in a log file: the
|
||||
// line the run died on here, the whole excerpt on hover and in the sheet
|
||||
liveLine = `<div class="well bad" title="${esc(failure.excerpt)}">` +
|
||||
`<span class="lead">·</span><span class="wbody">rc=${esc(failure.rc)} · ` +
|
||||
`${esc(whyFailed(failure))}</span></div>`;
|
||||
}
|
||||
|
||||
// tool chips: destinations, not statuses — they live in the card's footer
|
||||
@@ -1375,16 +1420,28 @@ function renderDrawer() {
|
||||
if (!S.selected) { panel.classList.remove('open'); body.innerHTML = ''; return; }
|
||||
const t = S.selected;
|
||||
const agent = agentOnTask(t.file);
|
||||
const pill = agent && agent.mode === 'review'
|
||||
const failure = failedRun(t);
|
||||
const pill = failure
|
||||
? { text: 'run failed', tint: 'var(--alarm)', bg: mix('var(--alarm)', 16) }
|
||||
: agent && agent.mode === 'review'
|
||||
? { text: 'reviewing', tint: 'var(--accent)', bg: mix('var(--accent)', 16) }
|
||||
: pillFor(t.stage, agent && agent.mode !== 'review');
|
||||
const when = new Date(t.mtime * 1000).toLocaleString();
|
||||
// the whole excerpt, not just the line the card shows — the sheet is
|
||||
// where you read what killed the run without opening files on disk
|
||||
const failBlock = failure
|
||||
? `<div class="well bad"><span class="lead">·</span><div class="wbody">` +
|
||||
`<b>run failed</b> · rc=${esc(failure.rc)} · ${esc(ago(failure.ended))} ago` +
|
||||
`<pre>${esc(failure.excerpt)}</pre>` +
|
||||
`<span style="color:var(--dim)">${esc(failure.log || '')}</span></div></div>`
|
||||
: '';
|
||||
body.innerHTML =
|
||||
`<div class="dhead">` +
|
||||
`<span class="mono" style="font-size:12px;color:var(--dim)">${t.number ? '#' + esc(t.number) : ''}</span>` +
|
||||
`<span class="pill" style="background:${pill.bg};color:${pill.tint}">${pill.text}</span>` +
|
||||
`<span class="spacer"></span>` +
|
||||
`<button id="closeDrawer">Close</button></div>` +
|
||||
failBlock +
|
||||
`<div class="dbody">${md(t.body)}</div>` +
|
||||
`<div class="dmeta">${esc(t.stage)}/${esc(t.file)} · ${t.words} words · edited ${esc(when)}</div>`;
|
||||
panel.classList.add('open');
|
||||
|
||||
@@ -15,6 +15,7 @@ from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import agents
|
||||
import config
|
||||
import github
|
||||
import state
|
||||
@@ -57,6 +58,9 @@ def narrate(prev: dict[str, set[str]], cur: dict[str, set[str]]) -> None:
|
||||
"actor": actor, "remote": remote,
|
||||
"summary": f"{f} moved {prev_loc[f]} → {stage} ({actor})",
|
||||
})
|
||||
# a failed run is worn by the card in the stage it died in —
|
||||
# wherever the card goes next, it arrives without the alarm
|
||||
agents.forget_failure(f)
|
||||
if stage == "review" and not remote:
|
||||
# a card entering review with a work branch gets a PR — on
|
||||
# the actor's board only, or the team gets one PR attempt
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
"""A failed agent run leaves a visible trace (task 11).
|
||||
|
||||
Three agents died in an API outage and the board's whole answer was one
|
||||
ticker line that scrolled away. So: a dead run is recorded on its launch
|
||||
record (exit code, ended-at, the log's cleaned tail), the person is told
|
||||
once by toast, the card wears it until the next launch or the next stage,
|
||||
and the untouched worktree a dead run left behind is cleared so ▸ start
|
||||
work is one click again.
|
||||
|
||||
Real launches through a real (stub) adapter — the adapter contract is what
|
||||
a dying agent actually comes through, so nothing is mocked but the SSE
|
||||
fan-out and the adapter's own binary.
|
||||
|
||||
python3 -m unittest discover -s tests -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO / "manager" / "core"))
|
||||
|
||||
import agents # noqa: E402
|
||||
import config # noqa: E402
|
||||
import state # noqa: E402
|
||||
import watch # noqa: E402
|
||||
|
||||
BOARD = REPO / "manager" / "core" / "board.html"
|
||||
|
||||
FILENAME = "11-a-run-that-dies.md"
|
||||
STEM = FILENAME[:-3]
|
||||
BRANCH = f"task/{STEM}"
|
||||
|
||||
CARD = """# 11 — A run that dies
|
||||
|
||||
**Status:** In Progress
|
||||
**Priority:** High
|
||||
|
||||
Body text, so the prompt has something to carry.
|
||||
"""
|
||||
|
||||
# What the outage looked like from the board's side: a line of output on
|
||||
# stdout and a non-zero exit.
|
||||
DIES = """#!/usr/bin/env python3
|
||||
import sys
|
||||
print("thinking…")
|
||||
print("API Error: 500 {\\"type\\":\\"error\\",\\"error\\":{\\"type\\":\\"api_error\\"}}")
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
# Same death, but it committed first: there is work to keep.
|
||||
DIES_WITH_WORK = """#!/usr/bin/env python3
|
||||
import os, subprocess, sys
|
||||
cwd = os.environ["AGENT_CWD"]
|
||||
open(os.path.join(cwd, "half.txt"), "w").write("half a feature\\n")
|
||||
subprocess.run(["git", "-C", cwd, "add", "-A"], check=True)
|
||||
subprocess.run(["git", "-C", cwd, "-c", "user.email=a@b", "-c", "user.name=stub",
|
||||
"commit", "-q", "-m", "half"], check=True)
|
||||
print("API Error: 529 overloaded")
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
SILENT_DEATH = """#!/usr/bin/env python3
|
||||
import sys
|
||||
sys.exit(1)
|
||||
"""
|
||||
|
||||
LIVES = """#!/usr/bin/env python3
|
||||
print("all good")
|
||||
"""
|
||||
|
||||
|
||||
def git(cwd: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-C", str(cwd), *args],
|
||||
capture_output=True, text=True)
|
||||
|
||||
|
||||
def wait_for(pred, timeout: float = 20.0) -> bool:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if pred():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
|
||||
class Launches(unittest.TestCase):
|
||||
"""One repo, one card, one adapter whose behaviour each test writes."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix="bench-failed-")).resolve()
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.repo = self.tmp / "repo"
|
||||
self.repo.mkdir()
|
||||
git(self.repo, "init", "-q", "-b", "main")
|
||||
git(self.repo, "config", "user.email", "t@t")
|
||||
git(self.repo, "config", "user.name", "tester")
|
||||
(self.repo / "code.txt").write_text("shipped\n", encoding="utf-8")
|
||||
git(self.repo, "add", "-A")
|
||||
git(self.repo, "commit", "-q", "-m", "root")
|
||||
|
||||
tasks = self.repo / "tasks"
|
||||
for slug in config.STAGE_DIRS:
|
||||
(tasks / slug).mkdir(parents=True)
|
||||
(tasks / "in-progress" / FILENAME).write_text(CARD, encoding="utf-8")
|
||||
|
||||
local = self.tmp / "local"
|
||||
(local / "adapters" / config.ADAPTER).mkdir(parents=True)
|
||||
self.adapter = local / "adapters" / config.ADAPTER / "run"
|
||||
|
||||
self.patch(REPO=self.repo, TASKS=tasks, LOCAL=local,
|
||||
WORKTREES=self.tmp / "worktrees",
|
||||
AGENT_DIR=self.tmp / "agent",
|
||||
SESSIONS_DIR=self.tmp / "sessions",
|
||||
COMMIT_MOVES=False, SYNC=False)
|
||||
|
||||
state.AGENTS.clear()
|
||||
state.BOARD_EVENTS.clear()
|
||||
state.EXPECTED_MOVES.clear()
|
||||
self.addCleanup(state.AGENTS.clear)
|
||||
self.addCleanup(state.BOARD_EVENTS.clear)
|
||||
|
||||
self.sent: list[dict] = []
|
||||
self.addCleanup(setattr, state, "broadcast", state.broadcast)
|
||||
state.broadcast = self.sent.append
|
||||
|
||||
def patch(self, **values) -> None:
|
||||
for attr, value in values.items():
|
||||
self.addCleanup(setattr, config, attr, getattr(config, attr))
|
||||
setattr(config, attr, value)
|
||||
|
||||
def adapter_is(self, script: str) -> None:
|
||||
self.adapter.write_text(script, encoding="utf-8")
|
||||
self.adapter.chmod(self.adapter.stat().st_mode | stat.S_IEXEC)
|
||||
|
||||
def run_agent(self, script: str, start=None) -> dict:
|
||||
"""Launch, wait for the reaper to be done with it, return the record.
|
||||
|
||||
Every reaper's last act is the agents broadcast, so counting those
|
||||
is the honest "it has finished" — the assertions then see a settled
|
||||
record rather than one mid-reap."""
|
||||
self.adapter_is(script)
|
||||
ended = self.sent.count({"type": "agents"})
|
||||
public = (start or (lambda: agents.start_agent(FILENAME, "in-progress")))()
|
||||
record = state.AGENTS[public["id"]]
|
||||
self.assertTrue(
|
||||
wait_for(lambda: self.sent.count({"type": "agents"}) > ended),
|
||||
f"the reaper never announced the ending (status {record['status']})")
|
||||
return record
|
||||
|
||||
def summaries(self) -> list[str]:
|
||||
return [e["summary"] for e in state.BOARD_EVENTS]
|
||||
|
||||
def toasts(self) -> list[dict]:
|
||||
return [m for m in self.sent if m.get("type") == "toast"]
|
||||
|
||||
def stage_of(self, filename: str = FILENAME) -> str | None:
|
||||
for slug in config.STAGE_DIRS:
|
||||
if (config.TASKS / slug / filename).is_file():
|
||||
return slug
|
||||
return None
|
||||
|
||||
|
||||
class TheOutcomeIsRecorded(Launches):
|
||||
def test_a_dead_work_run_lands_on_its_record(self):
|
||||
record = self.run_agent(DIES)
|
||||
self.assertEqual(record["status"], "failed")
|
||||
failure = record["failure"]
|
||||
self.assertEqual(failure["rc"], 1)
|
||||
self.assertIn("API Error: 500", failure["excerpt"])
|
||||
self.assertEqual(failure["stage"], "in-progress")
|
||||
self.assertGreaterEqual(failure["ended"], record["started"])
|
||||
self.assertTrue(Path(failure["log"]).is_file(),
|
||||
"the failure must name a log that exists")
|
||||
|
||||
def test_the_card_never_advances(self):
|
||||
self.run_agent(DIES)
|
||||
self.assertEqual(self.stage_of(), "in-progress")
|
||||
|
||||
def test_the_person_is_toasted_once(self):
|
||||
self.run_agent(DIES)
|
||||
toasts = self.toasts()
|
||||
self.assertEqual(len(toasts), 1, "a failure is one toast, not none or two")
|
||||
self.assertTrue(toasts[0]["error"], "a failure toast is an alarm")
|
||||
self.assertIn("API Error: 500", toasts[0]["message"])
|
||||
self.assertIn(FILENAME, toasts[0]["message"])
|
||||
|
||||
def test_the_ticker_line_still_records_it(self):
|
||||
"""This card adds surfaces; it does not move the permanent record."""
|
||||
self.run_agent(DIES)
|
||||
line = [s for s in self.summaries() if "rc=1" in s]
|
||||
self.assertTrue(line, "the event log lost the exit line")
|
||||
self.assertIn(FILENAME, line[0])
|
||||
self.assertIn("API Error: 500", line[0])
|
||||
|
||||
def test_the_public_payload_carries_it(self):
|
||||
"""The card reads the API, not the board's memory."""
|
||||
self.run_agent(DIES)
|
||||
public = agents.list_public()[0]
|
||||
self.assertEqual(public["status"], "failed")
|
||||
self.assertIn("API Error: 500", public["failure"]["excerpt"])
|
||||
self.assertIsNotNone(public["ended"])
|
||||
|
||||
def test_a_live_run_carries_no_failure(self):
|
||||
record = self.run_agent(LIVES)
|
||||
self.assertEqual(record["status"], "done")
|
||||
self.assertIsNone(agents.list_public()[0]["failure"])
|
||||
self.assertEqual(self.toasts(), [])
|
||||
|
||||
|
||||
class TheWayIsClearedForRelaunch(Launches):
|
||||
def test_an_untouched_worktree_goes(self):
|
||||
"""Nothing of value is lost — the run committed nothing — and
|
||||
▸ start work refuses while the worktree exists."""
|
||||
record = self.run_agent(DIES)
|
||||
self.assertFalse(Path(record["worktree"]).exists(),
|
||||
"a dead run with no commits must not block the relaunch")
|
||||
self.assertEqual(
|
||||
git(self.repo, "rev-parse", "--verify", "--quiet", BRANCH).returncode, 1,
|
||||
"the empty branch goes with the worktree")
|
||||
self.assertTrue(any("worktree cleared" in s for s in self.summaries()))
|
||||
|
||||
def test_the_relaunch_actually_works(self):
|
||||
self.run_agent(DIES)
|
||||
state.BOARD_EVENTS.clear()
|
||||
second = self.run_agent(LIVES)
|
||||
self.assertEqual(second["status"], "done")
|
||||
|
||||
def test_a_run_with_commits_keeps_its_worktree(self):
|
||||
record = self.run_agent(DIES_WITH_WORK)
|
||||
self.assertTrue(Path(record["worktree"]).exists(),
|
||||
"work that was committed is never thrown away")
|
||||
self.assertIn("kept", " ".join(self.summaries()))
|
||||
|
||||
|
||||
class EveryHeadlessKind(Launches):
|
||||
def test_a_dead_relevance_check_surfaces_the_same_way(self):
|
||||
"""No worktree, any stage — same state on the card."""
|
||||
record = self.run_agent(
|
||||
DIES, start=lambda: agents.start_review(FILENAME, "in-progress"))
|
||||
self.assertEqual(record["status"], "failed")
|
||||
self.assertIn("API Error: 500", record["failure"]["excerpt"])
|
||||
self.assertEqual(record["failure"]["stage"], "in-progress")
|
||||
self.assertEqual(len(self.toasts()), 1)
|
||||
|
||||
def test_a_death_before_the_agent_spoke_still_says_something(self):
|
||||
record = self.run_agent(SILENT_DEATH)
|
||||
self.assertIn("no output", record["failure"]["excerpt"])
|
||||
self.assertIn("no output", self.toasts()[0]["message"])
|
||||
|
||||
|
||||
class TheStateClears(Launches):
|
||||
def test_a_relaunch_replaces_it(self):
|
||||
"""Two records for one card, and the card reads the newest: the
|
||||
failure is superseded rather than cleared."""
|
||||
first = self.run_agent(DIES)
|
||||
time.sleep(1.1) # agent ids are stamped to the second
|
||||
second = self.run_agent(LIVES)
|
||||
self.assertNotEqual(first["id"], second["id"])
|
||||
latest = max(agents.list_public(), key=lambda a: a["started"])
|
||||
self.assertEqual(latest["id"], second["id"])
|
||||
self.assertIsNone(latest["failure"])
|
||||
self.assertIsNotNone(first["failure"],
|
||||
"the older run keeps its own history")
|
||||
|
||||
def test_a_stage_move_drops_it(self):
|
||||
record = self.run_agent(DIES)
|
||||
self.assertIsNotNone(record["failure"])
|
||||
watch.narrate({"in-progress": {FILENAME}, "to-do": set()},
|
||||
{"in-progress": set(), "to-do": {FILENAME}})
|
||||
self.assertIsNone(record.get("failure"),
|
||||
"a card arriving in a new stage wears no old alarm")
|
||||
|
||||
def test_forgetting_is_per_card(self):
|
||||
record = self.run_agent(DIES)
|
||||
self.assertFalse(agents.forget_failure("99-someone-else.md"))
|
||||
self.assertIsNotNone(record["failure"], "another card's move cleared this one")
|
||||
self.assertTrue(agents.forget_failure(FILENAME))
|
||||
self.assertFalse(agents.forget_failure(FILENAME), "clearing twice is a no-op")
|
||||
|
||||
|
||||
class ExcerptTests(unittest.TestCase):
|
||||
"""What the card shows, from the log alone."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix="bench-excerpt-")).resolve()
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
|
||||
def log(self, text: str) -> str:
|
||||
path = self.tmp / "run.log"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
def test_the_tail_is_what_you_get(self):
|
||||
excerpt = agents._failure_excerpt(
|
||||
self.log("\n".join(f"line {i}" for i in range(40))), lines=6)
|
||||
self.assertIn("line 39", excerpt)
|
||||
self.assertNotIn("line 20", excerpt)
|
||||
self.assertEqual(len(excerpt.splitlines()), 6)
|
||||
|
||||
def test_a_tiny_log_survives_whole(self):
|
||||
"""The MultiEdit flag error was 91 bytes; the excerpt handles it."""
|
||||
tiny = "error: unknown option '--allowedTools MultiEdit'\n"
|
||||
self.assertIn("MultiEdit", agents._failure_excerpt(self.log(tiny)))
|
||||
|
||||
def test_hook_noise_is_stripped(self):
|
||||
excerpt = agents._failure_excerpt(
|
||||
self.log("PostToolUse hook failed with status 1\nAPI Error: 500\n"))
|
||||
self.assertEqual(excerpt, "API Error: 500")
|
||||
|
||||
def test_nothing_at_all_says_so(self):
|
||||
for empty in (self.log(""), self.log(" \n\n"), str(self.tmp / "gone.log"), None):
|
||||
self.assertIn("no output", agents._failure_excerpt(empty))
|
||||
|
||||
def test_the_headline_is_the_last_line(self):
|
||||
"""A dying process says why last."""
|
||||
self.assertEqual(agents._headline("thinking…\nAPI Error: 500"), "API Error: 500")
|
||||
self.assertEqual(agents._headline(""), "no output")
|
||||
self.assertLessEqual(len(agents._headline("x" * 400)), 120)
|
||||
|
||||
|
||||
class TheCardWearsIt(unittest.TestCase):
|
||||
"""board.html is one file with inline JS and no frontend test runner, so
|
||||
these are source-level invariants — the ones that, if broken, put the
|
||||
failure back out of sight."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.html = BOARD.read_text(encoding="utf-8")
|
||||
|
||||
def rule(self, selector: str) -> str:
|
||||
m = re.search(re.escape(selector) + r"\{([^}]*)\}", self.html)
|
||||
self.assertIsNotNone(m, f"board.html lost its {selector} rule")
|
||||
return m.group(1)
|
||||
|
||||
def test_the_border_is_the_alarm_colour(self):
|
||||
self.assertIn("--alarm", self.rule(".card.run-failed"),
|
||||
"failed is terracotta — the design system's one word for it")
|
||||
|
||||
def test_the_pill_says_run_failed(self):
|
||||
m = re.search(r"\{ text: 'run failed'[^}]*\}", self.html)
|
||||
self.assertIsNotNone(m, "the status slot lost its `run failed` pill")
|
||||
self.assertIn("--alarm", m.group(0))
|
||||
|
||||
def test_the_failure_is_scoped_to_its_card_and_stage(self):
|
||||
"""The state belongs to the most recent run on THIS card, in the
|
||||
stage it died in: no leaking sideways, none into review/."""
|
||||
m = re.search(r"function failedRun\(task\) \{(.*?)\n\}", self.html, re.DOTALL)
|
||||
self.assertIsNotNone(m, "failedRun went missing")
|
||||
body = m.group(1)
|
||||
self.assertIn("lastRunOn(task.file)", body)
|
||||
self.assertIn("'failed'", body)
|
||||
self.assertIn("failure.stage === task.stage", body)
|
||||
|
||||
def test_the_latest_run_is_a_max_not_a_find(self):
|
||||
"""Records outlive their processes; picking the first match would
|
||||
pin a card to whichever run happens to be first in the list."""
|
||||
m = re.search(r"function lastRunOn\(file\) \{(.*?)\n\}", self.html, re.DOTALL)
|
||||
self.assertIsNotNone(m, "lastRunOn went missing")
|
||||
self.assertIn("started >", m.group(1))
|
||||
|
||||
def test_the_excerpt_is_one_hover_away(self):
|
||||
"""On the card: the alarm well, the line it died on, the whole
|
||||
excerpt in the tooltip."""
|
||||
m = re.search(r'<div class="well bad" title="\$\{esc\(failure\.excerpt\)\}"',
|
||||
self.html)
|
||||
self.assertIsNotNone(m, "the card's failure well lost its excerpt tooltip")
|
||||
self.assertIn("whyFailed(failure)", self.html)
|
||||
|
||||
def test_the_card_sheet_shows_the_whole_excerpt(self):
|
||||
self.assertRegex(self.html, r"<pre>\$\{esc\(failure\.excerpt\)\}</pre>",
|
||||
"the drawer must show the excerpt without opening files")
|
||||
self.assertRegex(self.html, r"#drawer \.well\.bad pre\{[^}]*max-height",
|
||||
"a long excerpt needs a scroll bound in the sheet")
|
||||
|
||||
def test_the_server_can_toast(self):
|
||||
"""The failure toast rides the generic server-toast channel."""
|
||||
self.assertIn("msg.type === 'toast'", self.html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user