Let the project define the Focus checks, resolved like prompts

What counts as a definition-of-done check was the origin project's
stack (pytest / lint-imports / frontend) frozen into core: emit.py
classified against inline literals and the Focus panel carried three
fixed rows with duplicated regexes. Per the three-layer law that is
project knowledge, so it now lives in one file: core/checks ships the
old rows as the default, and a checks file in manager/local/ replaces
it wholesale — the same filename-wins resolution as prompts.

- core/checks: '<label>: <command regex>' per line, self-documenting;
  read fresh on every use.
- emit.py classifies Bash commands against the resolved file (kind
  'check', the label carried into the summary) and judges pass/fail
  generically — counted results, broken totals, OK/FAILED verdict
  lines — since the hook payload carries no exit status. The runtime
  moved under a __main__ guard so the classifier is importable.
- config.checks() mirrors the parser (the bridge stays standalone) and
  httpd serves it in /api/state; the Focus panel renders one row per
  served entry, matching events with the same patterns — no fixed
  rows, no duplicated regexes.
- manager/local/checks gives bench its real definition (unittest), so
  the self-hosted board shows a check that can actually run.
- The default BOARD_AGENT_COMMANDS drops its pytest prefix: core no
  longer names any stack outside the shipped checks default, and a
  test walks manager/core to keep it that way.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
istos
2026-07-30 07:15:54 +02:00
co-authored by Claude Fable 5
parent a9158c4132
commit 7df286ba2e
8 changed files with 435 additions and 59 deletions
+6 -1
View File
@@ -21,7 +21,12 @@ BOARD_OPENCODE_BIN=opencode
# git/gh grants per launch intent are built in). Headless runs have no
# human at a permission prompt, so a test runner missing from this list
# is a test the work agent cannot run.
BOARD_AGENT_COMMANDS=python3 -m unittest,python3 -m pytest
BOARD_AGENT_COMMANDS=python3 -m unittest
# What counts as a definition-of-done check (the Focus view's CHECKS
# panel) is a file, not an env var: core/checks ships a generic default,
# and a `checks` file in manager/local/ replaces it wholesale. Format
# (`<label>: <command regex>`) is documented in the default file itself.
# Where work-agent worktrees are created, relative to the repo root.
BOARD_WORKTREES=.worktrees
+93 -46
View File
@@ -25,6 +25,10 @@ from pathlib import Path
EDIT_TOOLS = {"Edit", "Write", "MultiEdit", "NotebookEdit"}
# manager/ — the same distance up from core/adapters/claude/ as from a
# local/adapters/claude/ override, so both copies resolve the same files.
MANAGER = Path(__file__).resolve().parents[3]
def _txt(value, cap=600):
return value[:cap] if isinstance(value, str) else ""
@@ -51,6 +55,56 @@ def _resp_text(resp):
return ""
def check_defs():
"""The project's definition-of-done checks: `<label>: <command regex>`
per line, local/checks replacing core/checks wholesale — the same file
the board serves to the Focus panel, read here for classification so
the two never drift. (Core's config.py mirrors this parser; the bridge
stays standalone.) Read fresh per event; must never raise."""
for base in (MANAGER / "local", MANAGER / "core"):
try:
text = (base / "checks").read_text(encoding="utf-8")
except OSError:
continue
defs = []
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
label, sep, pattern = line.partition(":")
label, pattern = label.strip(), pattern.strip()
if not sep or not label or not pattern:
continue
try:
defs.append((label, re.compile(pattern)))
except re.error:
continue
return defs
return []
def judge(out):
"""Generic pass/fail from a check's output — the hook payload carries
no exit status (stdout/stderr/interrupted only), so judgment rests on
the summaries test tools print: counted results ('3 passed',
'1 failed', '2 errors'), lint-style broken totals, and OK/FAILED
verdict lines. Returns (ok, summary bit); (None, '') when the output
says nothing recognizable either way."""
failed = re.search(r"\b\d+ (?:failed|errors?)\b", out)
passed = re.search(r"\b\d+ passed\b", out)
if failed:
return False, failed.group(0) + (f", {passed.group(0)}" if passed else "")
if passed:
return True, passed.group(0)
broken = re.search(r"\b(\d+) broken\b", out)
if broken:
return broken.group(1) == "0", broken.group(0)
verdict = re.search(r"^(OK|FAILED)\b.*", out, re.M)
if verdict:
return verdict.group(1) == "OK", verdict.group(0)[:60]
return None, ""
def classify(hook, tool, tool_input, resp):
if hook == "SessionStart":
return {"kind": "session", "summary": "session started"}
@@ -88,31 +142,19 @@ def classify(hook, tool, tool_input, resp):
cmd = _txt(tool_input.get("command"), 240)
running = hook == "PreToolUse"
out = "" if running else _resp_text(resp)[:1200]
kind, ok = "command", None
if re.search(r"\bpytest\b", cmd):
kind = "test"
elif "lint-imports" in cmd or re.search(r"type-check|vue-tsc|\bnpm (run )?test\b|\bvitest\b", cmd):
kind = "check"
elif re.match(r"\s*git (commit|add|push|checkout|switch|merge|worktree)", cmd):
kind, ok, label = "command", None, None
for name, pattern in check_defs():
if pattern.search(cmd):
kind, label = "check", name
break
if kind == "command" and re.match(r"\s*git (commit|add|push|checkout|switch|merge|worktree)", cmd):
kind = "git"
if running:
summary = f"running: {cmd[:90]}"
elif kind == "test":
passed = re.search(r"(\d+) passed", out)
failed = re.search(r"(\d+) failed", out) or re.search(r"(\d+) error", out)
if failed:
ok = False
summary = f"pytest — {failed.group(0)}" + (f", {passed.group(0)}" if passed else "")
elif passed:
ok = True
summary = f"pytest — {passed.group(0)}"
else:
summary = f"ran: {cmd[:90]}"
elif kind == "check":
if "broken" in out:
ok = not re.search(r"[1-9]\d* broken", out)
summary = f"ran: {cmd[:90]}"
ok, bits = judge(out)
summary = f"{label}{bits}" if bits else f"ran: {cmd[:90]}"
elif kind == "git":
m = re.search(r"""-m ["']([^"']{1,90})""", cmd)
summary = f"git: {m.group(1) if m else cmd[:80]}"
@@ -129,7 +171,7 @@ def board_port():
if port:
return port
try:
env_file = Path(__file__).resolve().parents[3] / "local" / ".env"
env_file = MANAGER / "local" / ".env"
for line in env_file.read_text().splitlines():
key, _, value = line.strip().partition("=")
if key.strip() == "BOARD_PORT":
@@ -139,31 +181,36 @@ def board_port():
return port or "26071"
try:
payload = json.load(sys.stdin)
except Exception:
payload = {}
def main():
try:
payload = json.load(sys.stdin)
except Exception:
payload = {}
tool_input = payload.get("tool_input")
event = {
"v": 1,
"session": payload.get("session_id") or "unknown",
"agent": os.environ.get("BOARD_AGENT_ID"),
"task": os.environ.get("BOARD_TASK"),
**classify(payload.get("hook_event_name") or "?",
payload.get("tool_name") or "",
tool_input if isinstance(tool_input, dict) else {},
payload.get("tool_response")),
}
tool_input = payload.get("tool_input")
event = {
"v": 1,
"session": payload.get("session_id") or "unknown",
"agent": os.environ.get("BOARD_AGENT_ID"),
"task": os.environ.get("BOARD_TASK"),
**classify(payload.get("hook_event_name") or "?",
payload.get("tool_name") or "",
tool_input if isinstance(tool_input, dict) else {},
payload.get("tool_response")),
}
try:
request = urllib.request.Request(
f"http://127.0.0.1:{board_port()}/api/events",
data=json.dumps(event).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
urllib.request.urlopen(request, timeout=1).read()
except Exception:
pass
try:
request = urllib.request.Request(
f"http://127.0.0.1:{board_port()}/api/events",
data=json.dumps(event).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
urllib.request.urlopen(request, timeout=1).read()
except Exception:
pass
sys.exit(0)
sys.exit(0)
if __name__ == "__main__":
main()
+18 -10
View File
@@ -1295,7 +1295,7 @@ function renderFlight() {
}
const events = S.events[sid] || [];
const files = new Set(events.filter(e => e.file && e.kind === 'edit').map(e => e.file));
const tests = events.filter(e => e.kind === 'test').length;
const checks = events.filter(e => e.kind === 'test' || e.kind === 'check').length;
const agent = agentFor(sid);
const stopBtn = agent && agent.status === 'running'
? `<button id="stopagent" class="stopbtn" data-aid="${esc(agent.id)}">Hold</button>` : '';
@@ -1305,7 +1305,7 @@ function renderFlight() {
`<span class="sid">${esc(sid.slice(0, 8))}</span></div>` +
`<div class="s-line">${meta.task ? 'on ' + esc(meta.task) + ' · ' : ''}` +
`started ${fmtShort(meta.started)} · ${meta.count || 0} events · ` +
`${files.size} files edited · ${tests} test runs${branch}</div></div>` +
`${files.size} files edited · ${checks} check runs${branch}</div></div>` +
stopBtn + spark(events, meta);
const stop = $('#stopagent');
if (stop) stop.addEventListener('click', () => stopAgent(stop.dataset.aid));
@@ -1504,19 +1504,27 @@ function renderFocus() {
steps + act + `</div>`;
const rev = [...events].reverse();
const lastTest = rev.find(e => e.kind === 'test' && !e.running);
const lastLint = rev.find(e => e.kind === 'check' && (e.cmd || '').includes('lint-imports'));
const lastFront = rev.find(e => e.kind === 'check' && /type-check|vitest|npm/.test(e.cmd || ''));
const checkRow = (name, ev) => {
if (!ev) return `<div class="check-row none"><span class="glyph">—</span><span class="name">${name}</span><span class="state">not run</span></div>`;
if (!ev) return `<div class="check-row none"><span class="glyph">—</span><span class="name">${esc(name)}</span><span class="state">not run</span></div>`;
const cls = ev.ok === false ? 'fail' : ev.ok === true ? 'pass' : 'none';
const glyph = ev.ok === false ? '✕' : ev.ok === true ? '✓' : '·';
const state = esc(ev.summary.replace(/^pytest — /, '').replace(/^ran: /, '')) + ' · ' + fmtShort(ev.ts);
return `<div class="check-row ${cls}"><span class="glyph">${glyph}</span><span class="name">${name}</span><span class="state">${state}</span></div>`;
const text = ev.summary.startsWith(name + ' ')
? ev.summary.slice(name.length + 3) : ev.summary.replace(/^ran: /, '');
const state = esc(text) + ' · ' + fmtShort(ev.ts);
return `<div class="check-row ${cls}"><span class="glyph">${glyph}</span><span class="name">${esc(name)}</span><span class="state">${state}</span></div>`;
};
// One row per project-defined check — labels and command patterns come
// from the served checks definition (local/checks over core/checks),
// the same file the adapter classifies against.
const checkRows = (S.state?.checks || []).map(c => {
let re = null;
try { re = new RegExp(c.pattern); } catch { /* skip unparseable */ }
const ev = re && rev.find(e => !e.running && e.cmd && re.test(e.cmd));
return checkRow(c.label, ev);
}).join('');
const checksPanel = `<div class="panel"><div class="phead"><span class="label">Checks</span></div>` +
checkRow('pytest', lastTest) + checkRow('lint-imports', lastLint) +
checkRow('frontend', lastFront) + `</div>`;
(checkRows || `<div class="check-row none"><span class="glyph">—</span><span class="name">no checks defined</span><span class="state"></span></div>`) +
`</div>`;
let fileRows = '', fileHead = '';
const diff = agent && S.diffCache[agent.id];
+18
View File
@@ -0,0 +1,18 @@
# Definition-of-done checks — the project half of the Focus view's
# CHECKS panel. One check per line:
#
# <label>: <regex matched against the commands agents run>
#
# The board never runs anything: the agent adapter classifies each
# command an agent runs against these patterns, and Focus shows the last
# matching run per row, judged pass/fail from the command's own output.
#
# Core ships this default; a file named `checks` in manager/local/
# replaces it WHOLESALE (same resolution as prompts — the local file
# wins by filename). The adapter and the browser read the same file, so
# a label edited here flows to classification and rendering alike; keep
# patterns in the regex dialect Python and JavaScript share. Read fresh
# on every event and request — edits apply without a restart.
pytest: \bpytest\b
lint-imports: \blint-imports\b
frontend: type-check|vue-tsc|\bnpm (run )?test\b|\bvitest\b
+32 -2
View File
@@ -9,6 +9,7 @@ things are* lives here. No state, no behaviour.
from __future__ import annotations
import os
import re
import subprocess
from pathlib import Path
@@ -92,8 +93,7 @@ ADAPTER = setting("BOARD_AGENT_ADAPTER", "claude")
# test/check commands) — neutral, comma-separated; each adapter renders
# them in its own permission-rule syntax. The universal git/gh grants are
# the adapter's own knowledge; this list is the project's half.
AGENT_COMMANDS = setting("BOARD_AGENT_COMMANDS",
"python3 -m unittest,python3 -m pytest")
AGENT_COMMANDS = setting("BOARD_AGENT_COMMANDS", "python3 -m unittest")
# GitHub plumbing: the gh CLI (stub-able for tests) and the git remote PRs
# go to. Empty remote = auto-detect the first remote; no remote = no PRs.
@@ -115,6 +115,36 @@ def prompt(name: str) -> str:
return (CORE / "prompts" / name).read_text(encoding="utf-8")
def checks() -> list[dict]:
"""Definition-of-done checks for the Focus panel: core ships a default
(core/checks); a local/checks replaces it wholesale, like prompts. Each
line is `<label>: <command regex>`; invalid regexes are skipped. The
agent adapter reads the same file to classify commands (the claude
adapter's emit.py is standalone, so the parser is mirrored there), and
the browser matches with the served patterns keep them in the regex
dialect Python and JavaScript share. Read fresh on every request."""
for base in (LOCAL, CORE):
path = base / "checks"
if not path.is_file():
continue
entries = []
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
label, sep, pattern = line.partition(":")
label, pattern = label.strip(), pattern.strip()
if not sep or not label or not pattern:
continue
try:
re.compile(pattern)
except re.error:
continue
entries.append({"label": label, "pattern": pattern})
return entries
return []
def adapter_dir() -> Path | None:
"""The configured agent adapter's directory — local overrides core."""
for base in (LOCAL / "adapters", CORE / "adapters"):
+1
View File
@@ -35,6 +35,7 @@ def state_payload() -> dict:
"branches": github.task_branches(),
"commands": config.commands(),
"commandRuns": commands.public(),
"checks": config.checks(),
"archivedCount": taskfiles.archived_count(),
"boardEvents": board_events,
"now": time.time(),
+4
View File
@@ -0,0 +1,4 @@
# Bench's own definition of done: the stdlib test suite, nothing else —
# no lint step, no frontend build. Replaces core/checks wholesale; format
# documented there.
unittest: \bunittest\b
+263
View File
@@ -0,0 +1,263 @@
"""Project-owned definition-of-done checks (task 03).
What counts as a check is project knowledge: core ships a default
definition (core/checks), a same-named file in local/ replaces it
wholesale, the claude adapter classifies agent commands against the
resolved file, and the Focus panel renders one row per entry from the
same definition served over /api/state. These tests pin the resolution
order, the two parsers' agreement, the generic pass/fail judgment that
replaced per-tool output parsing, and the absence of the origin
project's stack anywhere else in core.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import re
import sys
import tempfile
import unittest
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "manager" / "core"))
sys.path.insert(0, str(REPO / "manager" / "core" / "adapters" / "claude"))
import config # noqa: E402
import emit # noqa: E402
CORE = REPO / "manager" / "core"
class ShippedDefault(unittest.TestCase):
"""A project defining nothing sees today's rows — as the default."""
def test_default_rows_and_patterns(self):
original = config.LOCAL
try:
with tempfile.TemporaryDirectory() as tmp:
config.LOCAL = Path(tmp) # no local/checks
checks = config.checks()
finally:
config.LOCAL = original
self.assertEqual([c["label"] for c in checks],
["pytest", "lint-imports", "frontend"])
by_label = {c["label"]: c["pattern"] for c in checks}
self.assertTrue(re.search(by_label["pytest"], "python3 -m pytest -q"))
self.assertTrue(re.search(by_label["lint-imports"], "lint-imports"))
for cmd in ("npm run test", "vue-tsc --noEmit", "npx vitest run"):
self.assertTrue(re.search(by_label["frontend"], cmd), cmd)
self.assertFalse(re.search(by_label["pytest"], "python3 -m unittest"))
class LocalOverride(unittest.TestCase):
"""local/checks beats core/checks wholesale, like prompts."""
def _with_local(self, text):
original = config.LOCAL
try:
with tempfile.TemporaryDirectory() as tmp:
config.LOCAL = Path(tmp)
(Path(tmp) / "checks").write_text(text, encoding="utf-8")
return config.checks()
finally:
config.LOCAL = original
def test_local_definition_wins(self):
checks = self._with_local("# ours\nsmoke: \\bmake smoke\\b\n"
"types: \\bmypy\\b\n")
self.assertEqual([c["label"] for c in checks], ["smoke", "types"])
def test_replacement_is_wholesale_even_when_empty(self):
self.assertEqual(self._with_local("# nothing to check\n"), [])
def test_malformed_lines_are_skipped(self):
checks = self._with_local("no separator here\n"
": pattern without label\n"
"label without pattern:\n"
"bad-regex: [unclosed\n"
"good: \\bok\\b\n")
self.assertEqual([c["label"] for c in checks], ["good"])
def test_bench_defines_its_own(self):
"""Bench's real local/checks: the stdlib suite, so the self-hosted
Focus view shows a check that can actually run here."""
checks = config.checks()
self.assertEqual([c["label"] for c in checks], ["unittest"])
self.assertTrue(re.search(checks[0]["pattern"], "python3 -m unittest"))
class AdapterReadsTheSameFile(unittest.TestCase):
"""emit.py resolves and parses the identical definition, so a label
edit flows to classification and rendering alike."""
def test_parsers_agree_on_the_shipped_default(self):
original = emit.MANAGER
try:
emit.MANAGER = Path(tempfile.mkdtemp()) # no local, no core
(emit.MANAGER / "core").mkdir()
(emit.MANAGER / "core" / "checks").write_text(
(CORE / "checks").read_text(encoding="utf-8"), encoding="utf-8")
adapter_view = [(label, pattern.pattern)
for label, pattern in emit.check_defs()]
finally:
emit.MANAGER = original
core_original = config.LOCAL
try:
with tempfile.TemporaryDirectory() as tmp:
config.LOCAL = Path(tmp)
board_view = [(c["label"], c["pattern"]) for c in config.checks()]
finally:
config.LOCAL = core_original
self.assertEqual(adapter_view, board_view)
def test_local_wins_in_the_adapter_too(self):
original = emit.MANAGER
try:
with tempfile.TemporaryDirectory() as tmp:
emit.MANAGER = Path(tmp)
(Path(tmp) / "core").mkdir()
(Path(tmp) / "core" / "checks").write_text(
"core-only: \\bx\\b\n", encoding="utf-8")
(Path(tmp) / "local").mkdir()
(Path(tmp) / "local" / "checks").write_text(
"ours: \\bmake check\\b\n", encoding="utf-8")
defs = emit.check_defs()
finally:
emit.MANAGER = original
self.assertEqual([label for label, _ in defs], ["ours"])
def test_missing_files_mean_no_checks_not_a_crash(self):
original = emit.MANAGER
try:
with tempfile.TemporaryDirectory() as tmp:
emit.MANAGER = Path(tmp)
self.assertEqual(emit.check_defs(), [])
finally:
emit.MANAGER = original
class Classification(unittest.TestCase):
"""Bash commands classify against the resolved definitions; the label
carries into the summary; judgment is generic, not per-tool."""
def setUp(self):
self._original = emit.MANAGER
self._tmp = tempfile.TemporaryDirectory()
emit.MANAGER = Path(self._tmp.name)
(emit.MANAGER / "core").mkdir()
(emit.MANAGER / "core" / "checks").write_text(
"suite: \\bunittest\\b\nlint: \\bmake lint\\b\n", encoding="utf-8")
def tearDown(self):
emit.MANAGER = self._original
self._tmp.cleanup()
def _bash(self, cmd, out="", hook="PostToolUse"):
return emit.classify(hook, "Bash", {"command": cmd},
{"stdout": out, "stderr": ""})
def test_matching_command_becomes_a_check_with_its_label(self):
ev = self._bash("python3 -m unittest discover -s tests",
"Ran 7 tests in 0.1s\n\nOK")
self.assertEqual(ev["kind"], "check")
self.assertTrue(ev["ok"])
self.assertTrue(ev["summary"].startswith("suite — "))
def test_label_edits_flow_to_the_event(self):
(emit.MANAGER / "core" / "checks").write_text(
"renamed: \\bunittest\\b\n", encoding="utf-8")
ev = self._bash("python3 -m unittest", "OK")
self.assertTrue(ev["summary"].startswith("renamed — "))
def test_counted_failures_fail(self):
ev = self._bash("python3 -m unittest", "2 failed, 5 passed in 1.2s")
self.assertEqual(ev["kind"], "check")
self.assertFalse(ev["ok"])
self.assertIn("2 failed", ev["summary"])
self.assertIn("5 passed", ev["summary"])
def test_unjudgeable_output_stays_neutral(self):
ev = self._bash("make lint", "some chatter")
self.assertEqual(ev["kind"], "check")
self.assertIsNone(ev["ok"])
def test_unmatched_commands_stay_commands(self):
self.assertEqual(self._bash("ls -la")["kind"], "command")
def test_git_classification_survives(self):
ev = self._bash('git commit -m "a message"')
self.assertEqual(ev["kind"], "git")
def test_running_events_keep_the_check_kind(self):
ev = self._bash("python3 -m unittest", hook="PreToolUse")
self.assertEqual(ev["kind"], "check")
self.assertTrue(ev["running"])
class GenericJudgment(unittest.TestCase):
"""No tool names: counts, broken totals and OK/FAILED verdict lines."""
def test_ladder(self):
cases = [
("3 passed in 0.5s", True),
("1 failed, 2 passed", False),
("2 errors", False),
("Ran 7 tests in 0.1s\n\nOK", True),
("Ran 7 tests in 0.1s\n\nFAILED (failures=1)", False),
("0 broken contracts", True),
("4 broken contracts", False),
("nothing recognizable", None),
("", None),
]
for out, expected in cases:
ok, _ = emit.judge(out)
self.assertEqual(ok, expected, f"judge({out!r})")
class ServedToTheBrowser(unittest.TestCase):
"""The state API carries the definition; Focus renders from it with
no fixed rows and no duplicated patterns."""
@classmethod
def setUpClass(cls):
cls.html = (CORE / "board.html").read_text(encoding="utf-8")
def test_state_payload_includes_checks(self):
import httpd
payload = httpd.state_payload()
self.assertIn("checks", payload)
self.assertEqual(payload["checks"], config.checks())
def test_focus_renders_from_served_definition(self):
self.assertIn("S.state?.checks", self.html)
self.assertIn("new RegExp(c.pattern)", self.html)
self.assertIn("no checks defined", self.html)
def test_no_fixed_rows_or_duplicated_patterns(self):
for fossil in ("lint-imports", "vitest", "vue-tsc", "pytest"):
self.assertNotIn(fossil, self.html, f"board.html still hardcodes {fossil}")
class NoFossilsInCore(unittest.TestCase):
"""Nothing in manager/core/ names the origin project's stack outside
the shipped default checks definition."""
def test_core_is_clean(self):
allowed = CORE / "checks"
pattern = re.compile(r"pytest|lint-imports|vue-tsc|vitest")
for path in sorted(CORE.rglob("*")):
if not path.is_file() or path == allowed:
continue
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
continue
self.assertIsNone(pattern.search(text),
f"{path.relative_to(REPO)} names a stack "
"that belongs in the checks definition")
if __name__ == "__main__":
unittest.main()