bench/tests/test_release_artifact.py

"""release.sh builds the distribution artifact from the manifest at
manager/core/release-manifest — and from nothing else. The tarball must
contain exactly what the manifest names (correct by construction: bench's
own cards, local/ content, state, tests and .claude/ were never in it),
sit at the tarball root, and unpack into a working, pristine install.

    python3 -m unittest discover -s tests
"""

import json
import os
import shutil
import socket
import subprocess
import sys
import tarfile
import tempfile
import time
import unittest
import urllib.request
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
MANIFEST = REPO / "manager" / "core" / "release-manifest"
STAMP_SOURCE = "example/bench"


def manifest_entries() -> list:
    entries = []
    for line in MANIFEST.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        kind, path = line.split(None, 1)
        entries.append((kind, path))
    return entries


def expected_files() -> set:
    """The artifact's exact file list, derived from the manifest the same
    way release.sh builds it — the tripwire for manifest drift."""
    files = set()
    for kind, path in manifest_entries():
        if kind in ("copy", "once", "seed"):
            files.add(path)
        elif kind == "keep":
            files.add(f"{path}/.gitkeep")
        elif kind == "tree":
            for p in (REPO / path).rglob("*"):
                if (p.is_file() and "__pycache__" not in p.parts
                        and p.name != ".DS_Store"):
                    files.add(p.relative_to(REPO).as_posix())
        else:
            raise AssertionError(f"unknown manifest class {kind}")
    return files


def build_artifact(out: Path) -> subprocess.CompletedProcess:
    return subprocess.run(
        ["bash", str(REPO / "release.sh"), "--tarball", str(out),
         "--source", STAMP_SOURCE],
        capture_output=True, text=True)


class ArtifactContents(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.scratch = Path(tempfile.mkdtemp(prefix="bench-artifact-")).resolve()
        cls.tarball = cls.scratch / "bench.tar.gz"
        result = build_artifact(cls.tarball)
        assert result.returncode == 0, result.stdout + result.stderr
        with tarfile.open(cls.tarball) as tar:
            cls.members = {m.name.removeprefix("./"): m
                           for m in tar.getmembers()}
        cls.files = {name for name, m in cls.members.items() if m.isfile()}

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.scratch, ignore_errors=True)

    def read_member(self, name: str) -> str:
        with tarfile.open(self.tarball) as tar:
            member = tar.extractfile(f"./{name}") or tar.extractfile(name)
            return member.read().decode("utf-8")

    def test_tarball_is_exactly_the_manifest(self):
        self.assertEqual(self.files, expected_files())

    def test_contents_sit_at_the_tarball_root(self):
        """The README one-liner pipes into `tar -xz -C .task-manager` —
        a version-named wrapper directory would land it one level deep."""
        self.assertIn("manager/core/VERSION", self.files)
        self.assertIn("start.sh", self.files)

    def test_none_of_benchs_own_state_ships(self):
        stages = ["backlog", "to-do", "in-progress", "review", "done",
                  "archive"]
        for name in self.files:
            for stage in stages:
                if name.startswith(f"tasks/{stage}/"):
                    self.assertEqual(name, f"tasks/{stage}/.gitkeep",
                                     f"a task card shipped: {name}")
            for top in ("plans/", "reference/"):
                if name.startswith(top):
                    self.assertEqual(name, f"{top}.gitkeep",
                                     f"content shipped under {top}: {name}")
            for forbidden in ("tests/", ".claude/", ".git/", ".worktrees/",
                              "manager/local/state"):
                self.assertFalse(name.startswith(forbidden),
                                 f"{forbidden} leaked into the artifact: {name}")
        self.assertNotIn("release.sh", self.files)
        self.assertNotIn("manager/local/checks", self.files)
        self.assertNotIn("manager/local/.env", self.files)

    def test_local_is_the_generated_starter_not_benchs_own(self):
        seeded = self.read_member("manager/local/CLAUDE.md")
        self.assertIn("This file is yours", seeded)
        self.assertNotEqual(
            seeded,
            (REPO / "manager" / "local" / "CLAUDE.md").read_text("utf-8"),
            "bench's own local notes must never ship")
        for sub in ("adapters", "commands", "driver", "prompts"):
            self.assertIn(f"manager/local/{sub}/.gitkeep", self.files)

    def test_shipped_update_sh_is_stamped_with_the_source(self):
        self.assertIn(f'BENCH_SOURCE_DEFAULT="{STAMP_SOURCE}"',
                      self.read_member("update.sh"))
        # The repo's own copy stays unstamped — dev clones must not
        # silently update from anywhere.
        self.assertIn('BENCH_SOURCE_DEFAULT=""',
                      (REPO / "update.sh").read_text("utf-8"))

    def test_scripts_are_executable_in_the_tarball(self):
        for name in ("start.sh", "stop.sh", "update.sh",
                     "manager/core/adapters/claude/run",
                     "manager/core/adapters/claude/wire"):
            mode = self.members[name].mode
            self.assertTrue(mode & 0o100, f"{name} lost its executable bit")


class ArtifactInstalls(unittest.TestCase):
    """Unpacking a release as .task-manager/ is the install: first boot
    has nothing to scrub, and the board serves from it."""

    @classmethod
    def setUpClass(cls):
        cls.scratch = Path(tempfile.mkdtemp(prefix="bench-install-")).resolve()
        cls.tarball = cls.scratch / "bench.tar.gz"
        result = build_artifact(cls.tarball)
        assert result.returncode == 0, result.stdout + result.stderr

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.scratch, ignore_errors=True)

    def make_install(self, name: str) -> Path:
        host = self.scratch / name
        (host / ".claude").mkdir(parents=True)
        tm = host / ".task-manager"
        tm.mkdir()
        with tarfile.open(self.tarball) as tar:
            tar.extractall(tm)
        return tm

    def test_first_boot_finds_nothing_to_clean(self):
        tm = self.make_install("pristine")
        env = {k: v for k, v in os.environ.items()
               if not k.startswith(("BOARD_", "BENCH_"))}
        result = subprocess.run(
            [sys.executable, str(tm / "install.py")],
            capture_output=True, text=True, cwd=tm.parent, env=env)
        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
        self.assertNotIn("removed", result.stdout)
        self.assertTrue((tm / "tasks" / "task-template.md").is_file())
        self.assertTrue((tm / "manager" / "local" / "state").is_dir())

    def test_board_serves_from_an_unpacked_artifact(self):
        tm = self.make_install("serving")

        probe = socket.socket()
        probe.bind(("127.0.0.1", 0))
        port = probe.getsockname()[1]
        probe.close()

        env = {k: v for k, v in os.environ.items()
               if not k.startswith(("BOARD_", "BENCH_"))}
        proc = subprocess.Popen(
            [sys.executable, str(tm / "manager" / "core" / "board.py"),
             "--port", str(port), "--no-open"],
            env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
        try:
            state = None
            for _ in range(50):
                if proc.poll() is not None:
                    break
                try:
                    with urllib.request.urlopen(
                            f"http://127.0.0.1:{port}/api/state",
                            timeout=1) as response:
                        state = json.load(response)
                    break
                except OSError:
                    time.sleep(0.2)
            if proc.poll() is not None:
                out, err = proc.communicate()
                self.fail(f"board died (rc={proc.returncode}):\n{out}\n{err}")
            self.assertIsNotNone(state, "board never answered /api/state")
            # Whatever the payload shape, the response must not mention
            # bench's own shipped cards.
            self.assertNotIn("install-ships-pristine-board",
                             json.dumps(state))
        finally:
            proc.terminate()
            proc.wait(timeout=10)
            for stream in (proc.stdout, proc.stderr):
                if stream:
                    stream.close()


if __name__ == "__main__":
    unittest.main()