diff --git a/.gitignore b/.gitignore index 48b47e5..798df6b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__/ .worktrees/ .claude/settings.local.json .DS_Store +site/dist/ diff --git a/tests/test_release_artifact.py b/tests/test_release_artifact.py index 20d752a..1f43bee 100644 --- a/tests/test_release_artifact.py +++ b/tests/test_release_artifact.py @@ -133,13 +133,25 @@ class ArtifactContents(unittest.TestCase): self.assertEqual(name, f"{top}.gitkeep", f"content shipped under {top}: {name}") for forbidden in ("tests/", ".claude/", ".git/", ".worktrees/", - "manager/local/state"): + "manager/local/state", "site/"): self.assertFalse(name.startswith(forbidden), f"{forbidden} leaked into the artifact: {name}") self.assertNotIn("release.sh", self.files) self.assertNotIn("manager/local/checks", self.files) self.assertNotIn("manager/local/.env", self.files) + def test_the_public_minisite_never_ships(self): + """site/ is bench.12vectors.com — a directory of pages generated + from this repo's own markdown, and no part of the tool anyone + installs. The manifest already excludes it by saying nothing + about it, but "correct by omission" is exactly the kind of thing + a future manifest edit undoes silently, so it is asserted.""" + leaked = sorted(n for n in self.files if n.startswith("site/")) + self.assertEqual([], leaked, "the minisite leaked into a release") + # The sweep must have reached a real tarball, or this passes on + # an empty one forever. + self.assertIn("manager/core/board.py", self.files) + def test_local_is_the_generated_starter_not_benchs_own(self): # The starter mirrors the root pair (task 13): AGENTS.md holds the # notes, CLAUDE.md is the compatibility pointer. diff --git a/tests/test_site_build.py b/tests/test_site_build.py new file mode 100644 index 0000000..cd470bf --- /dev/null +++ b/tests/test_site_build.py @@ -0,0 +1,478 @@ +"""site/build.py generates the minisite from the repo's own markdown, so +the failure that matters most is the quiet one: a section renamed in +AGENTS.md and a page that still builds, now empty. Every test here is +about that promise — the build stops, loudly, naming the route and the +heading — plus the two things the built pages owe a reader: real content +and not one third-party request. + + python3 -m unittest discover -s tests +""" + +import json +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +SITE = REPO / "site" +BUILD = SITE / "build.py" + +# The sources every page is cut from. A scratch repo needs these and +# nothing else to build the real manifest. +SOURCES = ["AGENTS.md", "README.md"] + +try: + import markdown_it # noqa: F401 + HAS_MARKDOWN_IT = True +except ImportError: # pragma: no cover - environment + HAS_MARKDOWN_IT = False + +needs_renderer = unittest.skipUnless( + HAS_MARKDOWN_IT, + "markdown-it-py is not installed: python3 -m pip install -r " + "site/requirements.txt") + + +def run_build(repo: Path, out: Path) -> subprocess.CompletedProcess: + """The build exactly as a person runs it — a real process, so the + exit code under test is the one a shell would see.""" + return subprocess.run( + [sys.executable, str(repo / "site" / "build.py"), "--out", str(out)], + capture_output=True, text=True, cwd=repo) + + +class ScratchRepo: + """A copy of site/ plus the markdown it reads, so a test can rename a + heading without touching the real AGENTS.md.""" + + def __init__(self, root: Path): + self.root = root + shutil.copytree(SITE, root / "site", + ignore=shutil.ignore_patterns("dist", "__pycache__")) + for name in SOURCES: + shutil.copy(REPO / name, root / name) + + @property + def out(self) -> Path: + return self.root / "site" / "dist" + + def manifest(self) -> dict: + return json.loads( + (self.root / "site" / "pages.json").read_text(encoding="utf-8")) + + def write_manifest(self, manifest: dict) -> None: + (self.root / "site" / "pages.json").write_text( + json.dumps(manifest, indent=2), encoding="utf-8") + + def edit(self, name: str, old: str, new: str) -> None: + path = self.root / name + text = path.read_text(encoding="utf-8") + if old not in text: # not assert: must survive python -O + raise RuntimeError(f"{name} does not contain {old!r}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + def build(self) -> subprocess.CompletedProcess: + return run_build(self.root, self.out) + + +class ScratchCase(unittest.TestCase): + def setUp(self): + root = Path(tempfile.mkdtemp(prefix="bench-site-")).resolve() + self.addCleanup(shutil.rmtree, root, True) + self.repo = ScratchRepo(root) + + +class TheRealSiteBuilds(unittest.TestCase): + """`python3 site/build.py` on a clean checkout, into a scratch + directory so the developer's own dist/ is left alone.""" + + @classmethod + def setUpClass(cls): + if not HAS_MARKDOWN_IT: + raise unittest.SkipTest("markdown-it-py is not installed") + cls.out = Path(tempfile.mkdtemp(prefix="bench-dist-")).resolve() + cls.result = run_build(REPO, cls.out) + if cls.result.returncode != 0: # not assert: must survive python -O + raise RuntimeError( + f"site/build.py failed:\n{cls.result.stdout}" + f"{cls.result.stderr}") + cls.manifest = json.loads( + (SITE / "pages.json").read_text(encoding="utf-8")) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "out"): + shutil.rmtree(cls.out, ignore_errors=True) + + def page(self, route: str) -> str: + return (self.out / route.strip("/") / "index.html").read_text("utf-8") + + def test_one_page_per_manifest_entry(self): + for entry in self.manifest["pages"]: + target = self.out / entry["path"].strip("/") / "index.html" + self.assertTrue(target.is_file(), + f'{entry["path"]} produced no page') + + def test_the_static_directory_travels_with_the_pages(self): + self.assertTrue((self.out / "static" / "site.css").is_file()) + self.assertTrue((self.out / "static" / "favicon.svg").is_file()) + + def test_both_layouts_render_real_content_from_agents_md(self): + """Not lorem: the words on the page are the words in the file.""" + home = self.page("/") + article = self.page("/concepts/claiming-a-card/") + # Sliced out of AGENTS.md's "## Stages" by the home entry. + self.assertIn("The directory a file sits in", home) + self.assertIn("a stale in-progress/ makes the board", + home) + # ...and out of "## Claiming a card" by the article entry. + self.assertIn("The first claim sticks", article) + self.assertIn("Identity is git's, so it collides like git's", + article) + self.assertIn("git config user.name", article) + + def test_the_page_title_is_not_repeated_by_the_body(self): + """The `from` heading is dropped: the layout renders the title.""" + article = self.page("/concepts/claiming-a-card/") + self.assertEqual(1, article.count("Claiming a card")) + self.assertNotIn("Claiming a card", article) + + def test_sub_headings_are_promoted_to_the_pages_own_level(self): + """AGENTS.md's `### backlog/` under `## Stages` becomes an

+ with an anchor, so the layout's contents list can reach it.""" + home = self.page("/") + self.assertIn('

backlog/

', home) + self.assertIn('

in-progress/

', home) + + def test_the_ia_comes_out_of_the_manifest(self): + article = self.page("/concepts/claiming-a-card/") + self.assertIn('href="/concepts/claiming-a-card/"', article) + self.assertIn("Concepts", article) + + def test_nothing_the_pages_load_is_third_party(self): + """The acceptance criterion, mechanised: every asset a browser + would fetch — stylesheet, icon, script, font — is same-origin, + and nothing preconnects anywhere. A is a + statement about this page, not a fetch, so it is not one of + these; anything that makes the browser open a connection is.""" + fetching = {"stylesheet", "icon", "shortcut icon", "preload", + "prefetch", "preconnect", "dns-prefetch", "manifest", + "modulepreload"} + for entry in self.manifest["pages"]: + html = self.page(entry["path"]) + for tag in re.findall(r"]*>", html, re.IGNORECASE): + rel = re.search(r'rel=["\']([^"\']+)', tag, re.IGNORECASE) + href = re.search(r'href=["\']([^"\']+)', tag, re.IGNORECASE) + if not rel or rel.group(1).lower() not in fetching: + continue + self.assertTrue( + href and href.group(1).startswith("/"), + f'{entry["path"]} loads {tag} from somewhere else') + for url in re.findall( + r"""<(?:script|img|iframe)\b[^>]*\bsrc=["']([^"']+)""", + html, re.IGNORECASE): + self.assertTrue( + url.startswith("/"), + f'{entry["path"]} loads {url} from somewhere else') + + css = (self.out / "static" / "site.css").read_text("utf-8") + self.assertNotIn("@import", css) + for url in re.findall(r"""url\(\s*["']?([^"')]+)""", css): + self.assertTrue(url.startswith("/static/"), + f"site.css fetches {url} from somewhere else") + + def test_the_stylesheet_carries_the_boards_four_state_tokens(self): + """The site and the board are one system, so colour is named the + same in both — see manager/core/board.html's :root.""" + css = (self.out / "static" / "site.css").read_text("utf-8") + for token in ("--accent:", "--calm:", "--alarm:", "--idle:"): + self.assertIn(token, css) + + def test_the_missing_fonts_are_named_rather_than_assumed(self): + """A woff2 the stylesheet wants and the build does not have is a + warning on stderr, never a silent fallback — and never a fetch + from a font CDN.""" + wanted = re.findall(r"""url\(\s*["']?(/static/fonts/[^"')]+)""", + (self.out / "static" / "site.css").read_text("utf-8")) + self.assertTrue(wanted, "the stylesheet self-hosts nothing") + for url in wanted: + present = (self.out / url.lstrip("/")).exists() + self.assertEqual(present, f"warning: {url}" not in self.result.stderr, + f"{url}: present on disk and warned about, or " + f"absent and silent") + + +@needs_renderer +class DriftStopsTheBuild(ScratchCase): + """The whole reason the site is generated: a heading that moved must + break the build, not empty a page.""" + + def test_a_renamed_heading_names_the_route_and_the_heading(self): + self.repo.edit("AGENTS.md", "## Claiming a card", + "## Claiming a task card") + result = self.repo.build() + + self.assertNotEqual(result.returncode, 0, + "a renamed heading built cleanly") + self.assertIn("/concepts/claiming-a-card/", result.stderr) + self.assertIn("AGENTS.md", result.stderr) + self.assertIn("## Claiming a card", result.stderr) + + def test_a_renamed_heading_emits_no_page_at_all(self): + """Not "a page with an empty body" — nothing is written. Every + page renders before the output directory is touched, so a build + that drifts leaves whatever was there alone rather than replacing + a working site with a broken one.""" + self.repo.edit("AGENTS.md", "## Claiming a card", + "## Claiming a task card") + self.repo.build() + self.assertFalse(self.repo.out.exists(), + "a failed build wrote pages anyway") + + def test_a_drifting_rebuild_leaves_the_last_good_build_standing(self): + self.assertEqual(0, self.repo.build().returncode) + good = (self.repo.out / "concepts" / "claiming-a-card" + / "index.html").read_text("utf-8") + self.repo.edit("AGENTS.md", "## Claiming a card", + "## Claiming a task card") + self.assertNotEqual(0, self.repo.build().returncode) + self.assertEqual(good, (self.repo.out / "concepts" + / "claiming-a-card" + / "index.html").read_text("utf-8")) + + def test_a_renamed_to_heading_is_caught_too(self): + self.repo.edit("AGENTS.md", "## Syncing boards", "## Board sync") + result = self.repo.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("## Syncing boards", result.stderr) + self.assertIn("/concepts/claiming-a-card/", result.stderr) + + def test_a_missing_source_file_is_readable(self): + (self.repo.root / "AGENTS.md").unlink() + result = self.repo.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("AGENTS.md", result.stderr) + self.assertIn("does not exist", result.stderr) + + def test_a_section_emptied_to_its_heading_fails(self): + """The subtler drift: the heading survives, its content moves + elsewhere. An empty page is a drift, not a page.""" + manifest = self.repo.manifest() + manifest["pages"] = [{ + "path": "/hollow/", "title": "Hollow", "layout": "article", + "section": "Concepts", "source": "AGENTS.md", + "from": "## Empty", "to": "## After", + }] + self.repo.write_manifest(manifest) + self.repo.edit("AGENTS.md", "## Claiming a card", + "## Empty\n\n## After\n\n## Claiming a card") + result = self.repo.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("empty", result.stderr) + self.assertIn("/hollow/", result.stderr) + + def test_a_heading_inside_a_code_fence_is_not_a_heading(self): + """AGENTS.md fences a task file template starting `# Task title`. + Matching that would slice the document in half.""" + manifest = self.repo.manifest() + manifest["pages"] = [{ + "path": "/fenced/", "title": "Fenced", "layout": "article", + "section": "Concepts", "source": "AGENTS.md", + "from": "# Task title", + }] + self.repo.write_manifest(manifest) + result = self.repo.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("# Task title", result.stderr) + + +@needs_renderer +class LinksComeOutWorking(ScratchCase): + """Markdown that links between repo files is a dead path on the web.""" + + def build_one(self, body: str, *, link_routes=None): + (self.repo.root / "SOURCE.md").write_text( + f"# Doc\n\n## Section\n\n{body}\n", encoding="utf-8") + manifest = self.repo.manifest() + manifest["link_routes"] = link_routes or {} + manifest["pages"] = [{ + "path": "/linked/", "title": "Linked", "layout": "article", + "section": "Concepts", "source": "SOURCE.md", + "from": "## Section", + }] + self.repo.write_manifest(manifest) + result = self.repo.build() + page = self.repo.out / "linked" / "index.html" + return result, page.read_text("utf-8") if page.exists() else "" + + def test_a_repo_relative_link_becomes_a_github_link(self): + result, html = self.build_one("See [the brief](AGENTS.md).") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + 'href="https://github.com/12vectors/bench/blob/main/AGENTS.md"', + html) + + def test_link_routes_win_over_github(self): + result, html = self.build_one( + "See [the brief](AGENTS.md#stages).", + link_routes={"AGENTS.md": "/concepts/stages/"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('href="/concepts/stages/#stages"', html) + + def test_an_absolute_link_and_an_anchor_are_left_alone(self): + result, html = self.build_one( + "[out](https://example.com/x) and [here](#section).") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('href="https://example.com/x"', html) + self.assertIn('href="#section"', html) + + def test_a_dead_relative_link_stops_the_build(self): + result, _ = self.build_one("See [gone](docs/gone.md).") + self.assertNotEqual(result.returncode, 0) + self.assertIn("/linked/", result.stderr) + self.assertIn("docs/gone.md", result.stderr) + + def test_a_link_escaping_the_repo_stops_the_build(self): + result, _ = self.build_one("See [up](../../etc/passwd).") + self.assertNotEqual(result.returncode, 0) + self.assertIn("outside the repository", result.stderr) + + +@needs_renderer +class TheArticleGutterFollowsTheBody(ScratchCase): + """"On this page" is not authored anywhere: it is the promoted body's + own h2s, which is why a section that grows a sub-heading grows a + contents entry without anyone editing the site.""" + + def test_promoted_sub_headings_become_the_contents_list(self): + manifest = self.repo.manifest() + manifest["pages"] = [{ + "path": "/concepts/team-mode/", "title": "Syncing boards", + "layout": "article", "section": "Concepts", + "source": "AGENTS.md", "from": "## Syncing boards", + "to": "## Task file format", + }] + self.repo.write_manifest(manifest) + result = self.repo.build() + self.assertEqual(result.returncode, 0, result.stderr) + + html = (self.repo.out / "concepts" / "team-mode" + / "index.html").read_text("utf-8") + # AGENTS.md's "### State syncs; reactions don't" under "## Syncing + # boards" — promoted to an

, anchored, and listed. + self.assertIn('

', html) + self.assertIn('href="#state-syncs-reactions-don-t"', html) + self.assertIn("On this page", html) + + +@needs_renderer +class TheManifestIsChecked(ScratchCase): + """pages.json is the only place the IA is written down, so a mistake + in it has to be a build failure rather than a surprise on the site.""" + + def only(self, page: dict): + manifest = self.repo.manifest() + manifest["pages"] = [page] + self.repo.write_manifest(manifest) + return self.repo.build() + + def test_an_unknown_layout_lists_the_ones_that_exist(self): + result = self.only({ + "path": "/x/", "title": "X", "layout": "logbook", + "source": None}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("logbook", result.stderr) + self.assertIn("article", result.stderr) + self.assertIn("home", result.stderr) + + def test_a_source_without_a_from_heading_is_refused(self): + result = self.only({ + "path": "/x/", "title": "X", "layout": "article", + "source": "AGENTS.md"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("from", result.stderr) + + def test_an_authored_page_says_so_with_a_null_source(self): + """The landing page 33 will write has no slice and no drift.""" + result = self.only({ + "path": "/", "title": "bench", "layout": "home", + "source": None}) + self.assertEqual(result.returncode, 0, result.stderr) + html = (self.repo.out / "index.html").read_text("utf-8") + self.assertIn("Put the agents", html) + + def test_a_route_must_be_a_directory_path(self): + result = self.only({ + "path": "/concepts/claiming", "title": "X", "layout": "article", + "source": None}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("/concepts/claiming", result.stderr) + + def test_two_entries_cannot_claim_one_route(self): + manifest = self.repo.manifest() + manifest["pages"] = [ + {"path": "/x/", "title": "X", "layout": "article", "source": None}, + {"path": "/x/", "title": "Y", "layout": "article", "source": None}, + ] + self.repo.write_manifest(manifest) + result = self.repo.build() + self.assertNotEqual(result.returncode, 0) + self.assertIn("twice", result.stderr) + + +@needs_renderer +class TheOutputDirectoryIsOurs(ScratchCase): + """--out is a path a person types, so it gets one guard: the builder + empties directories it wrote and refuses anything else.""" + + def test_a_rebuild_replaces_the_previous_build(self): + self.assertEqual(0, self.repo.build().returncode) + stale = self.repo.out / "gone" / "index.html" + stale.parent.mkdir(parents=True) + stale.write_text("stale", encoding="utf-8") + self.assertEqual(0, self.repo.build().returncode) + self.assertFalse(stale.exists()) + + def test_a_directory_it_did_not_write_is_refused(self): + theirs = self.repo.root / "not-ours" + theirs.mkdir() + (theirs / "important.txt").write_text("mine", encoding="utf-8") + result = run_build(self.repo.root, theirs) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Refusing", result.stderr) + self.assertTrue((theirs / "important.txt").exists()) + + +class TheRepositoryKnowsAboutSite(unittest.TestCase): + """The two facts about site/ that live outside site/.""" + + def test_dist_is_gitignored_and_the_sources_are_not(self): + ignored = subprocess.run( + ["git", "check-ignore", "site/dist/index.html"], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(ignored.returncode, 0, + "site/dist/ is not gitignored") + tracked = subprocess.run( + ["git", "check-ignore", "site/build.py", "site/pages.json"], + capture_output=True, text=True, cwd=REPO) + self.assertNotEqual(tracked.returncode, 0, + "site/ sources are gitignored") + + def test_the_release_manifest_does_not_mention_site(self): + """Belt to test_release_artifact.py's braces: nothing in the + manifest names site/, so nothing can ship it.""" + manifest = (REPO / "manager" / "core" / "release-manifest" + ).read_text("utf-8") + for line in manifest.splitlines(): + if line.strip().startswith("#") or not line.strip(): + continue + self.assertNotIn("site", line.split(None, 1)[1], + f"the release manifest names site/: {line}") + + +if __name__ == "__main__": + unittest.main()