bench/tests/test_site_deploy.py

"""bench.12vectors.com is a Cloudflare Worker serving site/dist/ as
static assets. What can be tested here is everything the deploy depends
on *before* it reaches Cloudflare: that the config says what the site
needs it to say, that the build produces the files that config names,
and that the three of them — wrangler.jsonc, pages.json, README.md —
cannot drift apart about which domain this is.

What cannot be tested here is a live response. `not_found_handling` is
asserted as configuration plus the 404.html it points at; that a request
to a missing path comes back 404 is Cloudflare's half, and the deploy
checklist in site/README.md is where a person confirms it.

    python3 -m unittest discover -s tests
"""

import json
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from urllib.parse import urlparse

REPO = Path(__file__).resolve().parents[1]
SITE = REPO / "site"
WRANGLER = SITE / "wrangler.jsonc"
HEADERS = SITE / "root" / "_headers"
README = SITE / "README.md"

# Set on every response, whatever it is. The task's three, plus the two
# that come free with them.
BASELINE = ("X-Content-Type-Options", "Referrer-Policy",
            "Strict-Transport-Security")

try:
    import markdown_it  # noqa: F401
    HAS_MARKDOWN_IT = True
except ImportError:  # pragma: no cover - environment
    HAS_MARKDOWN_IT = False


# ── reading the two config formats by hand ────────────────────────────

COMMENT = re.compile(r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/', re.DOTALL)


def read_jsonc(path: Path) -> dict:
    """wrangler.jsonc is JSON with comments, and comments are how that
    file explains itself. Strings are matched first so a `//` inside one
    survives."""
    def keep(match):
        text = match.group(0)
        return text if text.startswith('"') else " "
    return json.loads(COMMENT.sub(keep, path.read_text(encoding="utf-8")))


def read_headers(path: Path) -> list:
    """`_headers` as [(pattern, {header: value})], in file order — an
    unindented line opens a rule, an indented one adds a header to it."""
    rules, current = [], None
    for line in path.read_text(encoding="utf-8").splitlines():
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        if not line.startswith((" ", "\t")):
            current = (line.strip(), {})
            rules.append(current)
        elif current is not None:
            name, _, value = line.strip().partition(":")
            current[1][name.strip()] = value.strip()
    return rules


def readme_text() -> str:
    return README.read_text(encoding="utf-8")


def rule(pattern: str) -> dict:
    for found, headers in read_headers(HEADERS):
        if found == pattern:
            return headers
    raise AssertionError(f"_headers has no rule for {pattern}")


def run_build(repo: Path, out: Path) -> subprocess.CompletedProcess:
    return subprocess.run(
        [sys.executable, str(repo / "site" / "build.py"), "--out", str(out)],
        capture_output=True, text=True, cwd=repo)


class Built(unittest.TestCase):
    """The real site, built once into a scratch directory."""

    @classmethod
    def setUpClass(cls):
        if not HAS_MARKDOWN_IT:
            raise unittest.SkipTest("markdown-it-py is not installed")
        cls.out = Path(tempfile.mkdtemp(prefix="bench-deploy-")).resolve()
        result = run_build(REPO, cls.out)
        if result.returncode != 0:  # not assert: must survive python -O
            raise RuntimeError(
                f"site/build.py failed:\n{result.stdout}{result.stderr}")

    @classmethod
    def tearDownClass(cls):
        if hasattr(cls, "out"):
            shutil.rmtree(cls.out, ignore_errors=True)


# ── the worker config ─────────────────────────────────────────────────

class TheWorkerServesTheBuild(unittest.TestCase):
    """site/wrangler.jsonc, read the way wrangler reads it."""

    def setUp(self):
        self.config = read_jsonc(WRANGLER)
        self.assets = self.config.get("assets", {})

    def test_the_assets_directory_is_what_the_builder_writes(self):
        """Paths in wrangler config are relative to the config file, so
        this one has to resolve to site/dist and not to a sibling of the
        repo root."""
        directory = (WRANGLER.parent / self.assets["directory"]).resolve()
        self.assertEqual(directory, (SITE / "dist").resolve())

    def test_there_is_no_worker_script(self):
        """The site is files. A `main` would mean a fetch handler to
        maintain, and nothing here needs one."""
        self.assertNotIn("main", self.config)

    def test_a_url_has_exactly_one_form(self):
        """force-trailing-slash: /x redirects to /x/, which is the url
        the pages link and the one rel=canonical names. Anything else
        leaves two urls serving one page."""
        self.assertEqual("force-trailing-slash",
                         self.assets.get("html_handling"))

    def test_an_unknown_path_gets_the_sites_own_404(self):
        """Not single-page-application, which answers 200 with the
        landing page and tells a crawler every typo is a real url."""
        self.assertEqual("404-page", self.assets.get("not_found_handling"))

    def test_the_route_is_a_custom_domain(self):
        routes = self.config.get("routes", [])
        self.assertEqual(1, len(routes), "expected exactly one route")
        self.assertTrue(routes[0].get("custom_domain"),
                        "the route is not a custom domain")

    def test_the_worker_is_named(self):
        self.assertTrue(self.config.get("name"))

    def test_a_compatibility_date_is_pinned(self):
        self.assertRegex(self.config.get("compatibility_date", ""),
                         r"^\d{4}-\d{2}-\d{2}$")


class TheDomainIsWrittenDownOnce(unittest.TestCase):
    """Three files name this site's address. They are allowed to say it
    three times; they are not allowed to disagree."""

    def setUp(self):
        self.config = read_jsonc(WRANGLER)
        self.manifest = json.loads(
            (SITE / "pages.json").read_text(encoding="utf-8"))

    def test_the_route_is_the_host_the_pages_canonicalise_to(self):
        """A rel=canonical pointing somewhere the Worker does not answer
        is worse than none at all."""
        canonical = urlparse(self.manifest["site"]["base_url"])
        self.assertEqual("https", canonical.scheme)
        self.assertEqual(canonical.netloc,
                         self.config["routes"][0]["pattern"])

    def test_the_readme_names_the_worker_and_the_route(self):
        """Acceptance: the next person should not have to guess where
        the site lives."""
        readme = README.read_text(encoding="utf-8")
        self.assertIn(self.config["name"], readme)
        self.assertIn(self.config["routes"][0]["pattern"], readme)

    def test_the_readme_names_the_deploy_command(self):
        self.assertRegex(readme_text(), r"wrangler[^\n]*\bdeploy\b")

    def test_the_readme_names_the_account(self):
        self.assertRegex(readme_text(), r"(?i)\baccount\b")


# ── the 404 page ──────────────────────────────────────────────────────

class TheNotFoundPageIsAPage(Built):
    """`not_found_handling: "404-page"` is half of it. The other half is
    that a 404.html exists, looks like the rest of the site, and offers a
    way out."""

    def setUp(self):
        self.page = (self.out / "404.html").read_text("utf-8")

    def test_it_is_written_where_the_host_looks(self):
        """A 404/index.html would never be found: the host wants a file
        called 404.html at the root of the assets directory."""
        self.assertTrue((self.out / "404.html").is_file())
        self.assertFalse((self.out / "404" / "index.html").exists())

    def test_it_wears_the_sites_design(self):
        self.assertIn('rel="stylesheet" href="/static/site.css', self.page)
        self.assertIn("404", self.page)

    def test_it_offers_the_way_back(self):
        self.assertIn('href="/"', self.page)

    def test_it_claims_no_canonical_url(self):
        """A 404 is not a page to canonicalise to, and telling a crawler
        to index it is worse still."""
        self.assertNotIn('rel="canonical"', self.page)
        self.assertIn('name="robots" content="noindex"', self.page)

    def test_it_stays_out_of_the_nav_and_the_sidebar(self):
        home = (self.out / "index.html").read_text("utf-8")
        article = (self.out / "concepts" / "claiming-a-card"
                   / "index.html").read_text("utf-8")
        for html in (home, article):
            self.assertNotIn("404.html", html)


# ── headers ───────────────────────────────────────────────────────────

class TheHeaderPolicyTravelsWithTheBuild(Built):
    """site/root/_headers is the policy; the build has to carry it to
    the root of dist/ or the host never reads it."""

    def test_it_reaches_the_root_of_the_build(self):
        shipped = self.out / "_headers"
        self.assertTrue(shipped.is_file())
        self.assertEqual(HEADERS.read_text("utf-8"), shipped.read_text("utf-8"))

    def test_it_is_not_also_served_as_a_page(self):
        """The host consumes _headers rather than serving it, but the
        build must not have made a route out of it either."""
        self.assertFalse((self.out / "_headers" / "index.html").exists())

    def test_no_page_needs_what_the_policy_forbids(self):
        """The CSP carries no 'unsafe-inline', so an inline <script> or a
        style="" attribute is a page that renders right in every test
        here and wrong in production. A script with a src is fine only
        if the policy names where it comes from. The only place that can
        be caught before a deploy is the build output."""
        allowed = re.findall(r"https://[^\s;]+",
                             rule("/*")["Content-Security-Policy"])
        for path in sorted(self.out.rglob("*.html")):
            html = path.read_text("utf-8")
            where = path.relative_to(self.out)
            for tag in re.findall(r"<script[^>]*>", html, re.I):
                src = re.search(r'src="([^"]+)"', tag)
                self.assertIsNotNone(
                    src, f"{where} has an inline script the CSP would block")
                self.assertTrue(
                    src.group(1).startswith("/")
                    or any(src.group(1).startswith(o) for o in allowed),
                    f"{where} loads {src.group(1)}, which the CSP forbids")
            self.assertNotRegex(
                html, r"""\sstyle\s*=\s*["']""",
                f"{where} has an inline style the CSP would block")


class EveryResponseCarriesTheBaseline(unittest.TestCase):

    def setUp(self):
        self.everything = rule("/*")

    def test_the_three_headers_a_public_page_owes(self):
        for name in BASELINE:
            self.assertIn(name, self.everything)

    def test_hsts_is_a_year_and_does_not_preload(self):
        value = self.everything["Strict-Transport-Security"]
        self.assertIn("max-age=31536000", value)
        self.assertNotIn("preload", value,
                         "preload is a submission to browser vendors, "
                         "not a header to set in passing")

    def test_only_the_analytics_origin_can_load(self):
        """The site's promise as something the browser enforces rather
        than something a test asserted once at build time. Exactly one
        third party is named — the cookieless analytics — and the day a
        second one appears, this fails rather than shrugging."""
        policy = self.everything["Content-Security-Policy"]
        self.assertIn("default-src 'none'", policy)
        self.assertNotIn("unsafe-inline", policy)
        self.assertNotIn("unsafe-eval", policy)
        self.assertNotIn("*", policy)
        self.assertEqual({"https://cdn.usefathom.com"},
                         set(re.findall(r"https://[^\s;]+", policy)),
                         "the policy names an origin nobody decided on")
        self.assertNotIn("font-src https", policy,
                         "the fonts are self-hosted and stay that way")

    def test_the_analytics_origin_is_allowed_everything_it_uses(self):
        """Fathom sends its pageview as an *image* request, so naming the
        origin under script-src alone loads the script and blocks the one
        thing it exists to do. Nothing server-side can see that: the
        response is a clean 200 and the beacon dies in the browser. It
        cost a live deploy once — hence a test."""
        policy = self.everything["Content-Security-Policy"]
        for directive in ("img-src", "script-src", "connect-src"):
            with self.subTest(directive=directive):
                found = re.search(rf"{directive}([^;]*)", policy)
                self.assertIsNotNone(found, f"{directive} is not set")
                self.assertIn("https://cdn.usefathom.com", found.group(1),
                              f"{directive} does not allow the beacon")


class TheTwoKindsOfFileAreCachedDifferently(unittest.TestCase):
    """The edge case in the task: a stale HTML page must not survive a
    deploy, while the assets it links may live for a year."""

    def test_fingerprinted_assets_are_kept_for_a_year(self):
        value = rule("/static/*")["Cache-Control"]
        self.assertIn("max-age=31536000", value)
        self.assertIn("immutable", value)

    def test_exactly_one_rule_sets_cache_control(self):
        """The host concatenates a header two matching rules both set,
        it does not override — measured on the live site, where /* and
        /static/* together produced 'max-age=0, must-revalidate,
        max-age=31536000, immutable' and the first max-age won. So the
        year-long rule must be the only one, and HTML takes the host's
        own revalidating default."""
        setters = [pattern for pattern, headers in read_headers(HEADERS)
                   if "Cache-Control" in headers]
        self.assertEqual(["/static/*"], setters,
                         "a second Cache-Control rule would concatenate "
                         "with this one rather than lose to it")


# ── fingerprinting ────────────────────────────────────────────────────

class Scratch(unittest.TestCase):
    """A copy of site/ plus the markdown it reads, so a test can edit the
    stylesheet without touching the real one."""

    def setUp(self):
        if not HAS_MARKDOWN_IT:
            raise unittest.SkipTest("markdown-it-py is not installed")
        self.root = Path(tempfile.mkdtemp(prefix="bench-deploy-")).resolve()
        self.addCleanup(shutil.rmtree, self.root, True)
        shutil.copytree(SITE, self.root / "site",
                        ignore=shutil.ignore_patterns("dist", "__pycache__"))
        # The markdown the pages are cut from, the file the version shown
        # on them is read from, and whatever a link inside a slice points
        # at. One list, in tests/test_site_build.py — a scratch repo that
        # is missing one of them fails for a reason no test here is about.
        from tests.test_site_build import SOURCES

        for name in SOURCES:
            (self.root / name).parent.mkdir(parents=True, exist_ok=True)
            shutil.copy(REPO / name, self.root / name)
        self.out = self.root / "site" / "dist"

    def build(self):
        return run_build(self.root, self.out)

    def home(self) -> str:
        return (self.out / "index.html").read_text("utf-8")


class TheStylesheetUrlFollowsTheStylesheet(Scratch):
    """`immutable` for a year is only safe because the url changes when
    the file does. That is the claim under test."""

    STAMP = re.compile(r'href="(/static/site\.css\?v=([0-9a-f]{10}))"')

    def test_the_pages_link_a_stamped_url(self):
        self.assertEqual(0, self.build().returncode)
        self.assertRegex(self.home(), self.STAMP.pattern)

    def test_the_file_still_sits_at_its_plain_path(self):
        """Only the link carries the stamp — nothing in static/ is
        renamed, so the /static/* glob in _headers still matches and the
        tree stays readable."""
        self.assertEqual(0, self.build().returncode)
        self.assertTrue((self.out / "static" / "site.css").is_file())

    def test_editing_the_stylesheet_changes_the_stamp(self):
        self.assertEqual(0, self.build().returncode)
        before = self.STAMP.search(self.home()).group(2)

        css = self.root / "site" / "static" / "site.css"
        css.write_text(css.read_text("utf-8") + "\n.lost{color:red}\n",
                       encoding="utf-8")
        self.assertEqual(0, self.build().returncode)
        after = self.STAMP.search(self.home()).group(2)

        self.assertNotEqual(before, after,
                            "a changed stylesheet kept its url, so a "
                            "year-long cache would keep the old one")

    def test_a_rebuild_that_changes_nothing_keeps_the_stamp(self):
        """The other half: a deploy that did not touch the stylesheet
        must not invalidate everyone's cache."""
        self.assertEqual(0, self.build().returncode)
        before = self.STAMP.search(self.home()).group(2)
        self.assertEqual(0, self.build().returncode)
        self.assertEqual(before, self.STAMP.search(self.home()).group(2))


class TheRootTreeIsCopiedVerbatim(Scratch):

    def test_a_root_file_lands_at_the_top_of_the_build(self):
        (self.root / "site" / "root" / "robots.txt").write_text(
            "User-agent: *\n", encoding="utf-8")
        self.assertEqual(0, self.build().returncode)
        self.assertEqual("User-agent: *\n",
                         (self.out / "robots.txt").read_text("utf-8"))

    def test_a_root_file_a_route_also_claims_stops_the_build(self):
        """Both would be written, one would win, and which one is not
        something a person should have to work out from a diff."""
        (self.root / "site" / "root" / "404.html").write_text(
            "mine", encoding="utf-8")
        result = self.build()
        self.assertNotEqual(0, result.returncode)
        self.assertIn("404.html", result.stderr)


class TheDeployIsInTheRepository(unittest.TestCase):
    """The two facts about deployment that live outside a config file."""

    def test_the_worker_config_and_the_headers_are_tracked(self):
        tracked = subprocess.run(
            ["git", "check-ignore", "site/wrangler.jsonc", "site/root/_headers"],
            capture_output=True, text=True, cwd=REPO)
        self.assertNotEqual(tracked.returncode, 0,
                            "the deploy config is gitignored")

    def test_the_release_artifact_still_does_not_ship_the_site(self):
        """A Worker config in site/ must not start travelling into host
        projects with a release."""
        manifest = (REPO / "manager" / "core" / "release-manifest"
                    ).read_text("utf-8")
        for line in manifest.splitlines():
            if line.strip().startswith("#") or not line.strip():
                continue
            self.assertNotIn("site", line.split(None, 1)[1],
                             f"the release manifest names site/: {line}")


if __name__ == "__main__":
    unittest.main()