diff --git a/web/docs/README.md b/web/docs/README.md index e51c609..4e38086 100644 --- a/web/docs/README.md +++ b/web/docs/README.md @@ -8,8 +8,45 @@ npm install npm run dev # http://localhost:4321 npm run build # -> dist/ npm run preview + +npm run check:sitemap http://localhost:4321 # after npm run preview +npm run check:sitemap https://docs.beardrive.ai # or against production ``` +## The sitemap + +`/sitemap-index.xml` -> `/sitemap-0.xml`, generated by `@astrojs/sitemap`. +Starlight would add that integration itself; `astro.config.mjs` declares it +explicitly instead, which replaces Starlight's default rather than duplicating +it and is what makes the `lastmod` option reachable. + +Each URL carries a `` taken from the **commit date of the markdown +behind it** — see "Checkout depth" below for the one way that goes wrong. The +redirect stubs are correctly absent: Astro emits them as `noindex` meta-refresh +pages, and `@astrojs/sitemap` leaves them out. + +`public/robots.txt` exists to carry the `Sitemap:` line: robots.txt is the one +file a crawler fetches without being told, and the landing page's robots.txt is +on a different host so it cannot point here. + +**Verify that file survives the deploy.** This host is behind Cloudflare, which +was serving a managed robots.txt of its own (the content-signals policy block) +back when the origin had none — a body with no `Sitemap:` and, in fact, no +directives at all. Whether an origin robots.txt now replaces that or gets merged +with it is Cloudflare's call, not this repo's, so after deploying check the +served file rather than the built one: + +```sh +curl -s https://docs.beardrive.ai/robots.txt | grep -i sitemap +``` + +`check:sitemap` is the same set of checks Search Console runs — follow +robots.txt to the index, parse it, confirm every advertised URL returns 200 — +and takes any origin, so it works against a local preview and against +production. It is worth running against production after a deploy: a sitemap +that has gone stale or started advertising 404s looks completely fine until +something crawls it, which is weeks later. + ## Why this is a standalone site Unlike the hub frontend (`internal/webapp/static`) and the cloud landing page @@ -128,3 +165,23 @@ portable fallback, and it keeps local `npm run preview` honest. Note that the build reads a file **outside** `web/docs` (the token source), so the host must check out the whole repository rather than just this subdirectory. + +### Checkout depth + +Check out with **full history**, not a shallow clone. The sitemap's `` +for each page is the commit date of the markdown behind it, so a depth-1 +checkout — the default for `actions/checkout` and for most build hosts — has +nothing to read the dates from. + +```yaml +- uses: actions/checkout@v4 + with: + fetch-depth: 0 # sitemap comes from commit dates +``` + +Getting this wrong degrades rather than breaks: `astro.config.mjs` detects the +shallow clone and emits **no** `lastmod` at all, because the alternative is +stamping all 25 pages with the one commit a shallow clone has, and a sitemap +that claims the whole site changed on every deploy is one Google learns to +ignore. So the symptom is a silently less useful sitemap — check for `` +in the deployed `sitemap-0.xml` after changing hosts. diff --git a/web/docs/astro.config.mjs b/web/docs/astro.config.mjs index d90380e..46b3f7d 100644 --- a/web/docs/astro.config.mjs +++ b/web/docs/astro.config.mjs @@ -1,7 +1,9 @@ // @ts-check import { defineConfig } from "astro/config"; import starlight from "@astrojs/starlight"; +import sitemap from "@astrojs/sitemap"; import llmsTxt from "starlight-llms-txt"; +import { execFileSync } from "node:child_process"; // docs.beardrive.ai — the public product documentation. // @@ -14,6 +16,49 @@ import llmsTxt from "starlight-llms-txt"; // It lives in the OSS repo because that's what it documents: the CLI, the sync // model, self-hosting. "Edit this page" resolves to something an outside // contributor can actually open a PR against. + +// `lastmod` for the sitemap, from the commit that last touched each page. +// +// The tempting shortcut — stamp every URL with the build time — is worse than +// emitting nothing: a sitemap that reports the whole site changed on every +// deploy teaches Google to disregard its lastmod entirely, and freshness is +// most of what a sitemap is for on a docs site. +// +// Which is also why the shallow check exists. A CI checkout is depth-1 by +// default, and there `git log` attributes every file to the one commit it has +// — the same uniform lie in a different costume. No history, no lastmod. +// (A host that wants these dates must clone with full history: +// actions/checkout needs `fetch-depth: 0`.) +const lastmodBySource = (() => { + const git = (...args) => + execFileSync("git", args, { cwd: import.meta.dirname, encoding: "utf8" }).trim(); + try { + if (git("rev-parse", "--is-shallow-repository") === "true") return new Map(); + // One `git log` for every page rather than one per page. Newest commit + // first, so the first time a path appears is its last modification. + const log = git("log", "--format=%cI", "--name-only", "--relative", "--", "src/content/docs"); + const map = new Map(); + let when = ""; + for (const line of log.split("\n")) { + if (!line) continue; + else if (/^\d{4}-\d\d-\d\dT/.test(line)) when = line; + else if (!map.has(line)) map.set(line, when); + } + return map; + } catch { + return new Map(); // built from a tarball, or no git installed — not fatal + } +})(); + +/** `/reference/cli/` -> the date on `src/content/docs/reference/cli.md`. */ +function lastmodFor(url) { + const slug = new URL(url).pathname.replace(/^\/|\/$/g, "") || "index"; + for (const ext of [".md", ".mdx"]) { + const at = lastmodBySource.get(`src/content/docs/${slug}${ext}`); + if (at) return at; + } +} + export default defineConfig({ site: "https://docs.beardrive.ai", // The docs were reorganized around the agent-first path; these URLs were @@ -26,6 +71,11 @@ export default defineConfig({ "/manual/skills-and-hooks": "/manual/hooks/", }, integrations: [ + // Starlight adds @astrojs/sitemap itself, but only when the config hasn't + // already — declaring it here replaces that default rather than doubling + // it, which is the supported way to reach these options. (Starlight's own + // version only sets `i18n`, and this site is single-language.) + sitemap({ serialize: (item) => ({ ...item, lastmod: lastmodFor(item.url) }) }), starlight({ title: "BearDrive", description: diff --git a/web/docs/package.json b/web/docs/package.json index c878ba5..7165419 100644 --- a/web/docs/package.json +++ b/web/docs/package.json @@ -6,7 +6,8 @@ "dev": "npm run tokens && astro dev", "build": "npm run tokens && astro build", "preview": "astro preview", - "tokens": "node scripts/tokens.mjs" + "tokens": "node scripts/tokens.mjs", + "check:sitemap": "node scripts/check-sitemap.mjs" }, "dependencies": { "@astrojs/starlight": "^0.41.3", diff --git a/web/docs/public/robots.txt b/web/docs/public/robots.txt new file mode 100644 index 0000000..6899627 --- /dev/null +++ b/web/docs/public/robots.txt @@ -0,0 +1,22 @@ +# docs.beardrive.ai +# +# The Sitemap line is why this file exists. robots.txt is the one place a +# crawler looks without being told, and until this existed the docs sitemap +# was reachable only by guessing its URL or by submitting it in Search +# Console — the landing page's sitemap is on a different host and can't +# point here. +# +# Everything is allowed, including every AI crawler. That is the same posture +# as the landing's robots.txt, which spells each agent out one by one; this +# file doesn't need to, and the difference is not an oversight. That file +# carries Disallow rules for the hub's own routes (/api/, /auth/, /s/ — the +# app shares its origin), and a named user-agent stops reading the `*` group +# entirely, so each one has to be re-granted access explicitly. Nothing on +# this host is private, so nothing is disallowed, so one group covers all of +# them. Add a Disallow here and that stops being true — restore the +# enumeration if you ever do. + +User-agent: * +Allow: / + +Sitemap: https://docs.beardrive.ai/sitemap-index.xml diff --git a/web/docs/scripts/check-sitemap.mjs b/web/docs/scripts/check-sitemap.mjs new file mode 100644 index 0000000..d5d89ed --- /dev/null +++ b/web/docs/scripts/check-sitemap.mjs @@ -0,0 +1,133 @@ +// Check a site's sitemap the way a crawler would. +// +// node scripts/check-sitemap.mjs https://docs.beardrive.ai +// node scripts/check-sitemap.mjs http://localhost:4321 # npm run preview +// node scripts/check-sitemap.mjs https://beardrive.ai https://docs.beardrive.ai +// +// Every check below is one Search Console performs, in the order it performs +// them: find the sitemap from robots.txt, follow the index, parse it, then +// confirm the URLs it advertises actually resolve. A sitemap listing 404s or +// redirects is the most common Search Console complaint, and it is invisible +// until something crawls it -- which is weeks after the deploy that broke it. +// +// No dependencies and no test framework: it runs against a live origin, which +// is the only place these can actually be wrong. + +const W3C_DATETIME = /^\d{4}-\d\d-\d\d(T\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:\d\d))?$/; + +// The sitemap namespace, matched loosely. Tag names are compared with the +// namespace stripped, so a document declaring the schema with a prefix still +// parses -- the point is to catch a wrong ROOT element, not to validate XML. +const tag = (xml, name) => [...xml.matchAll(new RegExp(`<${name}>(.*?)`, "gs"))].map((m) => m[1]); + +async function fetchOk(url, method = "GET") { + const res = await fetch(url, { method, redirect: "manual" }); + return { status: res.status, type: res.headers.get("content-type") ?? "", body: method === "GET" ? await res.text() : "" }; +} + +async function check(origin) { + console.log(`\n=== ${origin} ===`); + const fail = []; + + // A built sitemap always holds absolute PRODUCTION URLs -- `site` in + // astro.config.mjs -- even when served from localhost. So the origin the + // sitemap claims is read from the sitemap itself, and requests are made + // against the origin being checked. Against production the two are the same + // and this does nothing; against a local server it is what makes the check + // possible at all. What it does NOT paper over is a sitemap listing more + // than one origin, which is a real misconfiguration and fails below. + let claimed = origin; + const here = (url) => origin + new URL(url).pathname; + + // robots.txt is how a crawler finds the sitemap without being told. + let declared = []; + const robots = await fetchOk(`${origin}/robots.txt`); + if (robots.status !== 200) { + fail.push(`robots.txt returned ${robots.status} — the sitemap is undiscoverable on this host`); + } else { + declared = [...robots.body.matchAll(/^\s*sitemap:\s*(\S+)/gim)].map((m) => m[1]); + console.log(` robots.txt 200, declares ${declared.join(", ") || "NOTHING"}`); + if (!declared.length) fail.push("robots.txt declares no Sitemap:"); + } + + const index = await fetchOk(`${origin}/sitemap-index.xml`); + console.log(` sitemap-index.xml ${index.status} ${index.type}`); + if (index.status !== 200) return [...fail, `sitemap-index.xml returned ${index.status}`]; + if (!index.type.includes("xml")) fail.push(`index served as ${index.type}, not XML`); + if (!/]/.test(index.body)) fail.push("index root is not "); + + const children = tag(index.body, "loc"); + if (children[0]) { + claimed = new URL(children[0]).origin; + if (claimed !== origin) console.log(` serving ${claimed} (checked at ${origin})`); + } + if (declared.length && !declared.includes(`${claimed}/sitemap-index.xml`)) { + fail.push(`robots.txt declares ${declared.join(", ")}, not ${claimed}/sitemap-index.xml`); + } + + const locs = []; + let dated = 0; + for (const child of children) { + if (new URL(child).origin !== claimed) { + fail.push(`index mixes origins: ${child} is not on ${claimed}`); + continue; + } + const doc = await fetchOk(here(child)); + console.log(` ${child.split("/").pop().padEnd(19)} ${doc.status} ${doc.type}`); + if (doc.status !== 200) { + fail.push(`index points at ${child}, which returned ${doc.status}`); + continue; + } + if (!/]/.test(doc.body)) fail.push(`${child} root is not `); + for (const entry of tag(doc.body, "url")) { + const [loc] = tag(entry, "loc"); + if (loc) locs.push(loc); + const [lastmod] = tag(entry, "lastmod"); + if (lastmod !== undefined) { + dated++; + if (!W3C_DATETIME.test(lastmod)) fail.push(`${loc} lastmod "${lastmod}" is not a W3C datetime`); + } + } + } + console.log(` urls ${locs.length} (${dated} with lastmod)`); + if (!locs.length) fail.push("the sitemap advertises no URLs"); + + // Two entries for one page split its ranking signals between them. + const dupes = [...new Set(locs.filter((l, i) => locs.indexOf(l) !== i))]; + if (dupes.length) fail.push(`duplicate : ${dupes.join(", ")}`); + + // A 3xx here is a finding, not a pass: a sitemap should list the URL a page + // actually lives at, which is why redirect: "manual" is set above. + let ok = 0; + for (const loc of locs) { + if (new URL(loc).origin !== claimed) { + fail.push(`${loc} is not on ${claimed}`); + continue; + } + const { status } = await fetchOk(here(loc), "HEAD"); + if (status === 200) ok++; + else fail.push(`${loc} returns ${status}`); + } + console.log(` resolve ${ok}/${locs.length} return 200`); + + return fail; +} + +const origins = process.argv.slice(2); +if (!origins.length) { + console.error("usage: node scripts/check-sitemap.mjs [origin...]"); + process.exit(2); +} + +const failures = []; +for (const origin of origins) { + failures.push(...(await check(origin.replace(/\/$/, ""))).map((f) => [origin, f])); +} + +console.log(); +if (failures.length) { + console.log(`FAIL — ${failures.length} problem(s):`); + for (const [origin, f] of failures) console.log(` [${origin}] ${f}`); + process.exit(1); +} +console.log("PASS — sitemaps are well-formed, complete, and discoverable.");