From 42e1dc9799f60c7dde46b9485e5f37a5a29630ae Mon Sep 17 00:00:00 2001 From: SnapOtter Date: Tue, 28 Jul 2026 17:08:32 +0800 Subject: [PATCH] fix(docs): keep the translated locale trees out of the search index (#662) Search Console flagged four reasons on 2026-07-28: soft 404, both duplicate-canonical variants, and noindex. All four sat on docs.snapotter.com; the landing site was clean. The cause was boilerplate dominance rather than a broken tag. /tools/video/crop-video carries 1.3 KB of unique body against 3.5 KB of identical chrome (nav, sidebar, 21-language switcher), so unrelated tool docs measured 52-60% full-page similarity. Across 20 locales that was 3,640 of 3,822 submitted URLs. Google read the lot as one duplicate cluster and began electing arbitrary representatives: /changelog became the canonical for /tools/image/favicon, and /uk/guide/getting-started for /nl/tools/image/resize. English tool docs indexed 2 of 10 and localized 4 of 10, while the landing page for those same five tools indexed 5 of 5. Translated pages now emit `noindex, follow` with a self-canonical, and sitemap.transformItems drops them from the sitemap. Self-canonical rather than pointing at English, since noindex paired with a cross-canonical sends two conflicting instructions. hreflang is removed outright: the annotation only means something between pages that can all be indexed. Readers see no change. The language switcher and every in-page link behave exactly as before. Verified against a real build: sitemap 3,822 to 182 URLs with zero translated entries, all 3,640 translated files carrying the noindex and no English file doing so, docs e2e 100 passed. --- apps/docs/.vitepress/config.mts | 39 ++++++++++++++++--- tests/e2e-docs/i18n.spec.ts | 21 ++++++---- tests/unit/surface-playwright-harness.test.ts | 2 +- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/apps/docs/.vitepress/config.mts b/apps/docs/.vitepress/config.mts index 5a0b9337..bf9b7067 100644 --- a/apps/docs/.vitepress/config.mts +++ b/apps/docs/.vitepress/config.mts @@ -9,6 +9,11 @@ import { pageOnlySidebar } from "./llms-sidebar.mjs"; const NON_EN = SUPPORTED_LOCALES.filter((l) => l.code !== "en"); const HOSTNAME = "https://docs.snapotter.com"; +// Matches a path whose first segment is one of the translated locales, with or +// without a leading slash (VitePress hands transformItems a relative page path, +// but the sitemap stream is happy either way). +const TRANSLATED_PREFIX = new RegExp(`^/?(${NON_EN.map((l) => l.code).join("|")})(/|$)`); + // Prefix every `link` in a sidebar/nav tree with /. // biome-ignore lint/suspicious/noExplicitAny: VitePress nav/sidebar item trees are recursively typed. function prefixLinks(items: any[], locale: string): any[] { @@ -126,7 +131,14 @@ export default defineConfig({ // serves the clean URL at 200, so no extra server config is needed. cleanUrls: true, - sitemap: { hostname: "https://docs.snapotter.com" }, + // Only the English tree is submitted. The translated trees are noindexed in + // transformHead below, and submitting pages we ask Google not to index is a + // contradiction that shows up in Search Console as sitemap coverage errors. + // Drops the submitted count from 3,822 to 182. + sitemap: { + hostname: HOSTNAME, + transformItems: (items) => items.filter((item) => !TRANSLATED_PREFIX.test(item.url)), + }, // VitePress defaults to shiki's `github-light`, whose comment (#6a737d, 4.45:1) // and string (#22863a, 4.28:1) tokens both miss AA against the code-block @@ -176,11 +188,28 @@ export default defineConfig({ code === "en" ? `${HOSTNAME}/${enRel}` : `${HOSTNAME}/${code}/${enRel}`; head.push(["link", { rel: "canonical", href: urlFor(current) }]); - head.push(["link", { rel: "alternate", hreflang: "x-default", href: urlFor("en") }]); - head.push(["link", { rel: "alternate", hreflang: "en", href: urlFor("en") }]); - for (const l of NON_EN) { - head.push(["link", { rel: "alternate", hreflang: l.code, href: urlFor(l.code) }]); + + // The translated trees are machine translations of pages whose unique body is + // already small next to the shared chrome (nav, sidebar, 21-language switcher). + // At 20 locales x 182 pages Google read the lot as one duplicate cluster and + // began electing arbitrary representatives across languages: it picked + // /changelog as the canonical for /tools/image/favicon, and + // /uk/guide/getting-started for /nl/tools/image/resize. Most were dropped, and + // carrying 3,640 pages it refused to index starved the English ones of crawl + // budget. + // + // Self-canonical rather than a cross-canonical to English: pairing noindex with + // a canonical pointing elsewhere sends two conflicting instructions. "follow" + // keeps the outbound links live. Readers lose nothing, since the language + // switcher and every in-page link behave exactly as before. + if (isLocale) { + head.push(["meta", { name: "robots", content: "noindex, follow" }]); } + + // No hreflang. The annotation only means something between pages that can all + // be indexed, and English is now the only one; pointing it at noindexed URLs is + // ignored at best. Restore this alongside the noindex above if the translated + // trees ever become indexable again. const ogLocale = current === "en" ? "en_US" : current.replace("-", "_"); head.push(["meta", { property: "og:locale", content: ogLocale }]); head.push(["meta", { property: "og:url", content: urlFor(current) }]); diff --git a/tests/e2e-docs/i18n.spec.ts b/tests/e2e-docs/i18n.spec.ts index 029bcc67..e9f6d756 100644 --- a/tests/e2e-docs/i18n.spec.ts +++ b/tests/e2e-docs/i18n.spec.ts @@ -38,14 +38,21 @@ test.describe("docs i18n (English + committed German locale)", () => { await expect(target).toBeInViewport(); }); - test("hreflang alternates are reciprocal and include x-default", async ({ page }) => { + test("English pages are indexable and self-canonical", async ({ page }) => { await page.goto("/guide/getting-started"); - const en = await page.locator('link[hreflang="en"]').getAttribute("href"); - const de = await page.locator('link[hreflang="de"]').getAttribute("href"); - const xd = await page.locator('link[hreflang="x-default"]').getAttribute("href"); - expect(en).toContain("/guide/getting-started"); - expect(de).toContain("/de/guide/getting-started"); - expect(xd).toBe(en); + const canonical = await page.locator('link[rel="canonical"]').getAttribute("href"); + expect(canonical).toBe("https://docs.snapotter.com/guide/getting-started"); + await expect(page.locator('meta[name="robots"]')).toHaveCount(0); + }); + + // Only English is indexable, so there is no hreflang cluster left to annotate. + // See the transformHead comment in apps/docs/.vitepress/config.mts. + test("translated pages are noindex, self-canonical, and carry no hreflang", async ({ page }) => { + await page.goto("/de/guide/getting-started"); + const canonical = await page.locator('link[rel="canonical"]').getAttribute("href"); + expect(canonical).toBe("https://docs.snapotter.com/de/guide/getting-started"); + await expect(page.locator('meta[name="robots"]')).toHaveAttribute("content", "noindex, follow"); + await expect(page.locator("link[hreflang]")).toHaveCount(0); }); test("pagefind returns results within the German locale", async ({ page }) => { diff --git a/tests/unit/surface-playwright-harness.test.ts b/tests/unit/surface-playwright-harness.test.ts index 0ad4629b..309e82dd 100644 --- a/tests/unit/surface-playwright-harness.test.ts +++ b/tests/unit/surface-playwright-harness.test.ts @@ -267,7 +267,7 @@ test("docs Playwright collection includes explicit Axe coverage", () => { for (const pageName of ["homepage", "getting started", "configuration", "REST API"]) { expect(output).toContain(`${pageName} has no Axe violations`); } - expect(output).toContain("Total: 99 tests in 7 files"); + expect(output).toContain("Total: 100 tests in 7 files"); }); test("Turbo passes surface harness overrides through task boundaries", () => {