feat(i18n): 21-language pipeline, landing/docs/API wiring, landing+API translations

Shared Claude Code translation pipeline (scripts/i18n, no API key) plus Astro/VitePress/Scalar i18n wiring. Landing and API reference translated into all 20 languages; docs i18n wiring + English source anchors. The translated docs markdown (apps/docs/<locale>/**, 3,620 files) follows in a companion PR because it exceeds GitHub's per-PR CI file limit.
This commit is contained in:
SnapOtter
2026-07-11 13:01:55 +08:00
committed by GitHub
parent 2e91368816
commit 00b651c9f8
353 changed files with 564607 additions and 2594 deletions
+147
View File
@@ -0,0 +1,147 @@
// scripts/i18n/adapters/api-spec.mjs
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import yaml from "js-yaml";
const __dirname = dirname(fileURLToPath(import.meta.url));
// The real spec lives next to the API source. Tests pass a `dir` override.
const DEFAULT_DIR = join(__dirname, "../../../apps/api/src");
const HTTP_METHODS = ["get", "post", "put", "patch", "delete", "head", "options"];
// structuredClone is global in Node 22+, used to deep-copy the parsed spec.
const clone = (value) => structuredClone(value);
/**
* Read and parse the English OpenAPI document.
* @param {string} dir
* @returns {any}
*/
function loadEnglishSpec(dir) {
const raw = readFileSync(join(dir, "openapi.yaml"), "utf8");
return yaml.load(raw);
}
/**
* Walk a parsed spec and yield [id, text] for every translatable prose field,
* in a deterministic order. This is the single source of the id contract used by
* extract, load, and write.
* @param {any} spec
* @returns {Array<[string, string]>}
*/
export function proseFields(spec) {
const out = [];
if (typeof spec?.info?.description === "string") {
out.push(["info.description", spec.info.description]);
}
for (const tag of spec?.tags ?? []) {
if (tag && typeof tag.name === "string" && typeof tag.description === "string") {
out.push([`tags.${tag.name}.description`, tag.description]);
}
}
for (const [path, methods] of Object.entries(spec?.paths ?? {})) {
for (const method of HTTP_METHODS) {
const op = methods?.[method];
if (!op) continue;
if (typeof op.summary === "string") {
out.push([`paths.${path}.${method}.summary`, op.summary]);
}
if (typeof op.description === "string") {
out.push([`paths.${path}.${method}.description`, op.description]);
}
}
}
return out;
}
/**
* Set a prose field on a cloned spec by the same id `proseFields` produced.
* @param {any} spec
* @param {string} id
* @param {string} value
*/
export function setProseField(spec, id, value) {
if (id === "info.description") {
spec.info.description = value;
return;
}
const tagMatch = id.match(/^tags\.(.+)\.description$/);
if (tagMatch) {
const tag = (spec.tags ?? []).find((t) => t?.name === tagMatch[1]);
if (tag) tag.description = value;
return;
}
// paths.<path>.<method>.<field> where <path> may itself contain dots.
const pathMatch = id.match(/^paths\.(.+)\.([a-z]+)\.(summary|description)$/);
if (pathMatch) {
const [, path, method, field] = pathMatch;
const op = spec.paths?.[path]?.[method];
if (op) op[field] = value;
}
}
/**
* Build the api-spec adapter.
* @param {{ dir?: string }} [opts]
*/
export function makeApiSpecAdapter({ dir = DEFAULT_DIR } = {}) {
return {
name: "api",
async extract() {
const spec = loadEnglishSpec(dir);
return proseFields(spec).map(([id, sourceText]) => ({ id, sourceText, kind: "text" }));
},
async write(locale, entries) {
const english = loadEnglishSpec(dir);
const localized = clone(english);
const stamp = {};
// Replace only the prose fields that have a translation; anything missing
// keeps its English text so the document is always complete and valid.
for (const [id] of proseFields(english)) {
const entry = entries.get(id);
if (!entry) continue;
setProseField(localized, id, entry.text);
stamp[id] = {
sourceHash: entry.sourceHash,
provenance: entry.provenance,
outputHash: entry.outputHash,
...(entry.stale ? { stale: true } : {}),
};
}
localized["x-i18n"] = {
locale,
generator: "scripts/i18n/adapters/api-spec.mjs",
entries: stamp,
};
const out = yaml.dump(localized, { lineWidth: -1, noRefs: true });
writeFileSync(join(dir, `openapi.${locale}.yaml`), out, "utf8");
},
async load(locale) {
const file = join(dir, `openapi.${locale}.yaml`);
const result = new Map();
if (!existsSync(file)) return result;
const spec = yaml.load(readFileSync(file, "utf8"));
const stamp = spec?.["x-i18n"]?.entries ?? {};
for (const [id, text] of proseFields(spec)) {
const meta = stamp[id];
if (!meta) continue; // untranslated fallback field, not a stored entry
result.set(id, {
text,
sourceHash: meta.sourceHash,
provenance: meta.provenance === "human" ? "human" : "machine",
outputHash: meta.outputHash,
...(meta.stale ? { stale: true } : {}),
});
}
return result;
},
};
}
export const adapter = makeApiSpecAdapter();
+338
View File
@@ -0,0 +1,338 @@
// scripts/i18n/adapters/docs-md.mjs
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join, relative, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { localeCodes } from "../lib/shared-i18n.mjs";
import { slugify } from "../lib/slugify.mjs";
const HERE = dirname(fileURLToPath(import.meta.url));
// scripts/i18n/adapters -> repo root -> apps/docs
const DEFAULT_ROOT = join(HERE, "..", "..", "..", "apps", "docs");
const LOCALE_DIRS = new Set(localeCodes().filter((c) => c !== "en"));
const SKIP_DIRS = new Set(["node_modules", ".vitepress", "public", ...LOCALE_DIRS]);
// Docs-dialect masking tokens (distinct delimiters from the shared mask lib so
// the two layers never collide).
const DTOKEN = (i) => `⟦DOCS${i}`;
const DTOKEN_RE = /⟦DOCS\d+⟧/g;
/**
* List every root markdown file (relative POSIX path), excluding locale subtrees
* and non-content dirs.
* @param {string} root
* @returns {Promise<string[]>}
*/
async function listRootMarkdown(root) {
const out = [];
async function walk(dir) {
const entries = await readdir(dir, { withFileTypes: true });
for (const ent of entries) {
const abs = join(dir, ent.name);
if (ent.isDirectory()) {
if (SKIP_DIRS.has(ent.name)) continue;
await walk(abs);
} else if (ent.name.endsWith(".md")) {
out.push(relative(root, abs).split(sep).join("/"));
}
}
}
await walk(root);
return out.sort();
}
/**
* Split frontmatter from body. Returns { fm, body } where fm is the raw
* frontmatter block WITHOUT the fences (or null), and body is everything after.
* @param {string} text
* @returns {{ fm: string|null, body: string }}
*/
function splitFrontmatter(text) {
const m = text.match(/^---\n([\s\S]*?)\n---\n?/);
if (!m) return { fm: null, body: text };
return { fm: m[1], body: text.slice(m[0].length) };
}
/**
* Read a single scalar value from a raw frontmatter block, e.g. key: value.
* @param {string|null} fm
* @param {string} key
* @returns {string|undefined}
*/
function fmGet(fm, key) {
if (!fm) return undefined;
const m = fm.match(new RegExp(`^${key}:\\s*(.*)$`, "m"));
if (!m) return undefined;
return m[1].trim().replace(/^["']|["']$/g, "");
}
/**
* Split a body into fenced-code segments and prose segments so heading/anchor and
* docs-mask logic never touch lines inside ``` fences.
* @param {string} body
* @returns {Array<{ code: boolean, text: string }>}
*/
function segmentFences(body) {
const parts = [];
const re = /(^|\n)(```|~~~)[\s\S]*?\n\2/g;
let last = 0;
let m;
while ((m = re.exec(body)) !== null) {
if (m.index > last) parts.push({ code: false, text: body.slice(last, m.index) });
parts.push({ code: true, text: m[0] });
last = m.index + m[0].length;
}
if (last < body.length) parts.push({ code: false, text: body.slice(last) });
return parts;
}
/**
* Inject `{#slug}` into ATX headings that lack an explicit anchor. Idempotent.
* Only runs over non-code segments.
* @param {string} body
* @returns {string}
*/
function injectAnchors(body) {
// De-duplicate slugs exactly like markdown-it-anchor: a base slug already used
// in the file gets "-1", "-2", ... appended (start index 1). Explicit {#id}
// anchors already in the source claim their slot too, so injected slugs never
// collide with them. Seen-set spans every prose segment, matching VitePress.
const seen = new Set();
const unique = (base) => {
let slug = base;
let i = 1;
while (seen.has(slug)) {
slug = `${base}-${i}`;
i += 1;
}
seen.add(slug);
return slug;
};
return segmentFences(body)
.map((seg) => {
if (seg.code) return seg.text;
return seg.text.replace(/^(#{1,6})[ \t]+(.+?)[ \t]*$/gm, (line, hashes, title) => {
const explicit = title.match(/\{#([^}]+)\}\s*$/);
if (explicit) {
seen.add(explicit[1]); // pre-existing anchor claims its slug
return line;
}
const clean = title.replace(/[ \t]+#*$/, ""); // drop optional closing ATX hashes
return `${hashes} ${clean} {#${unique(slugify(clean))}}`;
});
})
.join("");
}
/**
* Docs-dialect pre-mask over non-code segments: hide `:::` container markers
* (keep the title label), `[[toc]]`, and explicit `{#anchor}` slugs so the model
* never rewrites them. Returns masked text plus the token table.
* @param {string} body
* @returns {{ masked: string, tokens: string[] }}
*/
function maskDocs(body) {
const tokens = [];
const push = (v) => {
tokens.push(v);
return DTOKEN(tokens.length - 1);
};
const masked = segmentFences(body)
.map((seg) => {
if (seg.code) return seg.text;
let out = seg.text;
// ::: type (mask the "::: type " marker, leave the title text after it)
out = out.replace(/^(:{3,})[ \t]*([a-zA-Z-]+)[ \t]*/gm, (_m, colons, type) =>
push(`${colons} ${type} `),
);
// bare closing :::
out = out.replace(/^:{3,}[ \t]*$/gm, (m) => push(m));
// [[toc]]
out = out.replace(/\[\[toc\]\]/gi, (m) => push(m));
// explicit {#anchor}
out = out.replace(/\{#[^}\n]+\}/g, (m) => push(m));
return out;
})
.join("");
return { masked, tokens };
}
/**
* Restore docs-dialect tokens.
* @param {string} masked
* @param {string[]} tokens
* @returns {string}
*/
function restoreDocs(masked, tokens) {
return String(masked).replace(DTOKEN_RE, (t) => {
const m = t.match(/DOCS(\d+)/);
const i = m ? Number(m[1]) : Number.NaN;
return tokens[i] ?? t;
});
}
/**
* Rewrite internal absolute links (/guide/x, /tools/y, /api/z) to /<locale>/...
* Leaves external URLs, already-prefixed links, and root asset paths alone.
* @param {string} body
* @param {string} locale
* @returns {string}
*/
function rewriteLinks(body, locale) {
return body.replace(/(\]\()(\/[^)\s]*)(\))/g, (_m, open, url, close) => {
if (url.startsWith(`/${locale}/`)) return `${open}${url}${close}`;
// Do not prefix asset paths VitePress serves from root public/.
if (/^\/(fonts|screenshots|logo|favicon|apple-touch|og-|llms)/.test(url)) {
return `${open}${url}${close}`;
}
return `${open}/${locale}${url}${close}`;
});
}
/**
* Insert or replace scalar frontmatter keys, preserving existing ones.
* @param {string} text
* @param {Record<string,string>} fields
* @returns {string}
*/
function upsertFrontmatter(text, fields) {
const { fm, body } = splitFrontmatter(text);
const lines = fm != null ? fm.split("\n") : [];
const seen = new Set();
const next = lines.map((line) => {
const m = line.match(/^([A-Za-z0-9_]+):/);
if (m && fields[m[1]] !== undefined) {
seen.add(m[1]);
return `${m[1]}: ${fields[m[1]]}`;
}
return line;
});
for (const [k, v] of Object.entries(fields)) {
if (!seen.has(k)) next.push(`${k}: ${v}`);
}
return `---\n${next.join("\n")}\n---\n${body}`;
}
/**
* Double-quote bare `description`/`title` frontmatter values so a translated
* value containing a colon, `#`, or other YAML-significant character does not
* break frontmatter parsing (VitePress loads it as YAML).
* @param {string} text
* @returns {string}
*/
export function quoteFrontmatterScalars(text) {
const { fm, body } = splitFrontmatter(text);
if (fm == null) return text;
const next = fm.split("\n").map((line) => {
const m = line.match(/^(description|title):[ \t]*(.*)$/);
if (!m) return line;
const val = m[2];
// Leave empty, already-quoted, or block-scalar values alone.
if (val === "" || /^["'|>]/.test(val)) return line;
const escaped = val.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
return `${m[1]}: "${escaped}"`;
});
return `---\n${next.join("\n")}\n---\n${body}`;
}
/** Recursively list markdown under a locale dir, relative + POSIX. */
async function listLocaleMarkdown(dir) {
const out = [];
async function walk(d) {
const entries = await readdir(d, { withFileTypes: true });
for (const ent of entries) {
const abs = join(d, ent.name);
if (ent.isDirectory()) await walk(abs);
else if (ent.name.endsWith(".md")) out.push(relative(dir, abs).split(sep).join("/"));
}
}
await walk(dir);
return out.sort();
}
/**
* Build the docs adapter. `root` defaults to apps/docs; tests override it.
* @param {{ root?: string }} [opts]
*/
export function createDocsAdapter({ root = DEFAULT_ROOT } = {}) {
// id -> docs token table, populated by extract() and consumed by write() in the
// same run so the shared translator never needs to know about docs tokens.
const tokenTables = new Map();
return {
name: "docs",
async extract() {
const files = await listRootMarkdown(root);
const units = [];
for (const id of files) {
const abs = join(root, id);
const original = await readFile(abs, "utf8");
const { fm, body } = splitFrontmatter(original);
const anchored = injectAnchors(body);
const rebuilt = fm != null ? `---\n${fm}\n---\n${anchored}` : anchored;
// Persist anchored English source in place (idempotent) so the running
// site and every locale share the same stable slugs.
if (rebuilt !== original) await writeFile(abs, rebuilt, "utf8");
const { masked, tokens } = maskDocs(anchored);
tokenTables.set(id, tokens);
const sourceText = fm != null ? `---\n${fm}\n---\n${masked}` : masked;
units.push({ id, sourceText, kind: "markdown" });
}
return units;
},
async load(locale) {
const dir = join(root, locale);
const map = new Map();
let files = [];
try {
files = await listLocaleMarkdown(dir);
} catch {
return map; // no locale subtree yet
}
for (const rel of files) {
const text = await readFile(join(dir, rel), "utf8");
const { fm } = splitFrontmatter(text);
map.set(rel, {
text,
sourceHash: fmGet(fm, "i18n_source_hash") ?? "",
provenance: fmGet(fm, "i18n_provenance") === "human" ? "human" : "machine",
outputHash: fmGet(fm, "i18n_output_hash") ?? "",
stale: fmGet(fm, "i18n_stale") === "true",
});
}
return map;
},
async write(locale, entries) {
for (const [id, entry] of entries) {
const abs = join(root, locale, id);
await mkdir(dirname(abs), { recursive: true });
const table = tokenTables.get(id) ?? [];
const undone = restoreDocs(entry.text, table);
const linked = quoteFrontmatterScalars(rewriteLinks(undone, locale));
const withMeta = upsertFrontmatter(linked, {
i18n_source_hash: entry.sourceHash,
i18n_provenance: entry.provenance,
i18n_output_hash: entry.outputHash,
...(entry.stale ? { i18n_stale: "true" } : {}),
});
await writeFile(abs, withMeta, "utf8");
}
},
// Adapter extra (not part of the 3-method contract): write the English body
// under the locale path, flagged so a real translation replaces it later.
async writeFallback(locale, id) {
const abs = join(root, locale, id);
const original = await readFile(join(root, id), "utf8");
await mkdir(dirname(abs), { recursive: true });
const linked = rewriteLinks(original, locale);
const flagged = upsertFrontmatter(linked, { i18n_fallback: "true" });
await writeFile(abs, flagged, "utf8");
},
};
}
export const adapter = createDocsAdapter();
+114
View File
@@ -0,0 +1,114 @@
// scripts/i18n/adapters/landing-seo.mjs
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
const DEFAULT_OUT_DIR = join(__dirname, "../../../apps/landing/src/data/i18n");
// Only the alternatives pages are localized. Tool-detail pages are English-only
// (thin-content SEO risk), so their tool-seo prose is never rendered in a locale
// and is intentionally not extracted here.
const ALT_STRING_FIELDS = ["pageTitle", "h1", "metaDescription", "intro", "breadth"];
function pushIfString(units, id, value) {
if (typeof value === "string" && value.trim().length > 0) {
units.push({ id, sourceText: value, kind: "text" });
}
}
function extractAlternatives(units, alternatives) {
for (const alt of alternatives) {
const base = `alt:${alt.slug}`;
for (const field of ALT_STRING_FIELDS) pushIfString(units, `${base}:${field}`, alt[field]);
const rows = Array.isArray(alt.rows) ? alt.rows : [];
rows.forEach((row, i) => {
pushIfString(units, `${base}:rows.${i}.feature`, row.feature);
pushIfString(units, `${base}:rows.${i}.snapotter`, row.snapotter);
pushIfString(units, `${base}:rows.${i}.competitor`, row.competitor);
});
const faqs = Array.isArray(alt.faqs) ? alt.faqs : [];
faqs.forEach((faq, i) => {
pushIfString(units, `${base}:faqs.${i}.q`, faq.q);
pushIfString(units, `${base}:faqs.${i}.a`, faq.a);
});
}
}
async function readJson(path) {
try {
return JSON.parse(await readFile(path, "utf8"));
} catch (err) {
if (err && err.code === "ENOENT") return null;
throw err;
}
}
function altFileFor(dir, locale) {
return join(dir, `alternatives.${locale}.json`);
}
function toStored(record) {
const out = new Map();
for (const [id, e] of Object.entries(record ?? {})) {
out.set(id, {
text: e.text ?? "",
sourceHash: e._sourceHash ?? "",
provenance: e.provenance ?? "machine",
outputHash: e.outputHash ?? "",
stale: Boolean(e.stale),
});
}
return out;
}
function toRecord(entries) {
const record = {};
for (const id of [...entries.keys()].sort()) {
const e = entries.get(id);
record[id] = {
text: e.text,
_sourceHash: e.sourceHash,
provenance: e.provenance,
outputHash: e.outputHash,
stale: Boolean(e.stale),
};
}
return record;
}
/**
* Build the landing SEO data adapter (alternatives pages only).
* @param {{
* dir?: string,
* alternatives?: any[],
* }} [opts]
*/
export function makeLandingSeoAdapter(opts = {}) {
const dir = opts.dir ?? DEFAULT_OUT_DIR;
return {
name: "landing-seo",
async extract() {
const alternatives =
opts.alternatives ??
(await import("../../../apps/landing/src/data/alternatives.ts")).ALTERNATIVES;
const units = [];
extractAlternatives(units, alternatives);
return units;
},
async load(locale) {
const alt = await readJson(altFileFor(dir, locale));
return toStored(alt);
},
async write(locale, entries) {
const altRecord = toRecord(entries);
await mkdir(dir, { recursive: true });
await writeFile(altFileFor(dir, locale), `${JSON.stringify(altRecord, null, 2)}\n`);
},
};
}
export const adapter = makeLandingSeoAdapter();
+75
View File
@@ -0,0 +1,75 @@
// scripts/i18n/adapters/landing-ui.mjs
import { readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
const DEFAULT_DIR = join(__dirname, "../../../apps/landing/src/i18n");
async function readJson(path) {
try {
return JSON.parse(await readFile(path, "utf8"));
} catch (err) {
if (err && err.code === "ENOENT") return null;
throw err;
}
}
/**
* Build the landing UI catalog adapter.
* Runtime catalog: `<locale>.json` (flat { key: text }).
* Sidecar metadata: `<locale>.meta.json` ({ key: { sourceHash, provenance, outputHash, stale } }).
* @param {{ dir?: string }} [opts]
*/
export function makeLandingUiAdapter({ dir = DEFAULT_DIR } = {}) {
const enPath = join(dir, "en.json");
return {
name: "landing-ui",
async extract() {
const en = (await readJson(enPath)) ?? {};
return Object.entries(en).map(([id, sourceText]) => ({
id,
sourceText: String(sourceText),
kind: "text",
}));
},
async load(locale) {
const catalog = (await readJson(join(dir, `${locale}.json`))) ?? {};
const meta = (await readJson(join(dir, `${locale}.meta.json`))) ?? {};
const out = new Map();
for (const [id, text] of Object.entries(catalog)) {
const m = meta[id] ?? {};
out.set(id, {
text: String(text),
sourceHash: m.sourceHash ?? "",
provenance: m.provenance ?? "machine",
outputHash: m.outputHash ?? "",
stale: Boolean(m.stale),
});
}
return out;
},
async write(locale, entries) {
const catalog = {};
const meta = {};
// Stable key order for reviewable diffs.
for (const id of [...entries.keys()].sort()) {
const e = entries.get(id);
catalog[id] = e.text;
meta[id] = {
sourceHash: e.sourceHash,
provenance: e.provenance,
outputHash: e.outputHash,
stale: Boolean(e.stale),
};
}
await writeFile(join(dir, `${locale}.json`), `${JSON.stringify(catalog, null, 2)}\n`);
await writeFile(join(dir, `${locale}.meta.json`), `${JSON.stringify(meta, null, 2)}\n`);
},
};
}
export const adapter = makeLandingUiAdapter();
+23
View File
@@ -0,0 +1,23 @@
// scripts/i18n/adapters/registry.mjs
// Single source of truth for surface adapters. Both translate.mjs and
// check-parity.mjs import this so the surface list is defined once.
// Surface plans (02-04) uncomment/add their entry as each adapter lands.
export const ADAPTERS = {
"landing-ui": () => import("./landing-ui.mjs"),
"landing-seo": () => import("./landing-seo.mjs"),
docs: () => import("./docs-md.mjs"),
api: () => import("./api-spec.mjs"),
};
/**
* @param {string} spec "all" or a comma list of surface keys
* @returns {string[]} known surface keys
*/
export function resolveSurfaces(spec) {
const keys = Object.keys(ADAPTERS);
if (spec === "all") return keys;
return spec
.split(",")
.map((s) => s.trim())
.filter((k) => keys.includes(k));
}