mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
fix: release QA hardening across processing, media, security, and CI gates (#649)
A release-readiness QA pass over the whole product. The commits split into defects a user would hit and gates that were reporting green while measuring nothing. ## Fixes that change behaviour Rate limiting was bypassable on every install: TRUST_PROXY defaulted to true, so request.ip came from a client-set header and a forged X-Forwarded-For got past the login limiter. The default is now a private-network trust list. A transient Postgres outage stranded in-flight jobs, leaving finished output on disk with no row pointing at it. A reconciler now resolves those rows and adopts the bytes rather than dropping the work. A Redis connection that moved to a new address wedged every read-blocked consumer, so completions stopped signalling while health still answered 200. Socket timeouts plus subscriber pings recover it. Installing more than one AI bundle left the shared venv multi-versioned and silently broke three tools. The installer now reconciles distributions to one version each. Converting an image to JXL at quality 1 through 4 returned a 500, because libjxl 0.7 rejects the distance those values compute. The quality is floored at what the encoder honours. A missing ffmpeg was also reported to the user as a corrupt upload; it now says the engine is unavailable. RAW uploads reached an unpatched LibRaw on arm64, so it is built from source at 0.22.2, and the release scan was split so it can fail on an unfixed critical instead of hiding it behind ignore-unfixed. ## Gates that could not fail Two mutation lanes ran zero mutants because Stryker crawled the gitignored docs build; coverage discarded its whole report on any failing test; the lint gate skipped root tests, scripts, and two workspaces; and several generated matrices counted a host missing ffmpeg as a passing tool. Each now measures what it claims. Full evidence and the outstanding release items are tracked locally and are not part of this branch.
This commit is contained in:
SnapOtter
@@ -23,7 +23,13 @@ const out = resolve(repoRoot, "branding/social-preview.png");
|
||||
const htmlPath = resolve(repoRoot, "branding/.og.html");
|
||||
|
||||
// Trust badges (subset of Hero.astro's trustBadges list).
|
||||
const BADGES = ["Self-hosted", "Privacy-sensitive", "Compliance-friendly", "Air-gap capable", "Open source"];
|
||||
const BADGES = [
|
||||
"Self-hosted",
|
||||
"Privacy-sensitive",
|
||||
"Compliance-friendly",
|
||||
"Air-gap capable",
|
||||
"Open source",
|
||||
];
|
||||
|
||||
// lucide icon inner-SVG (stroke), matching the icons used in CategoryCards.astro.
|
||||
const ICON = {
|
||||
@@ -141,14 +147,14 @@ try {
|
||||
const browser = await chromium.launch();
|
||||
const ctx = await browser.newContext({ viewport: { width: 1280, height: 640 } });
|
||||
const page = await ctx.newPage();
|
||||
await page.goto("file://" + htmlPath, { waitUntil: "load" });
|
||||
await page.goto(`file://${htmlPath}`, { waitUntil: "load" });
|
||||
await page.evaluate(async () => {
|
||||
await document.fonts.ready;
|
||||
});
|
||||
await page.waitForTimeout(150);
|
||||
await page.screenshot({ path: out });
|
||||
await browser.close();
|
||||
console.log("wrote " + out);
|
||||
console.log(`wrote ${out}`);
|
||||
} finally {
|
||||
rmSync(htmlPath, { force: true });
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -148,7 +148,7 @@ function generateTags(name) {
|
||||
|
||||
async function downloadImage(url, dest) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error("HTTP " + res.status);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
writeFileSync(dest, buf);
|
||||
}
|
||||
@@ -167,7 +167,7 @@ async function main() {
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
const filename = slug + ".jpg";
|
||||
const filename = `${slug}.jpg`;
|
||||
const destPath = join(FULL_DIR, filename);
|
||||
|
||||
if (!existsSync(destPath)) {
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const policyPath = path.join(root, "config/production-license-policy.json");
|
||||
const noticesPath = path.join(root, "THIRD_PARTY_NOTICES.md");
|
||||
|
||||
function packageLabels(entries) {
|
||||
return entries
|
||||
.flatMap((entry) => {
|
||||
const versions = Array.isArray(entry.versions)
|
||||
? [...entry.versions].map(String).sort().join(",")
|
||||
: "unknown";
|
||||
return `${String(entry.name ?? "unnamed")}@${versions}`;
|
||||
})
|
||||
.sort();
|
||||
}
|
||||
|
||||
export function validateInventory(inventory, policy) {
|
||||
const allowed = new Set(policy.allowedExpressions ?? []);
|
||||
const denied = new Set(policy.deniedExpressions ?? []);
|
||||
const violations = [];
|
||||
|
||||
for (const expression of Object.keys(inventory).sort()) {
|
||||
const entries = Array.isArray(inventory[expression]) ? inventory[expression] : [];
|
||||
const packages = packageLabels(entries).join(", ");
|
||||
const declaredLicenses = new Set(entries.map((entry) => entry.license));
|
||||
if (declaredLicenses.size !== 1 || !declaredLicenses.has(expression)) {
|
||||
violations.push(`inventory group ${expression} has inconsistent package license metadata`);
|
||||
}
|
||||
if (/^(unknown|unlicensed)$/i.test(expression)) {
|
||||
violations.push(`unknown license expression ${expression}: ${packages}`);
|
||||
} else if (denied.has(expression)) {
|
||||
violations.push(`denied license expression ${expression}: ${packages}`);
|
||||
} else if (!allowed.has(expression)) {
|
||||
violations.push(`unapproved license expression ${expression}: ${packages}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const expression of denied) {
|
||||
if (allowed.has(expression)) {
|
||||
violations.push(`policy expression is both allowed and denied: ${expression}`);
|
||||
}
|
||||
}
|
||||
return violations.sort();
|
||||
}
|
||||
|
||||
// Native bindings ship one package per platform and arch, so `pnpm licenses
|
||||
// list` returns whichever set the current machine installed. Rendering those
|
||||
// verbatim makes the notices file describe the developer's laptop: it is
|
||||
// regenerated on macOS, then the Linux CI runner resolves the x64 bindings
|
||||
// instead and fails the very check that produced it.
|
||||
//
|
||||
// Collapse each family to its base name for the notices. The policy check above
|
||||
// still sees every package the current platform installed, so an unacceptable
|
||||
// license in a binding is caught wherever it is installed; only the attribution
|
||||
// list is normalized, and the upstream project is still credited once.
|
||||
// Three shapes in the wild: a suffix on the package name
|
||||
// (@img/sharp-darwin-arm64), the libc glued to the OS (@img/sharp-linuxmusl-arm64),
|
||||
// and the whole unscoped name (@esbuild/darwin-arm64), which collapses to the
|
||||
// scope alone.
|
||||
const PLATFORM_SUFFIX =
|
||||
/[-/](?:darwin|linux(?:musl)?|win32|freebsd|openbsd|android|sunos)(?:-(?:x64|arm64|arm|ia32|ppc64|s390x|riscv64|loong64))?(?:-(?:musl|gnu|gnueabihf|msvc))?$/;
|
||||
|
||||
// Packages that only exist on one platform, rather than shipping a build per
|
||||
// platform. fsevents is macOS-only by design and simply absent on Linux, so it
|
||||
// cannot be normalized into a shared family and has to be named.
|
||||
const PLATFORM_ONLY_PACKAGES = new Set(["fsevents"]);
|
||||
|
||||
function platformFamily(name) {
|
||||
return String(name).replace(PLATFORM_SUFFIX, "");
|
||||
}
|
||||
|
||||
export function renderNotices(inventory) {
|
||||
const lines = [
|
||||
"# Third-Party Production Node Dependency Notices",
|
||||
"",
|
||||
"This file is generated from the frozen pnpm production dependency graph.",
|
||||
"This inventory covers Node packages only; exact release artifacts publish separate full SBOMs.",
|
||||
"Run `pnpm check:production-node-licenses -- --write-notices` after an intentional dependency change.",
|
||||
"Package authors retain all rights granted by their respective licenses.",
|
||||
"",
|
||||
];
|
||||
|
||||
for (const expression of Object.keys(inventory).sort()) {
|
||||
lines.push(`## ${expression}`, "");
|
||||
const families = new Map();
|
||||
for (const entry of inventory[expression]) {
|
||||
if (PLATFORM_ONLY_PACKAGES.has(entry.name)) continue;
|
||||
const name = platformFamily(entry.name);
|
||||
const family = families.get(name) ?? { name, versions: new Set(), homepage: entry.homepage };
|
||||
for (const version of entry.versions) family.versions.add(String(version));
|
||||
family.homepage ??= entry.homepage;
|
||||
families.set(name, family);
|
||||
}
|
||||
const entries = [...families.values()].sort((left, right) =>
|
||||
left.name.localeCompare(right.name),
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const versions = [...entry.versions].sort().join(",");
|
||||
const label = `${entry.name}@${versions}`;
|
||||
lines.push(entry.homepage ? `- [${label}](${entry.homepage})` : `- ${label}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
return `${lines.join("\n").trimEnd()}\n`;
|
||||
}
|
||||
|
||||
function readPolicy() {
|
||||
const policy = JSON.parse(readFileSync(policyPath, "utf8"));
|
||||
if (
|
||||
policy.schemaVersion !== 1 ||
|
||||
!Array.isArray(policy.allowedExpressions) ||
|
||||
!Array.isArray(policy.deniedExpressions)
|
||||
) {
|
||||
throw new Error("production license policy has an unsupported schema");
|
||||
}
|
||||
return policy;
|
||||
}
|
||||
|
||||
function readInventory() {
|
||||
return JSON.parse(
|
||||
execFileSync("pnpm", ["licenses", "list", "--prod", "--json"], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function main() {
|
||||
const policy = readPolicy();
|
||||
const inventory = readInventory();
|
||||
const violations = validateInventory(inventory, policy);
|
||||
if (violations.length > 0) {
|
||||
for (const violation of violations) console.error(`ERROR: ${violation}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const expectedNotices = renderNotices(inventory);
|
||||
if (process.argv.includes("--write-notices")) {
|
||||
writeFileSync(noticesPath, expectedNotices);
|
||||
console.log(`Updated ${path.relative(root, noticesPath)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!existsSync(noticesPath) || readFileSync(noticesPath, "utf8") !== expectedNotices) {
|
||||
console.error(
|
||||
"ERROR: THIRD_PARTY_NOTICES.md is stale; review the dependency change, then run pnpm check:production-node-licenses -- --write-notices",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const packageCount = Object.values(inventory).reduce((sum, entries) => sum + entries.length, 0);
|
||||
console.log(
|
||||
`Production Node license policy passed: ${packageCount} packages across ${Object.keys(inventory).length} expressions`,
|
||||
);
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main();
|
||||
+239
-132
@@ -5,8 +5,8 @@
|
||||
* Usage: node scripts/codemod-fixture-refs.mjs [--dry-run]
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync } from "node:fs";
|
||||
import { join, relative, dirname } from "node:path";
|
||||
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, relative } from "node:path";
|
||||
|
||||
const DRY_RUN = process.argv.includes("--dry-run");
|
||||
const ROOT = process.cwd();
|
||||
@@ -99,8 +99,19 @@ const PATH_TO_REGISTRY = {
|
||||
"security/svg-xxe-ssrf.svg": "fixtures.security.svgXxeSsrf",
|
||||
};
|
||||
|
||||
const VIDEO_EXTS = new Set(["mp4", "mov", "webm", "mkv", "avi", "flv", "wmv", "m4v", "mpg", "mpeg", "ogv", "3gp", "m2ts", "mts"]);
|
||||
const AUDIO_EXTS = new Set(["mp3", "wav", "flac", "ogg", "m4a", "aac", "opus", "wma", "aiff", "amr", "ac3"]);
|
||||
const AUDIO_EXTS = new Set([
|
||||
"mp3",
|
||||
"wav",
|
||||
"flac",
|
||||
"ogg",
|
||||
"m4a",
|
||||
"aac",
|
||||
"opus",
|
||||
"wma",
|
||||
"aiff",
|
||||
"amr",
|
||||
"ac3",
|
||||
]);
|
||||
|
||||
function tinyAccessor(dir, file) {
|
||||
if (!file.startsWith("tiny.")) return null;
|
||||
@@ -158,7 +169,9 @@ function collectTestFiles(dir) {
|
||||
result.push(full);
|
||||
}
|
||||
}
|
||||
} catch (_) { /* dir doesn't exist */ }
|
||||
} catch (_) {
|
||||
/* dir doesn't exist */
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -181,7 +194,7 @@ for (const file of allTestFiles) {
|
||||
let needsFixtureRoot = false;
|
||||
const fileDir = dirname(file);
|
||||
let relPath = relative(fileDir, join(ROOT, "tests/fixtures/index.js"));
|
||||
if (!relPath.startsWith(".")) relPath = "./" + relPath;
|
||||
if (!relPath.startsWith(".")) relPath = `./${relPath}`;
|
||||
|
||||
// ── PASS 1: Inline buffer reads ────────────────────────────
|
||||
// readFileSync(join(__dirname, "..", "fixtures", "dir", "file"))
|
||||
@@ -192,18 +205,26 @@ for (const file of allTestFiles) {
|
||||
/readFileSync\(join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)",\s*"([^"]+)"\)\)/g,
|
||||
(match, dir, file) => {
|
||||
const reg = resolveFixturePath([dir, file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
// One segment inline buffer read
|
||||
src = src.replace(
|
||||
/readFileSync\(join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)"\)\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// await readFile(join(__dirname, "..", "fixtures", ...))
|
||||
@@ -211,17 +232,25 @@ for (const file of allTestFiles) {
|
||||
/await readFile\(join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)",\s*"([^"]+)"\)\)/g,
|
||||
(match, dir, file) => {
|
||||
const reg = resolveFixturePath([dir, file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/await readFile\(join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)"\)\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ── PASS 1b: Inline path-only refs ─────────────────────────
|
||||
@@ -230,18 +259,21 @@ for (const file of allTestFiles) {
|
||||
/join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)",\s*"([^"]+)"\)/g,
|
||||
(match, dir, file) => {
|
||||
const reg = resolveFixturePath([dir, file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)"\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
return match;
|
||||
src = src.replace(/join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"([^"]+)"\)/g, (match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
|
||||
// ── PASS 1c: Inline dir-level refs ─────────────────────────
|
||||
// join(__dirname, "..", "fixtures", "formats") -> fixtureDir.formats
|
||||
@@ -249,20 +281,26 @@ for (const file of allTestFiles) {
|
||||
// join(__dirname, "..", "fixtures") -> fixtureRoot
|
||||
src = src.replace(
|
||||
/join\(__dirname,\s*"[^"]+",\s*"fixtures",\s*"(formats|hostile|media|documents|data|content|security)"\)/g,
|
||||
(match, dir) => { needsFixtureDir = true; return `fixtureDir.${dir}`; }
|
||||
(_match, dir) => {
|
||||
needsFixtureDir = true;
|
||||
return `fixtureDir.${dir}`;
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(__dirname,\s*"[^"]+",\s*"[^"]+",\s*"fixtures",\s*"(formats|hostile|media|documents|data|content|security)"\)/g,
|
||||
(match, dir) => { needsFixtureDir = true; return `fixtureDir.${dir}`; }
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(__dirname,\s*"[^"]+",\s*"fixtures"\)/g,
|
||||
() => { needsFixtureRoot = true; return "fixtureRoot"; }
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(__dirname,\s*"[^"]+",\s*"[^"]+",\s*"fixtures"\)/g,
|
||||
() => { needsFixtureRoot = true; return "fixtureRoot"; }
|
||||
(_match, dir) => {
|
||||
needsFixtureDir = true;
|
||||
return `fixtureDir.${dir}`;
|
||||
},
|
||||
);
|
||||
src = src.replace(/join\(__dirname,\s*"[^"]+",\s*"fixtures"\)/g, () => {
|
||||
needsFixtureRoot = true;
|
||||
return "fixtureRoot";
|
||||
});
|
||||
src = src.replace(/join\(__dirname,\s*"[^"]+",\s*"[^"]+",\s*"fixtures"\)/g, () => {
|
||||
needsFixtureRoot = true;
|
||||
return "fixtureRoot";
|
||||
});
|
||||
|
||||
// ── PASS 2: FIXTURES/FIXTURES_DIR const-level patterns ─────
|
||||
// readFileSync(join(FIXTURES, "dir", "file"))
|
||||
@@ -270,18 +308,26 @@ for (const file of allTestFiles) {
|
||||
/(?:readFileSync|await readFile)\(join\((?:FIXTURES|FIXTURES_DIR|FIXTURES_ROOT),\s*"([^"]+)",\s*"([^"]+)"\)\)/g,
|
||||
(match, dir, file) => {
|
||||
const reg = resolveFixturePath([dir, file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
// readFileSync(join(FIXTURES, "file"))
|
||||
src = src.replace(
|
||||
/(?:readFileSync|await readFile)\(join\((?:FIXTURES|FIXTURES_DIR|FIXTURES_ROOT),\s*"([^"]+)"\)\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// readFileSync(join(FORMATS_DIR, "sample.ext"))
|
||||
@@ -289,18 +335,23 @@ for (const file of allTestFiles) {
|
||||
/readFileSync\((?:path\.)?join\(FORMATS_DIR,\s*"([^"]+)"\)\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath(["formats", file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// readFileSync(join(FORMATS, `sample.${ext}`))
|
||||
src = src.replace(
|
||||
/readFileSync\(join\(FORMATS,\s*`sample\.\$\{([^}]+)\}`\)\)/g,
|
||||
(match, ext) => {
|
||||
needsFixtures = true; needsReadFixture = true;
|
||||
(_match, ext) => {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(fixtures.image.formats(${ext}))`;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ── PASS 3: path-only join(FIXTURES, ...) ──────────────────
|
||||
@@ -309,44 +360,52 @@ for (const file of allTestFiles) {
|
||||
/join\((?:FIXTURES|FIXTURES_DIR|FIXTURES_ROOT),\s*"([^"]+)",\s*"([^"]+)"\)/g,
|
||||
(match, dir, file) => {
|
||||
const reg = resolveFixturePath([dir, file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
// join(FIXTURES, "file")
|
||||
src = src.replace(
|
||||
/join\((?:FIXTURES|FIXTURES_DIR|FIXTURES_ROOT),\s*"([^"]+)"\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath([file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
// Directory refs
|
||||
if (["formats", "hostile", "media", "documents", "data", "content", "security"].includes(file)) {
|
||||
if (
|
||||
["formats", "hostile", "media", "documents", "data", "content", "security"].includes(file)
|
||||
) {
|
||||
needsFixtureDir = true;
|
||||
return `fixtureDir.${file}`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ── PASS 4: FORMATS_DIR path-only ──────────────────────────
|
||||
src = src.replace(
|
||||
/(?:path\.)?join\(FORMATS_DIR,\s*"([^"]+)"\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath(["formats", file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
return match;
|
||||
src = src.replace(/(?:path\.)?join\(FORMATS_DIR,\s*"([^"]+)"\)/g, (match, file) => {
|
||||
const reg = resolveFixturePath(["formats", file]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
|
||||
// ── PASS 5: MEDIA path refs ────────────────────────────────
|
||||
src = src.replace(
|
||||
/join\(MEDIA,\s*"([^"]+)"\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath(["media", file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
return match;
|
||||
src = src.replace(/join\(MEDIA,\s*"([^"]+)"\)/g, (match, file) => {
|
||||
const reg = resolveFixturePath(["media", file]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
|
||||
// ── PASS 6: process.cwd() patterns ─────────────────────────
|
||||
src = src.replace(
|
||||
@@ -354,96 +413,129 @@ for (const file of allTestFiles) {
|
||||
(match, path) => {
|
||||
const segments = path.split("/");
|
||||
const reg = resolveFixturePath(segments);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(process\.cwd\(\),\s*"tests\/fixtures\/([^"]+)"\)/g,
|
||||
(match, path) => {
|
||||
const segments = path.split("/");
|
||||
const reg = resolveFixturePath(segments);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
return match;
|
||||
src = src.replace(/join\(process\.cwd\(\),\s*"tests\/fixtures\/([^"]+)"\)/g, (match, path) => {
|
||||
const segments = path.split("/");
|
||||
const reg = resolveFixturePath(segments);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
|
||||
// ── PASS 7: path.resolve(__dirname, "../../fixtures/formats") etc ──
|
||||
src = src.replace(
|
||||
/path\.resolve\(__dirname,\s*"[^"]*fixtures\/formats"\)/g,
|
||||
() => { needsFixtureDir = true; return "fixtureDir.formats"; }
|
||||
);
|
||||
src = src.replace(
|
||||
/path\.resolve\(__dirname,\s*"[^"]*fixtures"\)/g,
|
||||
() => { needsFixtureRoot = true; return "fixtureRoot"; }
|
||||
);
|
||||
src = src.replace(/path\.resolve\(__dirname,\s*"[^"]*fixtures\/formats"\)/g, () => {
|
||||
needsFixtureDir = true;
|
||||
return "fixtureDir.formats";
|
||||
});
|
||||
src = src.replace(/path\.resolve\(__dirname,\s*"[^"]*fixtures"\)/g, () => {
|
||||
needsFixtureRoot = true;
|
||||
return "fixtureRoot";
|
||||
});
|
||||
|
||||
// ── PASS 8: path.join(FIXTURES_DIR, "file") in unit tests ──
|
||||
src = src.replace(
|
||||
/readFileSync\(path\.join\((?:FIXTURES_DIR|fixtureRoot),\s*"([^"]+)"\)\)/g,
|
||||
(match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
// formats/sample.xxx as single string
|
||||
const segs = p.split("/");
|
||||
if (segs.length === 2) {
|
||||
const r2 = resolveFixturePath(segs);
|
||||
if (r2) { needsFixtures = true; needsReadFixture = true; return `readFixture(${r2})`; }
|
||||
if (r2) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${r2})`;
|
||||
}
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/path\.join\((?:FIXTURES_DIR|fixtureRoot),\s*"([^"]+)"\)/g,
|
||||
(match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
const segs = p.split("/");
|
||||
if (segs.length === 2) {
|
||||
const r2 = resolveFixturePath(segs);
|
||||
if (r2) { needsFixtures = true; return r2; }
|
||||
src = src.replace(/path\.join\((?:FIXTURES_DIR|fixtureRoot),\s*"([^"]+)"\)/g, (match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
const segs = p.split("/");
|
||||
if (segs.length === 2) {
|
||||
const r2 = resolveFixturePath(segs);
|
||||
if (r2) {
|
||||
needsFixtures = true;
|
||||
return r2;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
|
||||
// readFileSync(path.join(FORMATS_DIR|fixtureDir.formats, "sample.ext"))
|
||||
src = src.replace(
|
||||
/readFileSync\(path\.join\((?:FORMATS_DIR|fixtureDir\.formats),\s*"([^"]+)"\)\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath(["formats", file]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
src = src.replace(
|
||||
/path\.join\((?:FORMATS_DIR|fixtureDir\.formats),\s*"([^"]+)"\)/g,
|
||||
(match, file) => {
|
||||
const reg = resolveFixturePath(["formats", file]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// ── PASS 9: FIXTURE singular (media-engine.test.ts) ────────
|
||||
src = src.replace(
|
||||
/const FIXTURE = join\(process\.cwd\(\), "tests\/fixtures\/media\/tiny\.mp4"\);?\n?/g,
|
||||
() => { needsFixtures = true; return ""; }
|
||||
() => {
|
||||
needsFixtures = true;
|
||||
return "";
|
||||
},
|
||||
);
|
||||
// Replace FIXTURE usage (only if the def was removed)
|
||||
if (origSrc.includes('const FIXTURE = join(process.cwd(), "tests/fixtures/media/tiny.mp4")') &&
|
||||
!src.includes("const FIXTURE")) {
|
||||
if (
|
||||
origSrc.includes('const FIXTURE = join(process.cwd(), "tests/fixtures/media/tiny.mp4")') &&
|
||||
!src.includes("const FIXTURE")
|
||||
) {
|
||||
src = src.replace(/\bFIXTURE\b/g, 'fixtures.video.tiny("mp4")');
|
||||
}
|
||||
|
||||
// ── PASS 10: Remove now-orphaned const defs ────────────────
|
||||
// Remove const FIXTURES = join(...) if FIXTURES is no longer used
|
||||
const removeConstIfOrphaned = (constName, replacement) => {
|
||||
const removeConstIfOrphaned = (constName, _replacement) => {
|
||||
const defPatterns = [
|
||||
new RegExp(`const ${constName} = join\\(__dirname,\\s*"[^"]+",\\s*"fixtures"\\);?\\n?`, "g"),
|
||||
new RegExp(`const ${constName} = join\\(__dirname,\\s*"[^"]+",\\s*"[^"]+",\\s*"fixtures"\\);?\\n?`, "g"),
|
||||
new RegExp(
|
||||
`const ${constName} = join\\(__dirname,\\s*"[^"]+",\\s*"[^"]+",\\s*"fixtures"\\);?\\n?`,
|
||||
"g",
|
||||
),
|
||||
new RegExp(`const ${constName} = join\\(__dirname,\\s*"[^"]*fixtures"\\);?\\n?`, "g"),
|
||||
new RegExp(`const ${constName} = join\\(import\\.meta\\.dirname,\\s*"[^"]+",\\s*"[^"]+",\\s*"fixtures"\\);?\\n?`, "g"),
|
||||
new RegExp(
|
||||
`const ${constName} = join\\(import\\.meta\\.dirname,\\s*"[^"]+",\\s*"[^"]+",\\s*"fixtures"\\);?\\n?`,
|
||||
"g",
|
||||
),
|
||||
new RegExp(`const ${constName} = join\\(import\\.meta\\.dirname,\\s*"[^"]+"\\);?\\n?`, "g"),
|
||||
];
|
||||
for (const pat of defPatterns) {
|
||||
@@ -517,7 +609,10 @@ for (const file of allTestFiles) {
|
||||
src = src.replace(/const FORMATS = join\(FIXTURES, "formats"\);?\n?/g, () => {
|
||||
return "";
|
||||
});
|
||||
if (origSrc.includes('const FORMATS = join(FIXTURES, "formats")') && !src.includes("const FORMATS")) {
|
||||
if (
|
||||
origSrc.includes('const FORMATS = join(FIXTURES, "formats")') &&
|
||||
!src.includes("const FORMATS")
|
||||
) {
|
||||
if (/\bFORMATS\b/.test(src.replace(/\bFORMATS_/g, "").replace(/"FORMATS/g, ""))) {
|
||||
src = src.replace(/\bFORMATS\b(?!_|")/g, "fixtureDir.formats");
|
||||
needsFixtureDir = true;
|
||||
@@ -525,7 +620,11 @@ for (const file of allTestFiles) {
|
||||
}
|
||||
|
||||
// Remove MEDIA_DIR, DOCUMENTS_DIR, DATA_DIR from multimodal etc
|
||||
for (const [vname, dir] of [["MEDIA_DIR", "media"], ["DOCUMENTS_DIR", "documents"], ["DATA_DIR", "data"]]) {
|
||||
for (const [vname, dir] of [
|
||||
["MEDIA_DIR", "media"],
|
||||
["DOCUMENTS_DIR", "documents"],
|
||||
["DATA_DIR", "data"],
|
||||
]) {
|
||||
const p1 = new RegExp(`const ${vname} = join\\(FIXTURES_ROOT, "${dir}"\\);?\\n?`, "g");
|
||||
const p2 = new RegExp(`const ${vname} = join\\(fixtureRoot, "${dir}"\\);?\\n?`, "g");
|
||||
for (const p of [p1, p2]) {
|
||||
@@ -590,25 +689,30 @@ for (const file of allTestFiles) {
|
||||
}
|
||||
|
||||
// Remaining FIXTURES usages
|
||||
if (src.includes("FIXTURES") && !src.includes("const FIXTURES") &&
|
||||
!src.includes("fixtures.") && !src.includes("BY_EXT")) {
|
||||
if (
|
||||
src.includes("FIXTURES") &&
|
||||
!src.includes("const FIXTURES") &&
|
||||
!src.includes("fixtures.") &&
|
||||
!src.includes("BY_EXT")
|
||||
) {
|
||||
// Check for lingering join(FIXTURES, ...) that wasn't caught
|
||||
src = src.replace(
|
||||
/readFileSync\(join\(FIXTURES,\s*"([^"]+)"\)\)/g,
|
||||
(match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) { needsFixtures = true; needsReadFixture = true; return `readFixture(${reg})`; }
|
||||
return match;
|
||||
src = src.replace(/readFileSync\(join\(FIXTURES,\s*"([^"]+)"\)\)/g, (match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
needsReadFixture = true;
|
||||
return `readFixture(${reg})`;
|
||||
}
|
||||
);
|
||||
src = src.replace(
|
||||
/join\(FIXTURES,\s*"([^"]+)"\)/g,
|
||||
(match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) { needsFixtures = true; return reg; }
|
||||
return match;
|
||||
return match;
|
||||
});
|
||||
src = src.replace(/join\(FIXTURES,\s*"([^"]+)"\)/g, (match, p) => {
|
||||
const reg = resolveFixturePath([p]);
|
||||
if (reg) {
|
||||
needsFixtures = true;
|
||||
return reg;
|
||||
}
|
||||
);
|
||||
return match;
|
||||
});
|
||||
}
|
||||
|
||||
if (src === origSrc) continue;
|
||||
@@ -624,10 +728,7 @@ for (const file of allTestFiles) {
|
||||
const importLine = `import { ${importParts.join(", ")} } from "${relPath}";`;
|
||||
|
||||
if (src.includes('fixtures/index.js"')) {
|
||||
src = src.replace(
|
||||
/import \{[^}]+\} from "[^"]*fixtures\/index\.js";?/,
|
||||
importLine
|
||||
);
|
||||
src = src.replace(/import \{[^}]+\} from "[^"]*fixtures\/index\.js";?/, importLine);
|
||||
} else {
|
||||
const lines = src.split("\n");
|
||||
let lastImportIdx = -1;
|
||||
@@ -640,7 +741,7 @@ for (const file of allTestFiles) {
|
||||
lines.splice(lastImportIdx + 1, 0, importLine);
|
||||
src = lines.join("\n");
|
||||
} else {
|
||||
src = importLine + "\n" + src;
|
||||
src = `${importLine}\n${src}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -650,10 +751,13 @@ for (const file of allTestFiles) {
|
||||
if (!/readFileSync\s*\(/.test(src) && /import\s*\{[^}]*readFileSync/.test(src)) {
|
||||
src = src.replace(
|
||||
/(import\s*\{[^}]*)readFileSync,?\s*([^}]*\}\s*from\s*"node:fs")/,
|
||||
(match, before, after) => {
|
||||
const cleaned = (before + after).replace(/,\s*,/g, ",").replace(/\{\s*,/g, "{ ").replace(/,\s*\}/g, " }");
|
||||
(_match, before, after) => {
|
||||
const cleaned = (before + after)
|
||||
.replace(/,\s*,/g, ",")
|
||||
.replace(/\{\s*,/g, "{ ")
|
||||
.replace(/,\s*\}/g, " }");
|
||||
return cleaned;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -661,10 +765,13 @@ for (const file of allTestFiles) {
|
||||
if (!/\bjoin\s*\(/.test(src) && /import\s*\{[^}]*\bjoin\b/.test(src)) {
|
||||
src = src.replace(
|
||||
/(import\s*\{[^}]*)\bjoin\b,?\s*([^}]*\}\s*from\s*"node:path")/,
|
||||
(match, before, after) => {
|
||||
const cleaned = (before + after).replace(/,\s*,/g, ",").replace(/\{\s*,/g, "{ ").replace(/,\s*\}/g, " }");
|
||||
(_match, before, after) => {
|
||||
const cleaned = (before + after)
|
||||
.replace(/,\s*,/g, ",")
|
||||
.replace(/\{\s*,/g, "{ ")
|
||||
.replace(/,\s*\}/g, " }");
|
||||
return cleaned;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,16 @@ import { fileURLToPath } from "node:url";
|
||||
import AdmZip from "adm-zip";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const outPath = join(__dirname, "..", "..", "tests", "fixtures", "document", "formats", "tiny.pptx");
|
||||
const outPath = join(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"tests",
|
||||
"fixtures",
|
||||
"document",
|
||||
"formats",
|
||||
"tiny.pptx",
|
||||
);
|
||||
|
||||
const zip = new AdmZip();
|
||||
|
||||
|
||||
@@ -18,7 +18,9 @@ const outDir = join(root, "tests", "fixtures", "image", "hostile");
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
|
||||
// --- 1. truncated.jpg: a real JPEG cut off at 40% ---------------------------
|
||||
const realJpeg = readFileSync(join(root, "tests", "fixtures", "image", "valid", "sample-photo.jpg"));
|
||||
const realJpeg = readFileSync(
|
||||
join(root, "tests", "fixtures", "image", "valid", "sample-photo.jpg"),
|
||||
);
|
||||
writeFileSync(
|
||||
join(outDir, "truncated.jpg"),
|
||||
realJpeg.subarray(0, Math.floor(realJpeg.length * 0.4)),
|
||||
|
||||
@@ -163,9 +163,13 @@ if (existsSync(encPdf)) {
|
||||
const qpdf = whichBin("QPDF_PATH", "qpdf");
|
||||
if (qpdf) {
|
||||
const srcPdf = join(root, "tests/fixtures/document/valid/test-3page.pdf");
|
||||
const qres = spawnSync(qpdf, [srcPdf, "--encrypt", "test123", "owner123", "256", "--", encPdf], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
const qres = spawnSync(
|
||||
qpdf,
|
||||
[srcPdf, "--encrypt", "test123", "owner123", "256", "--", encPdf],
|
||||
{
|
||||
stdio: "inherit",
|
||||
},
|
||||
);
|
||||
if (qres.status !== 0) {
|
||||
console.error("qpdf encryption failed");
|
||||
process.exit(1);
|
||||
@@ -200,4 +204,6 @@ A paragraph with **bold** and a list:
|
||||
`,
|
||||
);
|
||||
|
||||
console.log("Fixtures written to tests/fixtures/{video,audio}/formats and tests/fixtures/document/formats");
|
||||
console.log(
|
||||
"Fixtures written to tests/fixtures/{video,audio}/formats and tests/fixtures/document/formats",
|
||||
);
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, relative, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { hash } from "../lib/hash.mjs";
|
||||
import { localeCodes } from "../lib/shared-i18n.mjs";
|
||||
import { slugify } from "../lib/slugify.mjs";
|
||||
|
||||
@@ -213,6 +214,22 @@ function upsertFrontmatter(text, fields) {
|
||||
return `---\n${next.join("\n")}\n---\n${body}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove pipeline-owned metadata before hashing or rendering a stored document.
|
||||
* The returned text is the canonical localized document a person can edit: it
|
||||
* includes rendered links/anchors/frontmatter quoting, but excludes the
|
||||
* self-referential i18n bookkeeping fields.
|
||||
*
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
function stripI18nFrontmatter(text) {
|
||||
const { fm, body } = splitFrontmatter(text);
|
||||
if (fm == null) return text;
|
||||
const kept = fm.split("\n").filter((line) => !/^i18n_[A-Za-z0-9_]+:/.test(line));
|
||||
return kept.length > 0 ? `---\n${kept.join("\n")}\n---\n${body}` : body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Double-quote bare `description`/`title` frontmatter values so a translated
|
||||
* value containing a colon, `#`, or other YAML-significant character does not
|
||||
@@ -294,11 +311,20 @@ export function createDocsAdapter({ root = DEFAULT_ROOT } = {}) {
|
||||
for (const rel of files) {
|
||||
const text = await readFile(join(dir, rel), "utf8");
|
||||
const { fm } = splitFrontmatter(text);
|
||||
const provenance = fmGet(fm, "i18n_provenance") === "human" ? "human" : "machine";
|
||||
const canonicalText = stripI18nFrontmatter(text);
|
||||
const storedOutputHash = fmGet(fm, "i18n_output_hash") ?? "";
|
||||
const hashVersion = fmGet(fm, "i18n_hash_version");
|
||||
map.set(rel, {
|
||||
text,
|
||||
text: canonicalText,
|
||||
sourceHash: fmGet(fm, "i18n_source_hash") ?? "",
|
||||
provenance: fmGet(fm, "i18n_provenance") === "human" ? "human" : "machine",
|
||||
outputHash: fmGet(fm, "i18n_output_hash") ?? "",
|
||||
provenance,
|
||||
// Version 1 hashed the pre-render translator output, while load()
|
||||
// returned the post-render on-disk document. Every machine entry was
|
||||
// therefore misclassified as a human edit. For legacy machine files,
|
||||
// trust the current canonical document once; the next write stamps v2.
|
||||
outputHash:
|
||||
hashVersion === "2" || provenance === "human" ? storedOutputHash : hash(canonicalText),
|
||||
stale: fmGet(fm, "i18n_stale") === "true",
|
||||
});
|
||||
}
|
||||
@@ -310,12 +336,13 @@ export function createDocsAdapter({ root = DEFAULT_ROOT } = {}) {
|
||||
const abs = join(root, locale, id);
|
||||
await mkdir(dirname(abs), { recursive: true });
|
||||
const table = tokenTables.get(id) ?? [];
|
||||
const undone = restoreDocs(entry.text, table);
|
||||
const undone = restoreDocs(stripI18nFrontmatter(entry.text), table);
|
||||
const linked = quoteFrontmatterScalars(rewriteLinks(undone, locale));
|
||||
const withMeta = upsertFrontmatter(linked, {
|
||||
i18n_source_hash: entry.sourceHash,
|
||||
i18n_provenance: entry.provenance,
|
||||
i18n_output_hash: entry.outputHash,
|
||||
i18n_output_hash: hash(linked),
|
||||
i18n_hash_version: "2",
|
||||
...(entry.stale ? { i18n_stale: "true" } : {}),
|
||||
});
|
||||
await writeFile(abs, withMeta, "utf8");
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const SEMVER =
|
||||
/^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$/;
|
||||
|
||||
function assertVersion(version) {
|
||||
if (!SEMVER.test(version)) throw new Error(`Invalid semantic version: ${version}`);
|
||||
}
|
||||
|
||||
export function archiveCustomReleaseNotes(root, version) {
|
||||
assertVersion(version);
|
||||
const source = path.join(root, ".release-notes.md");
|
||||
if (!existsSync(source)) return false;
|
||||
|
||||
const archiveDirectory = path.join(root, ".release-notes");
|
||||
const archived = path.join(archiveDirectory, `v${version}.md`);
|
||||
mkdirSync(archiveDirectory, { recursive: true });
|
||||
if (existsSync(archived)) {
|
||||
if (!readFileSync(source).equals(readFileSync(archived))) {
|
||||
throw new Error(`Archived release notes differ for v${version}`);
|
||||
}
|
||||
rmSync(source);
|
||||
} else {
|
||||
renameSync(source, archived);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function materializeReleaseNotes(root, version, output) {
|
||||
assertVersion(version);
|
||||
const archived = path.join(root, ".release-notes", `v${version}.md`);
|
||||
if (existsSync(archived)) {
|
||||
copyFileSync(archived, output);
|
||||
return true;
|
||||
}
|
||||
|
||||
const changelog = readFileSync(path.join(root, "CHANGELOG.md"), "utf8");
|
||||
const escapedVersion = version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const heading = new RegExp(`^# \\[${escapedVersion}\\](?:\\(|\\s|$)`, "m");
|
||||
const match = heading.exec(changelog);
|
||||
if (!match) throw new Error(`CHANGELOG.md has no generated notes for v${version}`);
|
||||
const tail = changelog.slice(match.index);
|
||||
const nextHeading = /\n# \[[0-9]/.exec(tail);
|
||||
const notes = tail.slice(0, nextHeading?.index ?? tail.length).trimEnd();
|
||||
writeFileSync(output, `${notes}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
function docsChangelogBody(notes) {
|
||||
let body = notes.replaceAll("\r\n", "\n");
|
||||
body = body.replace(/^# SnapOtter [^\n]+[ \t]*\n(?:[ \t]*\n)?/, "");
|
||||
body = body.replace(/^## Highlights[ \t]*\n/, "");
|
||||
const upgrade = /^## Upgrade[ \t]*$/m.exec(body);
|
||||
if (upgrade) {
|
||||
const tail = body.slice(upgrade.index);
|
||||
const divider = /^---[ \t]*$/m.exec(tail);
|
||||
body = `${body.slice(0, upgrade.index)}${divider ? tail.slice(divider.index + divider[0].length) : ""}`;
|
||||
}
|
||||
return body.replace(/^---[ \t]*$/gm, "").trim();
|
||||
}
|
||||
|
||||
export function syncDocsChangelog(
|
||||
root,
|
||||
version,
|
||||
previousVersion,
|
||||
relativePath = "apps/docs/changelog.md",
|
||||
) {
|
||||
assertVersion(version);
|
||||
assertVersion(previousVersion);
|
||||
const archived = path.join(root, ".release-notes", `v${version}.md`);
|
||||
if (!existsSync(archived)) return false;
|
||||
|
||||
const body = docsChangelogBody(readFileSync(archived, "utf8"));
|
||||
const entry = [
|
||||
`## v${version}`,
|
||||
"",
|
||||
body,
|
||||
"",
|
||||
`[Full diff on GitHub](https://github.com/snapotter-hq/SnapOtter/compare/v${previousVersion}...v${version})`,
|
||||
"",
|
||||
"---",
|
||||
].join("\n");
|
||||
const changelogPath = path.join(root, relativePath);
|
||||
const changelog = readFileSync(changelogPath, "utf8");
|
||||
if (changelog.includes(`\n\n${entry}\n`)) return false;
|
||||
|
||||
const escapedVersion = version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
if (new RegExp(`^## v${escapedVersion}(?:[ \\t]+\\{#[^}]+\\})?[ \\t]*$`, "m").test(changelog)) {
|
||||
throw new Error(`Published changelog entry differs for v${version}`);
|
||||
}
|
||||
const heading = /^# Changelog(?:[ \t]+\{#[^}]+\})?[ \t]*$/m.exec(changelog);
|
||||
if (!heading) throw new Error("Published changelog heading is missing");
|
||||
const insertion = heading.index + heading[0].length;
|
||||
const next = `${changelog.slice(0, insertion)}\n\n${entry}${changelog.slice(insertion)}`;
|
||||
writeFileSync(changelogPath, next);
|
||||
return true;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const [command, version, argument] = process.argv.slice(2);
|
||||
const rootFlag = process.argv.indexOf("--root");
|
||||
const root = rootFlag === -1 ? process.cwd() : path.resolve(process.argv[rootFlag + 1] ?? "");
|
||||
|
||||
if (command === "archive" && version) {
|
||||
process.stdout.write(`custom=${archiveCustomReleaseNotes(root, version)}\n`);
|
||||
return;
|
||||
}
|
||||
if (command === "materialize" && version && argument) {
|
||||
process.stdout.write(`custom=${materializeReleaseNotes(root, version, argument)}\n`);
|
||||
return;
|
||||
}
|
||||
if (command === "sync-docs" && version && argument) {
|
||||
process.stdout.write(`changed=${syncDocsChangelog(root, version, argument)}\n`);
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
"Usage: manage-release-notes.mjs <archive VERSION | materialize VERSION OUTPUT | sync-docs VERSION PREVIOUS_VERSION> [--root PATH]",
|
||||
);
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
|
||||
main();
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync, spawn } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const MAX_FUZZ_RUNS = 10_000;
|
||||
const MAX_FUZZ_SEED = 2_147_483_647;
|
||||
|
||||
export const FUZZ_USAGE = "usage: pnpm test:fuzz [--runs <1-10000>] [--seed <0-2147483647>]";
|
||||
|
||||
function parseInteger(name, value, { min, max }) {
|
||||
if (typeof value !== "string" || !/^-?\d+$/.test(value)) {
|
||||
throw new Error(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed)) throw new Error(`${name} must be an integer`);
|
||||
if (parsed < min || parsed > max) {
|
||||
throw new Error(`${name} must be between ${min} and ${max}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseFuzzArguments(argv) {
|
||||
const result = { help: false, runs: undefined, seed: undefined };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
if (argument === "--") continue;
|
||||
if (argument === "--help" || argument === "-h") {
|
||||
result.help = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const [name, inlineValue] = argument.split("=", 2);
|
||||
if (name !== "--runs" && name !== "--seed") {
|
||||
throw new Error(`unknown argument: ${argument}\n${FUZZ_USAGE}`);
|
||||
}
|
||||
const nextValue = inlineValue ?? argv[index + 1];
|
||||
if (inlineValue === undefined) index += 1;
|
||||
if (nextValue === undefined) throw new Error(`${name} requires a value`);
|
||||
|
||||
const key = name === "--runs" ? "runs" : "seed";
|
||||
if (result[key] !== undefined) throw new Error(`${name} may only be provided once`);
|
||||
result[key] = parseInteger(name, nextValue, {
|
||||
min: key === "runs" ? 1 : 0,
|
||||
max: key === "runs" ? MAX_FUZZ_RUNS : MAX_FUZZ_SEED,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function buildFuzzEnvironment(baseEnvironment, arguments_) {
|
||||
const environment = {
|
||||
...baseEnvironment,
|
||||
FUZZ: "1",
|
||||
VITEST_MAX_FORKS: "1",
|
||||
};
|
||||
if (arguments_.runs !== undefined) environment.FUZZ_RUNS = String(arguments_.runs);
|
||||
if (arguments_.seed !== undefined) {
|
||||
environment.FUZZ_SEED = String(arguments_.seed);
|
||||
delete environment.FC_SEED;
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
export function fuzzLockPath(cwd) {
|
||||
let repositoryIdentity = resolve(cwd);
|
||||
try {
|
||||
const gitCommonDirectory = execFileSync("git", ["rev-parse", "--git-common-dir"], {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
timeout: 5_000,
|
||||
windowsHide: true,
|
||||
}).trim();
|
||||
if (gitCommonDirectory) repositoryIdentity = resolve(cwd, gitCommonDirectory);
|
||||
} catch {
|
||||
// Outside a Git repository, keep the resolved working directory identity.
|
||||
}
|
||||
const repoHash = createHash("sha256").update(repositoryIdentity).digest("hex");
|
||||
return resolve(tmpdir(), `snapotter-fuzz-${repoHash}.lock`);
|
||||
}
|
||||
|
||||
function describeLockOwner(lockPath) {
|
||||
try {
|
||||
const owner = JSON.parse(readFileSync(resolve(lockPath, "owner.json"), "utf8"));
|
||||
return {
|
||||
pid: Number.isInteger(owner.pid) ? owner.pid : "unknown",
|
||||
startedAt: typeof owner.startedAt === "string" ? owner.startedAt : "unknown",
|
||||
};
|
||||
} catch {
|
||||
return { pid: "unknown", startedAt: "unknown" };
|
||||
}
|
||||
}
|
||||
|
||||
export function acquireFuzzLock(lockPath, owner) {
|
||||
try {
|
||||
mkdirSync(lockPath);
|
||||
} catch (error) {
|
||||
if (error?.code !== "EEXIST") throw error;
|
||||
const existing = describeLockOwner(lockPath);
|
||||
throw new Error(
|
||||
`another fuzz campaign is already running (pid=${existing.pid}, ` +
|
||||
`startedAt=${existing.startedAt}, lock=${lockPath})`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
writeFileSync(resolve(lockPath, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, {
|
||||
flag: "wx",
|
||||
});
|
||||
} catch (error) {
|
||||
rmSync(lockPath, { force: true, recursive: true });
|
||||
throw error;
|
||||
}
|
||||
|
||||
let released = false;
|
||||
return {
|
||||
release() {
|
||||
if (released) return;
|
||||
const currentOwner = JSON.parse(readFileSync(resolve(lockPath, "owner.json"), "utf8"));
|
||||
if (currentOwner.token !== owner.token) {
|
||||
throw new Error("refusing to release a fuzz lock owned by another process");
|
||||
}
|
||||
rmSync(lockPath, { recursive: true });
|
||||
released = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runVitest(cwd, environment) {
|
||||
const command = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
|
||||
const args = [
|
||||
"exec",
|
||||
"vitest",
|
||||
"run",
|
||||
"tests/integration/generated/fuzz-settings.test.ts",
|
||||
"--reporter=verbose",
|
||||
];
|
||||
|
||||
return new Promise((fulfill, reject) => {
|
||||
const child = spawn(command, args, { cwd, env: environment, stdio: "inherit" });
|
||||
const forwardSignal = (signal) => child.kill(signal);
|
||||
const interrupt = () => forwardSignal("SIGINT");
|
||||
const terminate = () => forwardSignal("SIGTERM");
|
||||
process.once("SIGINT", interrupt);
|
||||
process.once("SIGTERM", terminate);
|
||||
|
||||
const cleanup = () => {
|
||||
process.off("SIGINT", interrupt);
|
||||
process.off("SIGTERM", terminate);
|
||||
};
|
||||
child.once("error", (error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
});
|
||||
child.once("exit", (code) => {
|
||||
cleanup();
|
||||
fulfill(code ?? 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const arguments_ = parseFuzzArguments(process.argv.slice(2));
|
||||
if (arguments_.help) {
|
||||
console.info(FUZZ_USAGE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cwd = process.cwd();
|
||||
const lockPath = fuzzLockPath(cwd);
|
||||
const lock = acquireFuzzLock(lockPath, {
|
||||
pid: process.pid,
|
||||
startedAt: new Date().toISOString(),
|
||||
token: randomUUID(),
|
||||
});
|
||||
|
||||
try {
|
||||
console.info(`[fuzz-runner] acquired exclusive lock ${lockPath}; VITEST_MAX_FORKS=1`);
|
||||
return await runVitest(cwd, buildFuzzEnvironment(process.env, arguments_));
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main()
|
||||
.then((exitCode) => {
|
||||
process.exitCode = exitCode;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const RELEASE_PROJECTS = [
|
||||
"chromium",
|
||||
"firefox",
|
||||
"webkit",
|
||||
"chromium-widths",
|
||||
"mobile-chromium",
|
||||
"mobile-webkit",
|
||||
"tablet-chromium",
|
||||
"tablet-webkit",
|
||||
];
|
||||
|
||||
function hasVisualBaselines(platform) {
|
||||
const screenshotRoot = path.resolve(process.cwd(), "tests/e2e/__screenshots__");
|
||||
if (!existsSync(screenshotRoot)) return false;
|
||||
return readdirSync(screenshotRoot, { recursive: true }).some((entry) =>
|
||||
String(entry).endsWith(`-${platform}.png`),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildMainE2ePlan(platform = process.platform, coreOnly = false) {
|
||||
const projects = coreOnly ? ["chromium"] : RELEASE_PROJECTS;
|
||||
const standard = ["test", ...projects.map((project) => `--project=${project}`)];
|
||||
const visualBaselinesAvailable = hasVisualBaselines(platform);
|
||||
if (!visualBaselinesAvailable) standard.push("--grep-invert=@visual");
|
||||
|
||||
const plan = [standard, ["test", "--project=chromium-serial", "--workers=1"]];
|
||||
if (visualBaselinesAvailable) {
|
||||
plan.push(["test", "--project=chromium-visual", "--project=chromium-legacy-visual"]);
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const planIndex = process.argv.indexOf("--plan");
|
||||
const coreOnly = process.argv.includes("--core");
|
||||
const platform = planIndex === -1 ? process.platform : process.argv[planIndex + 1];
|
||||
if (!platform || platform.startsWith("--")) {
|
||||
throw new Error("--plan requires a platform name such as darwin, linux, or win32");
|
||||
}
|
||||
|
||||
const plan = buildMainE2ePlan(platform, coreOnly);
|
||||
if (planIndex !== -1) {
|
||||
process.stdout.write(JSON.stringify(plan));
|
||||
return;
|
||||
}
|
||||
|
||||
const pnpm = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
|
||||
for (const args of plan) {
|
||||
const result = spawnSync(pnpm, ["exec", "playwright", ...args], {
|
||||
env: process.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
if (result.error) throw result.error;
|
||||
if (result.status !== 0) process.exit(result.status ?? 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
|
||||
main();
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const SEMVER =
|
||||
/^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$/;
|
||||
|
||||
export function updateReleaseReferences(source, version) {
|
||||
return source
|
||||
.replace(/(SnapOtter\/(?:blob\/)?v)[^/]+(\/docker\/docker-compose\.yml)/g, `$1${version}$2`)
|
||||
.replace(
|
||||
/(snapotter-v)[0-9][0-9A-Za-z.+-]*?(?=-(?:release-subjects|image-linux-amd64-sbom))/g,
|
||||
`$1${version}`,
|
||||
)
|
||||
.replace(/(snapotter\/snapotter:)[0-9][0-9A-Za-z.+-]*/g, `$1${version}`);
|
||||
}
|
||||
|
||||
function releasePages(root) {
|
||||
const docsRoot = path.join(root, "apps/docs");
|
||||
const pages = [
|
||||
path.join(docsRoot, "guide/getting-started.md"),
|
||||
path.join(docsRoot, "guide/security.md"),
|
||||
];
|
||||
if (!existsSync(docsRoot)) return pages;
|
||||
for (const entry of readdirSync(docsRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name === "guide") continue;
|
||||
pages.push(
|
||||
path.join(docsRoot, entry.name, "guide/getting-started.md"),
|
||||
path.join(docsRoot, entry.name, "guide/security.md"),
|
||||
);
|
||||
}
|
||||
return pages.filter(existsSync);
|
||||
}
|
||||
|
||||
export function syncPublishedDocsVersion(root, version) {
|
||||
if (!SEMVER.test(version)) throw new Error(`Invalid semantic version: ${version}`);
|
||||
let updated = 0;
|
||||
for (const page of releasePages(root)) {
|
||||
const source = readFileSync(page, "utf8");
|
||||
const next = updateReleaseReferences(source, version);
|
||||
if (next !== source) {
|
||||
writeFileSync(page, next);
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const version = process.argv[2];
|
||||
const rootFlag = process.argv.indexOf("--root");
|
||||
const root =
|
||||
rootFlag === -1
|
||||
? path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
|
||||
: path.resolve(process.argv[rootFlag + 1] ?? "");
|
||||
if (!version) throw new Error("Usage: sync-published-docs-version.mjs <version> [--root path]");
|
||||
const updated = syncPublishedDocsVersion(root, version);
|
||||
console.log(`Updated ${updated} published documentation version file(s) -> ${version}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url)
|
||||
main();
|
||||
+25
-2
@@ -8,16 +8,24 @@
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${1:?Usage: sync-version.sh <version>}"
|
||||
if [[ ! "$VERSION" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
|
||||
echo "Invalid semantic version: $VERSION" >&2
|
||||
exit 2
|
||||
fi
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
# All workspace package.json files to sync
|
||||
PACKAGES=(
|
||||
"apps/web/package.json"
|
||||
"apps/api/package.json"
|
||||
"apps/demo/package.json"
|
||||
"apps/docs/package.json"
|
||||
"apps/landing/package.json"
|
||||
"packages/shared/package.json"
|
||||
"packages/doc-engine/package.json"
|
||||
"packages/enterprise/package.json"
|
||||
"packages/image-engine/package.json"
|
||||
"packages/media-engine/package.json"
|
||||
"packages/ai/package.json"
|
||||
)
|
||||
|
||||
@@ -45,7 +53,22 @@ if [ -f "$CONSTANTS" ]; then
|
||||
echo " Updated APP_VERSION -> $VERSION"
|
||||
fi
|
||||
|
||||
# Clean up release notes file (consumed by CI, not needed after release commit)
|
||||
rm -f "$ROOT/.release-notes.md"
|
||||
# Keep the release-specific commands in every published documentation locale
|
||||
# aligned with the tag and immutable artifact names created by this release.
|
||||
node "$ROOT/scripts/sync-published-docs-version.mjs" "$VERSION"
|
||||
|
||||
# Archive optional custom notes under their immutable release version before the
|
||||
# semantic-release git plugin commits and tags them. This makes a tag-only retry
|
||||
# able to reconstruct the exact draft body and published docs changelog.
|
||||
node "$ROOT/scripts/manage-release-notes.mjs" archive "$VERSION" --root "$ROOT" >/dev/null
|
||||
if [ -f "$ROOT/.release-notes/v$VERSION.md" ]; then
|
||||
PREVIOUS_TAG="$(git -C "$ROOT" describe --tags --abbrev=0 --match 'v[0-9]*' 2>/dev/null || true)"
|
||||
if [[ ! "$PREVIOUS_TAG" =~ ^v(.+)$ ]]; then
|
||||
echo "Cannot update the published changelog without a previous release tag" >&2
|
||||
exit 1
|
||||
fi
|
||||
node "$ROOT/scripts/manage-release-notes.mjs" sync-docs "$VERSION" "${PREVIOUS_TAG#v}" \
|
||||
--root "$ROOT" >/dev/null
|
||||
fi
|
||||
|
||||
echo "All versions synced to $VERSION"
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Gate the vulnerabilities a Trivy `ignore-unfixed` scan cannot see.
|
||||
*
|
||||
* The release workflow blocks on CRITICAL/HIGH findings that have a fix
|
||||
* available, which is the actionable gate: a fix exists and we did not take
|
||||
* it. Everything without a fix was dropped silently, so a shipped image could
|
||||
* carry an unfixed CRITICAL and the release scan would still print zero. That
|
||||
* reads exactly like a clean scan.
|
||||
*
|
||||
* This closes the gap without making the pipeline permanently red on things
|
||||
* nobody can act on. Findings that are known, written down and re-checked pass;
|
||||
* anything new fails. Stale allowlist entries only warn, because the release
|
||||
* matrix scans one architecture per job and the two do not carry the same set.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/trivy-unfixed-gate.mjs <report.json> [options]
|
||||
*
|
||||
* --allow <file> allowlist path (default .trivy-unfixed-allow)
|
||||
* --severity <list> comma-separated severities (default CRITICAL)
|
||||
* --label <text> artifact name for the report heading
|
||||
* --summary <file> append the markdown report here (GITHUB_STEP_SUMMARY)
|
||||
*/
|
||||
|
||||
import { appendFileSync, readFileSync } from "node:fs";
|
||||
|
||||
const ALLOW_ENTRY = /^(?:CVE|GHSA|PYSEC|DLA|DSA|TEMP|OSV)-[\w.-]+$/i;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
report: undefined,
|
||||
allow: ".trivy-unfixed-allow",
|
||||
severity: ["CRITICAL"],
|
||||
label: "artifact",
|
||||
summary: "",
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg === "--allow") options.allow = argv[++i];
|
||||
else if (arg === "--severity")
|
||||
options.severity = argv[++i].split(",").map((s) => s.trim().toUpperCase());
|
||||
else if (arg === "--label") options.label = argv[++i];
|
||||
else if (arg === "--summary") options.summary = argv[++i];
|
||||
else if (!options.report) options.report = arg;
|
||||
else throw new Error(`Unexpected argument: ${arg}`);
|
||||
}
|
||||
if (!options.report)
|
||||
throw new Error(
|
||||
"Usage: trivy-unfixed-gate.mjs <report.json> [--allow f] [--severity l] [--label t] [--summary f]",
|
||||
);
|
||||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect unfixed findings, keyed by vulnerability ID.
|
||||
*
|
||||
* "Unfixed" matches Trivy's own `--ignore-unfixed` rule (no FixedVersion) so
|
||||
* this view and the blocking scan partition the report between them with no
|
||||
* finding falling in the gap.
|
||||
*/
|
||||
export function collectUnfixed(report, severities) {
|
||||
const wanted = new Set(severities);
|
||||
const byId = new Map();
|
||||
for (const result of report.Results ?? []) {
|
||||
for (const vuln of result.Vulnerabilities ?? []) {
|
||||
if (!wanted.has(vuln.Severity)) continue;
|
||||
if (vuln.FixedVersion) continue;
|
||||
const entry = byId.get(vuln.VulnerabilityID) ?? {
|
||||
id: vuln.VulnerabilityID,
|
||||
severity: vuln.Severity,
|
||||
title: vuln.Title ?? "",
|
||||
packages: new Set(),
|
||||
};
|
||||
entry.packages.add(`${vuln.PkgName} ${vuln.InstalledVersion ?? "?"}`);
|
||||
byId.set(vuln.VulnerabilityID, entry);
|
||||
}
|
||||
}
|
||||
return [...byId.values()]
|
||||
.map((entry) => ({ ...entry, packages: [...entry.packages].sort() }))
|
||||
.sort((a, b) => a.id.localeCompare(b.id));
|
||||
}
|
||||
|
||||
export function parseAllowlist(text) {
|
||||
const ids = [];
|
||||
for (const raw of text.split(/\r?\n/)) {
|
||||
const line = raw.replace(/#.*$/, "").trim();
|
||||
if (!line) continue;
|
||||
if (!ALLOW_ENTRY.test(line)) throw new Error(`Not a vulnerability ID: "${line}"`);
|
||||
ids.push(line.toUpperCase());
|
||||
}
|
||||
return new Set(ids);
|
||||
}
|
||||
|
||||
function markdown(label, severities, findings, unexpected, stale) {
|
||||
const lines = [
|
||||
`### Unfixed ${severities.join("/")} findings: ${label}`,
|
||||
"",
|
||||
findings.length === 0
|
||||
? "None. Every finding at this severity has a fix available and is covered by the blocking scan."
|
||||
: `${findings.length} finding${findings.length === 1 ? "" : "s"} with no fix available upstream.`,
|
||||
"",
|
||||
];
|
||||
if (findings.length > 0) {
|
||||
lines.push(
|
||||
"| ID | Severity | Package | Allowed | Summary |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
);
|
||||
for (const f of findings) {
|
||||
// Backslashes first: escaping the pipe first would then re-escape the
|
||||
// backslash we just added, and a title containing a literal \| would
|
||||
// break out of the table cell.
|
||||
const summary = f.title.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").slice(0, 90);
|
||||
const allowed = unexpected.some((u) => u.id === f.id) ? "**no**" : "yes";
|
||||
lines.push(
|
||||
`| ${f.id} | ${f.severity} | ${f.packages.join("<br>")} | ${allowed} | ${summary} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
if (unexpected.length > 0) {
|
||||
lines.push(
|
||||
`**${unexpected.length} not in the allowlist.** Fix them, or add each ID to \`.trivy-unfixed-allow\` with an owner, what unblocks the fix, and a re-check date.`,
|
||||
"",
|
||||
);
|
||||
}
|
||||
if (stale.length > 0) {
|
||||
lines.push(`Allowlist entries not seen in this scan: ${stale.join(", ")}.`, "");
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const report = JSON.parse(readFileSync(options.report, "utf8"));
|
||||
const findings = collectUnfixed(report, options.severity);
|
||||
const allowed = parseAllowlist(readFileSync(options.allow, "utf8"));
|
||||
|
||||
const unexpected = findings.filter((f) => !allowed.has(f.id.toUpperCase()));
|
||||
const seen = new Set(findings.map((f) => f.id.toUpperCase()));
|
||||
const stale = [...allowed].filter((id) => !seen.has(id)).sort();
|
||||
|
||||
const body = markdown(options.label, options.severity, findings, unexpected, stale);
|
||||
process.stdout.write(`${body}\n`);
|
||||
if (options.summary) appendFileSync(options.summary, `${body}\n`);
|
||||
|
||||
// Stale entries warn rather than fail: the release matrix scans one platform
|
||||
// per job, and an entry that is live on arm64 is absent from the amd64 job.
|
||||
// One aggregate annotation, since the per-ID detail is in the summary table.
|
||||
if (stale.length > 0) {
|
||||
console.log(
|
||||
`::warning::${stale.length} allowlist entr${stale.length === 1 ? "y is" : "ies are"} absent from ${options.label}; re-check whether they are still needed`,
|
||||
);
|
||||
}
|
||||
for (const finding of unexpected) {
|
||||
console.log(
|
||||
`::error::${finding.id} (${finding.severity}, ${finding.packages.join(", ")}) has no fix available and is not in .trivy-unfixed-allow`,
|
||||
);
|
||||
}
|
||||
|
||||
if (unexpected.length > 0) {
|
||||
console.error(
|
||||
`\n${unexpected.length} unfixed ${options.severity.join("/")} finding(s) in ${options.label} are not accounted for.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) main();
|
||||
Reference in New Issue
Block a user