External/truthmark
1
0
Fork 0
mirror of https://github.com/merlinhu1/truthmark.git synced 2026-08-25 07:53:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add workflow helper validators and host-native surfaces (#7)

Browse Source 
* fix: harden workflow helper manifests and reports

* docs: add helper script portability design

* fix: reject failed helper statuses in completed reports

* fix: align helper report sections metadata

* fix: preserve sync helper statuses in parser

* test: harden helper validator negatives

* feat: add Copilot and Gemini Truthmark surfaces

- generate Copilot and Gemini workflow skill packages with helper manifests
- add Gemini subagent surfaces and validate helper CLI plumbing
- update README/docs for helper-package support

Verification: npm run check

* docs: simplify README workflow surface overview

Replace long generated file lists with conceptual layers and Mermaid architecture diagrams across localized READMEs.

Verification: node dist/main.js check --json; npm run package:check

* docs: show agent CLI feedback loop in README diagram

Clarify that host-native agent workflows can call the installed Truthmark CLI for validation, indexing, and helper checks.

Verification: node dist/main.js check --json; npm run package:check

* fix: wrap validate JSON output in command envelope

Return helper validation results under data.validation for --json output while preserving the existing human-readable validate output.

Verification: npm run check; truthmark check/index JSON diagnostics.

* fix: clarify helper validation status reporting

Add explicit helper-status policy to Truth Sync and Truth Document surfaces so standalone Copilot prompts and Gemini commands only report ran/passed after the installed CLI validator succeeds.

Update README and workflow docs to describe the shared installed-CLI validator contract instead of saying standalone surfaces mark helper packages unavailable.

Verification: npm run check; npm run package:check; truthmark check/index JSON diagnostics.

* fix: normalize helper status ids during validation

Parse helper status entries with the same regex used for syntax validation and compare captured helper IDs against required helpers.

Adds regressions for extra whitespace after helper bullets in Truth Sync and Truth Document reports.

Verification: npm run check; npm run package:check; truthmark check/index JSON diagnostics.

* fix: parse write lease YAML structurally

* chore: bump version to 1.5.0

* docs: include validate in repo CLI boundary

* fix: enforce workflow report validator contracts

* fix: require manual review files in blocked sync reports

---------

Co-authored-by: Hermes Agent <hermes-agent@users.noreply.github.com>
This commit is contained in:
Merlin's Cat
2026-05-20 11:15:24 +10:00
committed by GitHub
co-authored by Hermes Agent
parent cc11b8e26c
commit 17ed1599a3
215 changed files with 5608 additions and 665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/agents/truth-document.test.ts
+17
View File
@@ -9,9 +9,11 @@ import {
} from "../../src/agents/truth-document.js";
import {
renderTruthmarkDocumentClaudeSkill,
renderTruthmarkCopilotDocumentPrompt,
renderTruthmarkDocumentLocalSkill,
renderTruthmarkDocumentSkill,
renderTruthmarkDocumentSkillMetadata,
renderTruthmarkGeminiDocumentCommand,
} from "../../src/templates/workflow-surfaces.js";
import { TRUTHMARK_VERSION } from "../../src/version.js";
@@ -199,5 +201,20 @@ describe("Truth Document generated surfaces", () => {
expect(renderTruthmarkDocumentSkillMetadata()).toContain(
`version: "${TRUTHMARK_VERSION}"`,
);
for (const surface of [
renderTruthmarkGeminiDocumentCommand(),
renderTruthmarkCopilotDocumentPrompt(),
]) {
expect(surface).toContain(
"Validate the report body before adding this validator's own success status; the body may omit `validate-document-report` while validation is pending.",
);
expect(surface).toContain(
"After `truthmark validate document-report <report-file> --json` returns `data.validation.ok: true`, append or update `validate-document-report: ran, passed` in the final report.",
);
expect(surface).toContain(
"If the installed Truthmark CLI is unavailable or the helper is skipped, record `validate-document-report: skipped, <reason>` and manually validate the report shape.",
);
expect(surface).not.toContain("helper package unavailable");
}
});
});
tests/agents/truth-sync.test.ts
+15
View File
@@ -219,6 +219,21 @@ describe("Truth Sync generated metadata", () => {
expect(renderTruthmarkCopilotSyncPrompt()).toContain(
"description: 'Use automatically at finish-time after functional code changes",
);
for (const surface of [
renderTruthmarkGeminiSyncCommand(),
renderTruthmarkCopilotSyncPrompt(),
]) {
expect(surface).toContain(
"Validate the report body before adding this validator's own success status; the body may omit `validate-sync-report` while validation is pending.",
);
expect(surface).toContain(
"After `truthmark validate sync-report <report-file> --json` returns `data.validation.ok: true`, append or update `validate-sync-report: ran, passed` in the final report.",
);
expect(surface).toContain(
"If the installed Truthmark CLI is unavailable or the helper is skipped, record `validate-sync-report: skipped, <reason>` and manually validate the report shape.",
);
expect(surface).not.toContain("helper package unavailable");
}
});
it("adds host-specific subagent guidance without changing generic surfaces", () => {
tests/agents/workflow-helper-scripts.test.ts
+879
View File
@@ -0,0 +1,879 @@
import fs from "node:fs/promises";
import path from "node:path";
import { parse } from "yaml";
import { afterEach, describe, expect, it } from "vitest";
import { renderTruthmarkSkillPackage } from "../../src/templates/workflow-surfaces.js";
import { runCli } from "../helpers/run-cli.js";
import { createTempRepo } from "../helpers/temp-repo.js";
type HelperResult = {
exitCode: number;
stdout: string;
stderr: string;
json: {
ok: boolean;
helper: string;
checks?: string[];
errors?: string[];
};
};
type ValidationEnvelope = {
command: string;
summary: string;
diagnostics: unknown[];
data?: {
validation?: HelperResult["json"];
};
};
const parseValidationEnvelope = (stdout: string): HelperResult["json"] => {
const parsed = JSON.parse(stdout) as ValidationEnvelope;
const validation = parsed.data?.validation;
if (validation === undefined) {
throw new Error(`missing data.validation in helper JSON\nstdout:\n${stdout}`);
}
return validation;
};
const tempRepos: Array<Awaited<ReturnType<typeof createTempRepo>>> = [];
const snapshotFiles = async (rootDir: string): Promise<Record<string, string>> => {
const entries = await fs.readdir(rootDir, {
recursive: true,
withFileTypes: true,
});
const filePaths = entries
.filter((entry) => entry.isFile())
.map((entry) => path.relative(rootDir, path.join(entry.parentPath, entry.name)))
.sort();
return Object.fromEntries(
await Promise.all(
filePaths.map(async (filePath) => [
filePath,
await fs.readFile(path.join(rootDir, filePath), "utf8"),
]),
),
);
};
const runCliHelper = async ({
files,
args,
}: {
files: Record<string, string>;
args: string[];
}): Promise<HelperResult> => {
const repo = await createTempRepo();
tempRepos.push(repo);
for (const [filePath, content] of Object.entries(files)) {
await repo.writeFile(filePath, content);
}
const before = await snapshotFiles(repo.rootDir);
const result = await runCli(args, { cwd: repo.rootDir });
const after = await snapshotFiles(repo.rootDir);
expect(after).toEqual(before);
if (result.stdout === "") {
throw new Error(`helper printed no JSON\nstderr:\n${result.stderr}`);
}
return {
exitCode: result.exitCode ?? 1,
stdout: result.stdout,
stderr: result.stderr,
json: parseValidationEnvelope(result.stdout),
};
};
const getGeneratedReportExample = (workflowId: "truthmark-document" | "truthmark-sync"): string => {
const files = renderTruthmarkSkillPackage({
skillPath: `.codex/skills/${workflowId}/SKILL.md`,
workflowId,
host: "codex",
});
const reportTemplate = files.find((file) => file.path.endsWith("/support/report-template.md"));
const match = reportTemplate?.content.match(/```md\n([\s\S]*?)\n```/u);
if (match === null || match === undefined) {
throw new Error(`missing generated report example for ${workflowId}`);
}
return match[1];
};
const materializeSkillPackage = async (workflowId: "truthmark-document" | "truthmark-sync") => {
const repo = await createTempRepo();
tempRepos.push(repo);
for (const file of renderTruthmarkSkillPackage({
skillPath: `.codex/skills/${workflowId}/SKILL.md`,
workflowId,
host: "codex",
})) {
await repo.writeFile(file.path, file.content);
}
return repo;
};
const syncReportWithEvidence = (evidenceChecked: string): string => `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Evidence checked:
${evidenceChecked}
Helper scripts:
- validate-sync-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
const documentReportWithEvidence = (evidenceChecked: string): string => `Truth Document: completed
Implementation reviewed:
- src/templates/workflow-surfaces.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs created:
- docs/truth/workflows/helpers.md
Evidence checked:
${evidenceChecked}
Helper scripts:
- validate-document-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
const runSyncReport = async (report: string): Promise<HelperResult> =>
runCliHelper({
files: { "report.md": report },
args: ["validate", "sync-report", "report.md", "--json"],
});
const runDocumentReport = async (report: string): Promise<HelperResult> =>
runCliHelper({
files: { "report.md": report },
args: ["validate", "document-report", "report.md", "--json"],
});
const runWriteLease = async ({
lease,
changedFiles,
}: {
lease: string;
changedFiles: string;
}): Promise<HelperResult> =>
runCliHelper({
files: {
"lease.yml": lease,
"changed-files.txt": changedFiles,
},
args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"],
});
afterEach(async () => {
const repos = tempRepos.splice(0);
await Promise.all(repos.map((repo) => repo.cleanup()));
});
describe("workflow helper scripts", () => {
it("renders parseable helper manifests for every generated workflow host", () => {
const packageTargets = [
{ host: "codex", skillRoot: ".codex" },
{ host: "opencode", skillRoot: ".opencode" },
{ host: "claude-code", skillRoot: ".claude" },
] as const;
const workflowIds = ["truthmark-document", "truthmark-sync"] as const;
for (const { host, skillRoot } of packageTargets) {
for (const workflowId of workflowIds) {
const files = renderTruthmarkSkillPackage({
skillPath: `${skillRoot}/skills/${workflowId}/SKILL.md`,
workflowId,
host,
});
const manifest = files.find((file) =>
file.path.endsWith("/helper-manifest.yml"),
);
const entrypoint = files.find((file) => file.path.endsWith("/SKILL.md"));
expect(manifest).toBeDefined();
expect(entrypoint?.content).toContain("- helper-manifest.yml");
const parsed = parse(manifest?.content ?? "") as {
helpers?: Record<string, { fallback?: unknown }>;
};
const expectedReportHelper =
workflowId === "truthmark-sync"
? "validate-sync-report"
: "validate-document-report";
expect(parsed.helpers).toBeDefined();
expect(parsed.helpers?.[expectedReportHelper]?.fallback).toEqual(
expect.any(String),
);
if (workflowId === "truthmark-sync") {
expect(parsed.helpers?.[expectedReportHelper]?.fallback).toContain(
"Result: supported",
);
}
}
}
});
it("runs generated helper manifest argv through the Truthmark CLI", async () => {
const repo = await materializeSkillPackage("truthmark-sync");
const skillDirectory = path.join(repo.rootDir, ".codex/skills/truthmark-sync");
const manifest = await fs.readFile(
path.join(skillDirectory, "helper-manifest.yml"),
"utf8",
);
const parsed = parse(manifest) as {
helpers?: Record<string, { command?: { argv?: string[] } }>;
};
const argv = parsed.helpers?.["validate-sync-report"]?.command?.argv;
expect(argv).toEqual([
"truthmark",
"validate",
"sync-report",
"<report-file>",
"--json",
]);
await expect(
fs.access(path.join(skillDirectory, "scripts/validate-sync-report.mjs")),
).rejects.toThrow();
await fs.writeFile(
path.join(skillDirectory, "report.md"),
getGeneratedReportExample("truthmark-sync"),
"utf8",
);
const result = await runCli(["validate", "sync-report", "report.md", "--json"], {
cwd: skillDirectory,
});
expect(result.exitCode).toBe(0);
expect(JSON.parse(result.stdout)).toMatchObject({
data: { validation: { ok: true } },
});
});
it("accepts a valid completed Truth Sync report", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Evidence checked:
- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported
Helper scripts:
- validate-sync-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`,
},
args: ["validate", "sync-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(0);
expect(result.stderr).toBe("");
expect(result.json.ok).toBe(true);
expect(result.json.helper).toBe("validate-sync-report");
});
it("accepts a completed Truth Sync report body before its own helper status is appended", async () => {
const result = await runSyncReport(
syncReportWithEvidence(`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported`).replace("- validate-sync-report: ran, passed\n", ""),
);
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("accepts Truth Sync helper status bullets with flexible whitespace", async () => {
const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported`)
.replace("- validate-sync-report: ran, passed", "- validate-sync-report: ran, passed")
.replace(
"- validate-write-lease: skipped, no write lease used",
"-\tvalidate-write-lease: skipped, no write lease used",
);
const result = await runSyncReport(report);
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("accepts the generated Truth Sync report template example", async () => {
const result = await runCliHelper({
files: {
"report.md": getGeneratedReportExample("truthmark-sync"),
},
args: ["validate", "sync-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("rejects a completed Truth Sync report missing helper script statuses", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Evidence checked:
- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported
Notes:
- Complete.
`,
},
args: ["validate", "sync-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Helper scripts");
});
it.each([
["blocked", "Reason"],
["skipped", "Reason"],
])("rejects a Truth Sync %s report with no required body", async (status, expectedError) => {
const result = await runSyncReport(`Truth Sync: ${status}\n`);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain(expectedError);
});
it("rejects a blocked Truth Sync report missing manual-review files", async () => {
const result = await runSyncReport(`Truth Sync: blocked
Reason:
- route ownership is ambiguous
Next action:
- run Truth Structure before rerunning Truth Sync
`);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Files requiring manual review");
});
it.each([
["Changed code reviewed"],
["Ownership reviewed"],
["Truth docs updated"],
["Notes"],
])("rejects a completed Truth Sync report with empty %s", async (label) => {
const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported`).replace(new RegExp(`${label}:\\n- [^\\n]+`, "u"), `${label}:`);
const result = await runSyncReport(report);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain(label);
});
it("rejects a completed Truth Sync report with evidence labels outside Evidence checked", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Evidence checked:
- malformed entry only
Notes:
- Claim: appears outside the evidence section.
- Evidence: appears outside the evidence section.
- Result: appears outside the evidence section.
`,
},
args: ["validate", "sync-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Evidence checked");
});
it("rejects a completed Truth Sync report missing Evidence checked", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Notes:
- Missing evidence.
`,
},
args: ["validate", "sync-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Evidence checked");
});
it.each([
["empty Evidence checked", ""],
[
"missing Result",
`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts`,
],
[
"invalid Result enum",
`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: guessed`,
],
[
"malformed indentation",
`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported`,
],
])("rejects a completed Truth Sync report with %s", async (_name, evidenceChecked) => {
const result = await runSyncReport(syncReportWithEvidence(evidenceChecked));
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Evidence checked");
});
it.each([
["validate-sync-report"],
["validate-write-lease"],
])("rejects a completed Truth Sync report when %s ran and failed", async (helperId) => {
const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported`).replace(
`- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`,
`- ${helperId}: ran, failed`,
);
const result = await runSyncReport(report);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("ran, failed");
});
it("accepts a valid completed Truth Document report", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Document: completed
Implementation reviewed:
- src/templates/workflow-surfaces.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs created:
- docs/truth/workflows/helpers.md
Evidence checked:
- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported
Helper scripts:
- validate-document-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`,
},
args: ["validate", "document-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
expect(result.json.helper).toBe("validate-document-report");
});
it("accepts a completed Truth Document report body before its own helper status is appended", async () => {
const result = await runDocumentReport(
documentReportWithEvidence(`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported`).replace("- validate-document-report: ran, passed\n", ""),
);
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("accepts Truth Document helper status bullets with flexible whitespace", async () => {
const report = documentReportWithEvidence(`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported`)
.replace(
"- validate-document-report: ran, passed",
"- validate-document-report: ran, passed",
)
.replace(
"- validate-write-lease: skipped, no write lease used",
"-\tvalidate-write-lease: skipped, no write lease used",
);
const result = await runDocumentReport(report);
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("accepts the generated Truth Document report template example", async () => {
const result = await runCliHelper({
files: {
"report.md": getGeneratedReportExample("truthmark-document"),
},
args: ["validate", "document-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it("rejects a Truth Document blocked report with no required body", async () => {
const result = await runDocumentReport("Truth Document: blocked\n");
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Reason");
});
it.each([
["Implementation reviewed"],
["Ownership reviewed"],
["Truth docs created"],
["Notes"],
])("rejects a completed Truth Document report with empty %s", async (label) => {
const report = documentReportWithEvidence(`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported`).replace(new RegExp(`${label}:\\n- [^\\n]+`, "u"), `${label}:`);
const result = await runDocumentReport(report);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain(label);
});
it("rejects a completed Truth Document report with malformed Evidence checked entries", async () => {
const result = await runCliHelper({
files: {
"report.md": `Truth Document: completed
Implementation reviewed:
- src/templates/workflow-surfaces.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs created:
- docs/truth/workflows/helpers.md
Evidence checked:
- malformed entry only
Notes:
- Complete.
`,
},
args: ["validate", "document-report", "report.md", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Evidence checked");
});
it.each([
["empty Evidence checked", ""],
[
"labels outside Evidence checked",
`- malformed entry only
Notes:
- Claim: appears outside the evidence section.
- Evidence: appears outside the evidence section.
- Result: appears outside the evidence section.`,
],
[
"missing Result",
`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts`,
],
[
"invalid Result enum",
`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: guessed`,
],
[
"malformed indentation",
`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported`,
],
])("rejects a completed Truth Document report with %s", async (_name, evidenceChecked) => {
const result = await runDocumentReport(documentReportWithEvidence(evidenceChecked));
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("Evidence checked");
});
it.each([
["validate-document-report"],
["validate-write-lease"],
])("rejects a completed Truth Document report when %s ran and failed", async (helperId) => {
const report = documentReportWithEvidence(`- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported`).replace(
`- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`,
`- ${helperId}: ran, failed`,
);
const result = await runDocumentReport(report);
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("ran, failed");
});
it.each([
[
"block list",
`allowedWrites:
- docs/truth/**
forbiddenWrites:
- src/**
`,
],
["flow list", "allowedWrites: [docs/truth/**]\nforbiddenWrites: []\n"],
[
"quoted paths and comments",
`# parent-issued lease
allowedWrites:
- "docs/truth/**" # canonical truth docs
forbiddenWrites:
- 'src/**' # functional code
`,
],
[
"worker report with nested lease",
`status: completed
worker: truth_doc_writer
workflow: truthmark-sync
writeLease:
allowedWrites: [docs/truth/**]
forbiddenWrites:
- src/**
filesChanged:
- docs/truth/workflows/overview.md
`,
],
])("accepts write-lease %s YAML", async (_name, lease) => {
const result = await runWriteLease({
lease,
changedFiles: "docs/truth/workflows/overview.md\n",
});
expect(result.exitCode).toBe(0);
expect(result.json.ok).toBe(true);
});
it.each([
[
"invalid YAML",
`allowedWrites:
- docs/truth/**
forbiddenWrites: [src/**
`,
"invalid write lease YAML",
],
[
"non-list allowedWrites",
`allowedWrites: docs/truth/**
forbiddenWrites: []
`,
"allowedWrites must be an array of strings",
],
[
"non-list forbiddenWrites",
`allowedWrites: [docs/truth/**]
forbiddenWrites: src/**
`,
"forbiddenWrites must be an array of strings",
],
[
"non-string allowedWrites item",
`allowedWrites:
- docs/truth/**
- 42
forbiddenWrites: []
`,
"allowedWrites must be an array of strings",
],
])("rejects write-lease %s", async (_name, lease, expectedError) => {
const result = await runWriteLease({
lease,
changedFiles: "docs/truth/workflows/overview.md\n",
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain(expectedError);
});
it("rejects write-lease changes outside allowedWrites", async () => {
const result = await runCliHelper({
files: {
"lease.yml": `allowedWrites:
- docs/truth/**
forbiddenWrites:
- src/**
`,
"changed-files.txt": "src/init/init.ts\n",
},
args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("outside allowedWrites");
});
it.each([
["parent-directory changed file", "../outside.md", "invalid changed file path"],
["absolute changed file", "/docs/truth/workflows/overview.md", "invalid changed file path"],
["normalized-outside changed file", "docs/truth/../../src/init.ts", "invalid changed file path"],
])("rejects write-lease %s", async (_name, changedFiles, expectedError) => {
const result = await runWriteLease({
lease: `allowedWrites:
- docs/truth/**
forbiddenWrites:
- src/**
`,
changedFiles: `${changedFiles}\n`,
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain(expectedError);
});
it.each([
["parent-directory allowedWrites", "../docs/truth/**"],
["absolute allowedWrites", "/docs/truth/**"],
["normalized-outside allowedWrites", "docs/truth/../../src/**"],
])("rejects write-lease %s", async (_name, allowedWrite) => {
const result = await runWriteLease({
lease: `allowedWrites:
- ${allowedWrite}
forbiddenWrites: []
`,
changedFiles: "docs/truth/workflows/overview.md\n",
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("invalid allowedWrites path");
});
it("rejects unsupported write-lease glob patterns with manual-validation guidance", async () => {
const result = await runCliHelper({
files: {
"lease.yml": `allowedWrites:
- docs/**/*.md
forbiddenWrites: []
`,
"changed-files.txt": "docs/truth/workflows/overview.md\n",
},
args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"],
});
expect(result.exitCode).toBe(1);
expect(result.json.ok).toBe(false);
expect(result.json.errors?.join("\n")).toContain("manual-validation");
});
});
tests/agents/workflow-manifest.test.ts
+33
View File
@@ -55,6 +55,39 @@ describe("Truthmark workflow manifest", () => {
}
});
it("declares optional helper metadata with manual fallbacks", () => {
const sync = getTruthmarkWorkflow("truthmark-sync");
const document = getTruthmarkWorkflow("truthmark-document");
expect(sync.helpers?.map((helper) => helper.id)).toEqual([
"validate-sync-report",
"validate-write-lease",
]);
expect(document.helpers?.map((helper) => helper.id)).toEqual([
"validate-document-report",
"validate-write-lease",
]);
for (const workflow of [sync, document]) {
expect(workflow.reportSections).toContain("Helper scripts");
for (const helper of workflow.helpers ?? []) {
expect(helper.optional).toBe(true);
expect(helper.runner).toMatch(/^truthmark>=/u);
expect(helper.command.argv).toEqual(
expect.arrayContaining(["truthmark", "validate", "--json"]),
);
expect(helper.command.argv.join(" ")).not.toContain("node scripts/");
expect(helper.inputs.length).toBeGreaterThan(0);
expect(helper.output).toBe("json");
expect(helper.writes).toBe(false);
expect(helper.fallback).toMatch(/manual/i);
}
}
expect(getTruthmarkWorkflow("truthmark-preview").helpers).toBeUndefined();
});
it("defines read-only and write-capable subagent recommendations by workflow", () => {
expect(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-preview"].subagents).toEqual([
"truth_route_auditor",
tests/cli/validate.test.ts
+173
View File
@@ -0,0 +1,173 @@
import { describe, expect, it } from "vitest";
import { runCli } from "../helpers/run-cli.js";
import { createTempRepo } from "../helpers/temp-repo.js";
const validSyncReport = `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs updated:
- docs/truth/init-and-scaffold.md
Evidence checked:
- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported
Helper scripts:
- validate-sync-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
const validDocumentReport = `Truth Document: completed
Implementation reviewed:
- src/templates/workflow-surfaces.ts
Ownership reviewed:
- docs/truthmark/areas.md
Truth docs created:
- docs/truth/workflows/helpers.md
Evidence checked:
- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported
Helper scripts:
- validate-document-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
describe("truthmark validate CLI helpers", () => {
it("validates sync reports through the Truthmark CLI", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("report.md", validSyncReport);
const result = await runCli(["validate", "sync-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
command: string;
summary: string;
diagnostics: unknown[];
data?: {
validation?: {
ok: boolean;
helper: string;
checks?: string[];
};
};
};
expect(result.exitCode).toBe(0);
expect(output).toMatchObject({
command: "validate sync-report",
summary: "Validation passed",
diagnostics: [],
data: {
validation: { ok: true, helper: "validate-sync-report" },
},
});
expect(output.data?.validation?.checks).toContain("status: completed");
} finally {
await repo.cleanup();
}
});
it("validates document reports through the Truthmark CLI", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("report.md", validDocumentReport);
const result = await runCli(["validate", "document-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; helper: string } };
};
expect(result.exitCode).toBe(0);
expect(output).toMatchObject({
command: "validate document-report",
summary: "Validation passed",
diagnostics: [],
data: {
validation: { ok: true, helper: "validate-document-report" },
},
});
} finally {
await repo.cleanup();
}
});
it("rejects failed helper statuses in completed sync reports", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile(
"report.md",
validSyncReport.replace("validate-sync-report: ran, passed", "validate-sync-report: ran, failed"),
);
const result = await runCli(["validate", "sync-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; errors?: string[] } };
};
expect(result.exitCode).toBe(1);
expect(output).toMatchObject({
command: "validate sync-report",
summary: "Validation failed",
diagnostics: [],
data: { validation: { ok: false } },
});
expect(output.data?.validation?.errors?.join("\n")).toContain("ran, failed");
} finally {
await repo.cleanup();
}
});
it("rejects write-lease path traversal and Windows absolute changed paths", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("lease.yml", "allowedWrites:\n - docs/truth/**\nforbiddenWrites:\n - src/**\n");
await repo.writeFile("changed-files.txt", "C:/repo/docs/truth/secret.md\n");
const result = await runCli(
["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"],
{ cwd: repo.rootDir },
);
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; errors?: string[] } };
};
expect(result.exitCode).toBe(1);
expect(output).toMatchObject({
command: "validate write-lease",
summary: "Validation failed",
diagnostics: [],
data: { validation: { ok: false } },
});
expect(output.data?.validation?.errors?.join("\n")).toContain(
"invalid changed file path",
);
} finally {
await repo.cleanup();
}
});
});
tests/init/init.test.ts
+79 -60
View File
@@ -185,9 +185,18 @@ describe("runInit", () => {
const documentOpenCodeSkill = await repo.readFile(
".opencode/skills/truthmark-document/SKILL.md",
);
const documentHelperManifest = await repo.readFile(
".codex/skills/truthmark-document/helper-manifest.yml",
);
const syncSkill = await repo.readFile(
".codex/skills/truthmark-sync/SKILL.md",
);
const syncHelperManifest = await repo.readFile(
".codex/skills/truthmark-sync/helper-manifest.yml",
);
const syncHelperPolicy = await repo.readFile(
".codex/skills/truthmark-sync/support/helper-policy.md",
);
const syncProcedure = await repo.readFile(
".codex/skills/truthmark-sync/support/procedure.md",
);
@@ -361,7 +370,9 @@ describe("runInit", () => {
expect(documentSkill).not.toContain("Truth Document: completed");
expect(documentSkill).toContain("must not write functional code");
expect(documentSkill).not.toContain("truth_doc_writer");
expect(documentSkill).toContain("support/helper-policy.md");
expect(documentSkill.split("\n").length).toBeLessThanOrEqual(55);
expect(documentHelperManifest).toContain("validate-document-report:");
expect(documentSubagents).toContain("truth_doc_writer");
expect(documentSubagents).toContain("write lease");
expect(documentReportTemplate).toContain("Truth Document: completed");
@@ -388,10 +399,31 @@ describe("runInit", () => {
expect(syncSkill).toContain("support/procedure.md");
expect(syncSkill).toContain("support/report-template.md");
expect(syncSkill).toContain("support/subagents-and-leases.md");
expect(syncSkill).toContain("support/helper-policy.md");
expect(syncSkill).not.toContain("validate-sync-report.mjs report.md");
expect(syncSkill).not.toContain("host supports subagent dispatch");
expect(syncSkill).not.toContain("truth_doc_writer");
expect(syncSkill).not.toContain("Truth Sync: completed");
expect(syncSkill.split("\n").length).toBeLessThanOrEqual(55);
expect(syncHelperManifest).toContain("validate-sync-report:");
expect(syncHelperManifest).toContain("optional: true");
expect(syncHelperManifest).toContain("runner: truthmark>=");
expect(syncHelperManifest).toContain("command:");
expect(syncHelperManifest).toContain("argv:");
expect(syncHelperManifest).toContain("- truthmark");
expect(syncHelperManifest).toContain("- validate");
expect(syncHelperManifest).toContain("- sync-report");
expect(syncHelperManifest).toContain("- <report-file>");
expect(syncHelperManifest).toContain("- --json");
expect(syncHelperManifest).not.toContain("cd .codex/skills/truthmark-sync");
expect(syncHelperManifest).not.toContain("node scripts/");
expect(syncHelperManifest).toContain("writes: false");
expect(syncHelperPolicy).toContain("Optional helper CLI commands");
expect(syncHelperPolicy).toContain("manual fallback");
expect(syncHelperPolicy).toContain("Helper scripts:");
await expect(
repo.readFile(".codex/skills/truthmark-sync/scripts/validate-sync-report.mjs"),
).rejects.toThrow();
expect(syncProcedure).toContain("host supports subagent dispatch");
expect(syncSubagents).toContain("truth_doc_writer");
expect(syncSubagents).toContain("write lease");
@@ -420,10 +452,14 @@ describe("runInit", () => {
expect(syncOpenCodeSubagents).toContain("@truth-route-auditor");
expect(syncOpenCodeSubagents).toContain("@truth-claim-verifier");
expect(syncOpenCodeSubagents).toContain("@truth-doc-writer");
expect(
await repo.readFile(".opencode/skills/truthmark-sync/helper-manifest.yml"),
).toContain("validate-sync-report:");
expect(syncCopilotPrompt).toContain("Copilot custom-agent mode:");
expect(syncCopilotPrompt).toContain("@truth-route-auditor");
expect(syncCopilotPrompt).toContain("@truth-claim-verifier");
expect(syncCopilotPrompt).toContain("@truth-doc-writer");
expect(syncCopilotPrompt).not.toContain("scripts/validate-sync-report.mjs");
expect(syncClaudeSkill).toContain("name: truthmark-sync");
expect(syncClaudeSkill).toContain(
"Use this skill automatically before finishing",
@@ -434,6 +470,9 @@ describe("runInit", () => {
expect(syncClaudeSubagents).toContain("truth-route-auditor subagent");
expect(syncClaudeSubagents).toContain("truth-claim-verifier subagent");
expect(syncClaudeSubagents).toContain("truth-doc-writer subagent");
expect(
await repo.readFile(".claude/skills/truthmark-sync/helper-manifest.yml"),
).toContain("validate-sync-report:");
expect(realizeSkill).toContain("name: truthmark-realize");
expect(realizeSkill).toContain("user-invocable: true");
expect(realizeSkill).toContain("may write functional code only");
@@ -474,6 +513,7 @@ describe("runInit", () => {
expect(previewClaudeSkill).toContain("name: truthmark-preview");
expect(previewCopilotPrompt).toContain("Truth Preview: completed");
expect(previewGeminiCommand).toContain("/truthmark:preview");
expect(previewGeminiCommand).not.toContain("helper-manifest.yml");
expect(checkSkill).toContain("name: truthmark-check");
expect(checkSkill).toContain("support/procedure.md");
expect(checkSkill).toContain("support/report-template.md");
@@ -742,6 +782,24 @@ Agent-specific:
expect(
await repo.readFile(".github/prompts/truthmark-document.prompt.md"),
).toContain("GitHub Copilot /truthmark-document");
await expect(
fs.stat(`${repo.rootDir}/.github/skills/truthmark-sync/SKILL.md`),
).resolves.toBeTruthy();
expect(
await repo.readFile(".github/skills/truthmark-sync/SKILL.md"),
).toContain("Use as a Copilot agent skill.");
expect(
await repo.readFile(".github/skills/truthmark-sync/SKILL.md"),
).toContain("helper-manifest.yml");
expect(
await repo.readFile(".github/skills/truthmark-sync/support/subagents-and-leases.md"),
).toContain("@truth-doc-writer");
expect(
await repo.readFile(".github/skills/truthmark-document/helper-manifest.yml"),
).toContain("validate-document-report:");
expect(
await repo.readFile(".github/skills/truthmark-document/support/helper-policy.md"),
).not.toContain("scripts/validate-document-report.mjs");
await expect(fs.stat(`${repo.rootDir}/GEMINI.md`)).resolves.toBeTruthy();
await expect(
fs.stat(`${repo.rootDir}/.gemini/commands/truthmark/structure.toml`),
@@ -774,6 +832,27 @@ Agent-specific:
).toContain(
`description = "${getTruthmarkWorkflow("truthmark-realize").description}"`,
);
await expect(
fs.stat(`${repo.rootDir}/.gemini/skills/truthmark-sync/SKILL.md`),
).resolves.toBeTruthy();
expect(
await repo.readFile(".gemini/skills/truthmark-sync/SKILL.md"),
).toContain("Use as a Gemini CLI Agent Skill");
expect(
await repo.readFile(".gemini/skills/truthmark-sync/SKILL.md"),
).toContain("helper-manifest.yml");
expect(
await repo.readFile(".gemini/skills/truthmark-document/helper-manifest.yml"),
).toContain("validate-document-report:");
expect(
await repo.readFile(".gemini/skills/truthmark-sync/support/subagents-and-leases.md"),
).toContain("@truth-doc-writer");
await expect(
fs.stat(`${repo.rootDir}/.gemini/agents/truth-route-auditor.md`),
).resolves.toBeTruthy();
await expect(
fs.stat(`${repo.rootDir}/.gemini/agents/truth-doc-writer.md`),
).resolves.toBeTruthy();
const geminiInstructions = await repo.readFile("GEMINI.md");
expect(geminiInstructions).not.toContain("/truthmark:sync");
expect(geminiInstructions).toContain(
@@ -864,66 +943,6 @@ ignore: []
}
});
it("installs GitHub Copilot prompt files when only github-copilot is configured", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile(
".truthmark/config.yml",
`version: 1
platforms:
- github-copilot
authority:
- docs/truthmark/areas.md
instruction_targets:
- AGENTS.md
frontmatter:
required: []
recommended: []
ignore: []
`,
);
await runInit(repo.rootDir);
expect(await repo.readFile(".github/copilot-instructions.md")).toContain(
"Truthmark Workflow",
);
expect(
await repo.readFile(".github/prompts/truthmark-sync.prompt.md"),
).toContain("GitHub Copilot /truthmark-sync");
expect(
await repo.readFile(".github/agents/truth-route-auditor.agent.md"),
).toContain("tools: [read, search]");
expect(
await repo.readFile(".github/agents/truth-claim-verifier.agent.md"),
).toContain("unsupportedClaims");
expect(
await repo.readFile(".github/prompts/truthmark-structure.prompt.md"),
).toContain("name: truthmark-structure");
expect(
await repo.readFile(".github/prompts/truthmark-structure.prompt.md"),
).toContain("@truth-route-auditor");
expect(
await repo.readFile(".github/prompts/truthmark-document.prompt.md"),
).toContain("GitHub Copilot /truthmark-document");
expect(
await repo.readFile(".github/prompts/truthmark-check.prompt.md"),
).toContain("name: truthmark-check");
expect(
await repo.readFile(".github/prompts/truthmark-realize.prompt.md"),
).toContain("GitHub Copilot /truthmark-realize");
expect(
await repo.readFile(".github/agents/truth-doc-reviewer.agent.md"),
).toContain("recommendedWorkflow");
await expect(
fs.stat(`${repo.rootDir}/.codex/skills/truthmark-sync/SKILL.md`),
).rejects.toThrow();
} finally {
await repo.cleanup();
}
});
it("preserves existing docs and authored AGENTS content while scaffolding hierarchy", async () => {
const repo = await createTempRepo();
tests/repo-index/build.test.ts
+16
View File
@@ -1,3 +1,5 @@
import fs from "node:fs/promises";
import { afterEach, describe, expect, it } from "vitest";
import { runConfig } from "../../src/config/command.js";
@@ -46,6 +48,20 @@ describe("buildRepoIndex", () => {
expect(result.routeMap.routes.some((route) => route.truthDocs.length > 0)).toBe(true);
});
it("skips tracked files that are deleted from the worktree", async () => {
const repo = await createTempRepo();
repos.push(repo);
await repo.writeFile("src/deleted.ts", "export const deleted = true;\n");
await repo.runGit(["add", "src/deleted.ts"]);
await repo.runGit(["commit", "-m", "track deleted fixture"]);
await fs.rm(`${repo.rootDir}/src/deleted.ts`);
const result = await buildRepoIndex(repo.rootDir);
expect(result.files.map((file) => file.path)).not.toContain("src/deleted.ts");
expect(result.exports.map((entry) => entry.path)).not.toContain("src/deleted.ts");
});
it("excludes files ignored by gitignore", async () => {
const repo = await createTempRepo();
repos.push(repo);
tests/sync/report.test.ts
+73 -9
View File
@@ -11,6 +11,7 @@ describe("Truth Sync reporting", () => {
it("renders completed handoff notes in the README shape", () => {
const report = renderTruthSyncCompletedReport({
changedCode: ["src/auth/session.ts"],
ownershipReviewed: ["docs/truthmark/areas/repository.md"],
truthDocsUpdated: ["docs/truth/authentication.md"],
evidenceChecked: [
{
@@ -30,6 +31,9 @@ describe("Truth Sync reporting", () => {
Changed code reviewed:
- src/auth/session.ts
Ownership reviewed:
- docs/truthmark/areas/repository.md
Truth docs updated:
- docs/truth/authentication.md
@@ -43,6 +47,7 @@ Notes:
expect(parseTruthSyncReport(report)).toEqual({
status: "completed",
changedCode: ["src/auth/session.ts"],
ownershipReviewed: ["docs/truthmark/areas/repository.md"],
truthDocsUpdated: ["docs/truth/authentication.md"],
evidenceChecked: [
{
@@ -58,6 +63,33 @@ Notes:
});
});
it("round-trips optional helper script statuses", () => {
const report = renderTruthSyncCompletedReport({
changedCode: ["src/auth/session.ts"],
ownershipReviewed: ["docs/truthmark/areas/repository.md"],
truthDocsUpdated: ["docs/truth/authentication.md"],
evidenceChecked: [
{
claim: "Session timeout behavior is documented in the authentication truth doc.",
evidence: ["src/auth/session.ts:12"],
result: "supported",
},
],
helperScripts: [
"validate-sync-report: ran, passed",
"validate-write-lease: skipped, no write lease used",
],
notes: ["Updated session timeout behavior."],
});
expect(parseTruthSyncReport(report)).toMatchObject({
helperScripts: [
"validate-sync-report: ran, passed",
"validate-write-lease: skipped, no write lease used",
],
});
});
it("renders skipped handoff notes in the README shape", () => {
expect(
renderTruthSyncSkippedReport({ reason: "documentation-only change" }),
@@ -104,19 +136,51 @@ Notes:
).toThrow("Evidence checked");
});
it("omits the manual review section when the file list is empty", () => {
expect(
it("rejects completed reports with empty claim or evidence content", () => {
expect(() =>
parseTruthSyncReport(`Truth Sync: completed
Changed code reviewed:
- src/auth/session.ts
Truth docs updated:
- docs/truth/authentication.md
Evidence checked:
- Claim:${" "}
Evidence: src/auth/session.ts:12
Result: supported
Notes:
- Updated session timeout behavior.`),
).toThrow("claim is required");
expect(() =>
parseTruthSyncReport(`Truth Sync: completed
Changed code reviewed:
- src/auth/session.ts
Truth docs updated:
- docs/truth/authentication.md
Evidence checked:
- Claim: Session timeout behavior is documented.
Evidence:${" "}
Result: supported
Notes:
- Updated session timeout behavior.`),
).toThrow("evidence is required");
});
it("rejects blocked reports without manual-review files", () => {
expect(() =>
renderTruthSyncBlockedReport({
reason: "routing repair is not allowed",
manualReviewFiles: [],
nextAction: "update routing metadata and rerun Truth Sync",
}),
).toBe(`Truth Sync: blocked
Reason:
- routing repair is not allowed
Next action:
- update routing metadata and rerun Truth Sync`);
).toThrow("Files requiring manual review must include at least one file");
});
});