Files
truthmark/tests/cli/validate.test.ts
T
a083b703fa feat: add Truthmark health scorecard and bounded workflow contracts (#13)
* feat: add OpenSpec-driven workflow state

* docs: plan generated playbooks workflow contract

* feat: add truth health scorecard

Release Truthmark 2.1.0 with check JSON scorecard output.

Archive the completed OpenSpec pass 4 scorecard change and update generated truth surfaces.

* fix: unblock truthmark scorecard review

- restore and update coverage for scorecard/evidence/workflow behavior
- make truthmark-sync select a cheap local base when --base is omitted
- report malformed evidence YAML blocks as diagnostics instead of throwing

* fix: tighten workflow boundaries

* fix: fail closed when sync has no comparison base

* fix: tighten truth health review coverage

- only parse fenced YAML evidence blocks with top-level evidence markers
- convert coverage fixtures to valid v2 config and assert non-vacuous diagnostics
- update Truthmark truth docs for source traceability and workflow-state tests

* docs: clarify source traceability diagnostics

* docs: harden optional CLI workflow boundary

* docs: clarify product boundary is repo-local

* fix: bound truthmark workflow fallback behavior

- make live workflow preflight use the one-call instructions contract
- keep missing-CLI fallback bounded for Sync, Document, Structure, and Realize
- add routing and generated-surface coverage plus refreshed host surfaces

* test: cover no-cli workflow fallback bounds

- assert route-first fallback wording in generated workflow surfaces
- clarify progressive-disclosure support files are conditional
- align overview docs with bounded no-CLI behavior

* docs: route workflow eval coverage

* fix: remove workflow instructions preflight

* fix: remove stale workflow preflight wording

* fix: remove generic workflow validation blocks

* fix: avoid duplicated markdown context JSON

* fix: keep workflow status manifest-only

* docs: align truth docs with v2 hierarchy

---------

Co-authored-by: MerlinH <merlinh221@gmail.com>
2026-06-13 04:09:45 +10:00

174 lines
4.8 KiB
TypeScript

import { describe, expect, it } from "vitest";
import { runCli } from "../helpers/run-cli.js";
import { createTempRepo } from "../helpers/temp-repo.js";
const validSyncReport = `Truth Sync: completed
Changed code reviewed:
- src/init/init.ts
Ownership reviewed:
- docs/truthmark/routes/areas.md
Truth docs updated:
- docs/truthmark/truth/init-and-scaffold.md
Evidence checked:
- Claim: Init writes generated workflow files.
Evidence: src/init/init.ts
Result: supported
Helper scripts:
- validate-sync-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
const validDocumentReport = `Truth Document: completed
Implementation reviewed:
- src/templates/workflow-surfaces.ts
Ownership reviewed:
- docs/truthmark/routes/areas.md
Truth docs created:
- docs/truthmark/truth/workflows/helpers.md
Evidence checked:
- Claim: Helpers are optional.
Evidence: src/agents/workflow-manifest.ts
Result: supported
Helper scripts:
- validate-document-report: ran, passed
- validate-write-lease: skipped, no write lease used
Notes:
- Complete.
`;
describe("truthmark validate CLI helpers", () => {
it("validates sync reports through the Truthmark CLI", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("report.md", validSyncReport);
const result = await runCli(["validate", "sync-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
command: string;
summary: string;
diagnostics: unknown[];
data?: {
validation?: {
ok: boolean;
helper: string;
checks?: string[];
};
};
};
expect(result.exitCode).toBe(0);
expect(output).toMatchObject({
command: "validate sync-report",
summary: "Validation passed",
diagnostics: [],
data: {
validation: { ok: true, helper: "validate-sync-report" },
},
});
expect(output.data?.validation?.checks).toContain("status: completed");
} finally {
await repo.cleanup();
}
});
it("validates document reports through the Truthmark CLI", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("report.md", validDocumentReport);
const result = await runCli(["validate", "document-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; helper: string } };
};
expect(result.exitCode).toBe(0);
expect(output).toMatchObject({
command: "validate document-report",
summary: "Validation passed",
diagnostics: [],
data: {
validation: { ok: true, helper: "validate-document-report" },
},
});
} finally {
await repo.cleanup();
}
});
it("rejects failed helper statuses in completed sync reports", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile(
"report.md",
validSyncReport.replace("validate-sync-report: ran, passed", "validate-sync-report: ran, failed"),
);
const result = await runCli(["validate", "sync-report", "report.md", "--json"], {
cwd: repo.rootDir,
});
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; errors?: string[] } };
};
expect(result.exitCode).toBe(1);
expect(output).toMatchObject({
command: "validate sync-report",
summary: "Validation failed",
diagnostics: [],
data: { validation: { ok: false } },
});
expect(output.data?.validation?.errors?.join("\n")).toContain("ran, failed");
} finally {
await repo.cleanup();
}
});
it("rejects write-lease path traversal and Windows absolute changed paths", async () => {
const repo = await createTempRepo();
try {
await repo.writeFile("lease.yml", "allowedWrites:\n - docs/truthmark/truth/**\nforbiddenWrites:\n - src/**\n");
await repo.writeFile("changed-files.txt", "C:/repo/docs/truthmark/truth/secret.md\n");
const result = await runCli(
["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"],
{ cwd: repo.rootDir },
);
const output = JSON.parse(result.stdout) as {
data?: { validation?: { ok: boolean; errors?: string[] } };
};
expect(result.exitCode).toBe(1);
expect(output).toMatchObject({
command: "validate write-lease",
summary: "Validation failed",
diagnostics: [],
data: { validation: { ok: false } },
});
expect(output.data?.validation?.errors?.join("\n")).toContain(
"invalid changed file path",
);
} finally {
await repo.cleanup();
}
});
});