Files
Merlin's CatGitHubCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>MerlinHCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
9fa9ac25a4 feat: add workflow eval framework (#32)
* feat: add workflow eval framework

Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows.

Move research notes under docs/research and migrate tests from Vitest to node:test.

* Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

---------

Co-authored-by: MerlinH <merlinh221@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
2026-06-30 20:57:55 +10:00

243 lines
6.9 KiB
TypeScript

import { describe, it } from "node:test";
import { expect } from "expect";
import {
TRUTHMARK_WRITE_WORKER_REPORT_FIELDS,
parseTruthmarkWriteWorkerReport,
type TruthmarkWriteLease,
validateTruthmarkWriteWorkerAcceptance,
validateTruthmarkWriteLeaseChanges,
} from "../../src/agents/write-lease.js";
const lease: TruthmarkWriteLease = {
workflow: "truthmark-sync",
worker: "truth_doc_writer",
shard: "workflow-sync-doc",
objective: "Update one routed workflow truth doc.",
requiredReads: [
".truthmark/config.yml",
"docs/truthmark/routes/areas.md",
"docs/truthmark/truth/workflows/truth-sync.md",
],
allowedWrites: [
"docs/truthmark/truth/workflows/truth-sync.md",
"docs/truthmark/routes/areas/**/*.md",
],
forbiddenWrites: ["src/**", ".codex/**", ".opencode/**"],
evidenceRequired: ["implemented source or generated surface for each claim"],
reportFields: [...TRUTHMARK_WRITE_WORKER_REPORT_FIELDS],
};
const completedReport = `
status: completed
worker: truth_doc_writer
workflow: truthmark-sync
shard: workflow-sync-doc
filesChanged:
- docs/truthmark/truth/workflows/truth-sync.md
claimsChecked:
- Sync parent validation claim
evidenceChecked:
- src/agents/write-lease.ts
offLeaseChanges: []
blockers: []
notes:
- Updated leased doc only.
`;
describe("validateTruthmarkWriteLeaseChanges", () => {
it("accepts only files covered by allowed writes", () => {
expect(
validateTruthmarkWriteLeaseChanges(lease, [
"docs/truthmark/truth/workflows/truth-sync.md",
"./docs/truthmark/routes/areas/workflows.md",
]),
).toMatchObject({
allowedChanges: expect.arrayContaining([
"docs/truthmark/truth/workflows/truth-sync.md",
"docs/truthmark/routes/areas/workflows.md",
]),
forbiddenChanges: [],
offLeaseChanges: [],
});
});
it("reports off-lease and explicitly forbidden changes", () => {
expect(
validateTruthmarkWriteLeaseChanges(lease, [
"docs/truthmark/truth/workflows/truth-sync.md",
"src/agents/truth-sync.ts",
".opencode/agents/truth-doc-writer.md",
"docs/truthmark/truth/workflows/truth-document.md",
]),
).toEqual({
allowedChanges: ["docs/truthmark/truth/workflows/truth-sync.md"],
forbiddenChanges: [
".opencode/agents/truth-doc-writer.md",
"src/agents/truth-sync.ts",
],
offLeaseChanges: [
".opencode/agents/truth-doc-writer.md",
"docs/truthmark/truth/workflows/truth-document.md",
"src/agents/truth-sync.ts",
],
});
});
});
describe("validateTruthmarkWriteWorkerAcceptance", () => {
it("accepts a completed worker report only when the actual diff matches the lease", () => {
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
}),
).toMatchObject({
status: "accepted",
reasons: [],
changeValidation: {
allowedChanges: ["docs/truthmark/truth/workflows/truth-sync.md"],
forbiddenChanges: [],
offLeaseChanges: [],
},
});
});
it("rejects worker success when the actual diff includes off-lease changes", () => {
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
actualChangedFiles: [
"docs/truthmark/truth/workflows/truth-sync.md",
"docs/truthmark/truth/workflows/truth-document.md",
],
}).reasons.map((reason) => reason.code),
).toEqual(["off-lease-actual-diff", "reported-files-mismatch"]);
});
it("rejects worker success when the actual diff includes forbidden changes", () => {
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
actualChangedFiles: [
"docs/truthmark/truth/workflows/truth-sync.md",
"src/agents/truth-sync.ts",
],
}).reasons.map((reason) => reason.code),
).toEqual([
"forbidden-actual-diff",
"off-lease-actual-diff",
"reported-files-mismatch",
]);
});
it("rejects reports that omit required fields", () => {
const report = parseTruthmarkWriteWorkerReport(`
status: completed
worker: truth_doc_writer
workflow: truthmark-sync
shard: workflow-sync-doc
filesChanged:
- docs/truthmark/truth/workflows/truth-sync.md
claimsChecked: []
evidenceChecked: []
offLeaseChanges: []
blockers: []
`);
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: report,
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
}).reasons.map((reason) => reason.code),
).toEqual(["missing-report-field"]);
});
it("rejects reports with invalid identity field types", () => {
const report = parseTruthmarkWriteWorkerReport(`
status: completed
worker: 7
workflow: truthmark-sync
shard: workflow-sync-doc
filesChanged:
- docs/truthmark/truth/workflows/truth-sync.md
claimsChecked: []
evidenceChecked: []
offLeaseChanges: []
blockers: []
notes: []
`);
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: report,
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
}).reasons.map((reason) => reason.code),
).toEqual(["invalid-report-field"]);
});
it("rejects completed reports that self-report off-lease changes or blockers", () => {
const report = parseTruthmarkWriteWorkerReport(`
status: completed
worker: truth_doc_writer
workflow: truthmark-sync
shard: workflow-sync-doc
filesChanged:
- docs/truthmark/truth/workflows/truth-sync.md
claimsChecked:
- Sync parent validation claim
evidenceChecked:
- src/agents/write-lease.ts
offLeaseChanges:
- docs/truthmark/truth/workflows/truth-document.md
blockers:
- ownership ambiguous
notes:
- Parent review required.
`);
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: report,
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
}).reasons.map((reason) => reason.code),
).toEqual([
"completed-with-reported-off-lease-changes",
"completed-with-blockers",
]);
});
it("blocks, rather than accepts, blocked worker reports with blockers", () => {
const report = parseTruthmarkWriteWorkerReport(`
status: blocked
worker: truth_doc_writer
workflow: truthmark-sync
shard: workflow-sync-doc
filesChanged: []
claimsChecked:
- Sync parent validation claim
evidenceChecked:
- src/agents/write-lease.ts
offLeaseChanges: []
blockers:
- ownership ambiguous
notes:
- Parent review required.
`);
expect(
validateTruthmarkWriteWorkerAcceptance({
lease,
workerReport: report,
actualChangedFiles: [],
}).status,
).toBe("blocked");
});
});