mirror of
https://github.com/merlinhu1/truthmark.git
synced 2026-08-25 07:53:25 +02:00
* feat: add workflow eval framework Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows. Move research notes under docs/research and migrate tests from Vitest to node:test. * Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: MerlinH <merlinh221@gmail.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
243 lines
6.9 KiB
TypeScript
243 lines
6.9 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import { expect } from "expect";
|
|
|
|
import {
|
|
TRUTHMARK_WRITE_WORKER_REPORT_FIELDS,
|
|
parseTruthmarkWriteWorkerReport,
|
|
type TruthmarkWriteLease,
|
|
validateTruthmarkWriteWorkerAcceptance,
|
|
validateTruthmarkWriteLeaseChanges,
|
|
} from "../../src/agents/write-lease.js";
|
|
|
|
const lease: TruthmarkWriteLease = {
|
|
workflow: "truthmark-sync",
|
|
worker: "truth_doc_writer",
|
|
shard: "workflow-sync-doc",
|
|
objective: "Update one routed workflow truth doc.",
|
|
requiredReads: [
|
|
".truthmark/config.yml",
|
|
"docs/truthmark/routes/areas.md",
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
],
|
|
allowedWrites: [
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"docs/truthmark/routes/areas/**/*.md",
|
|
],
|
|
forbiddenWrites: ["src/**", ".codex/**", ".opencode/**"],
|
|
evidenceRequired: ["implemented source or generated surface for each claim"],
|
|
reportFields: [...TRUTHMARK_WRITE_WORKER_REPORT_FIELDS],
|
|
};
|
|
|
|
const completedReport = `
|
|
status: completed
|
|
worker: truth_doc_writer
|
|
workflow: truthmark-sync
|
|
shard: workflow-sync-doc
|
|
filesChanged:
|
|
- docs/truthmark/truth/workflows/truth-sync.md
|
|
claimsChecked:
|
|
- Sync parent validation claim
|
|
evidenceChecked:
|
|
- src/agents/write-lease.ts
|
|
offLeaseChanges: []
|
|
blockers: []
|
|
notes:
|
|
- Updated leased doc only.
|
|
`;
|
|
|
|
describe("validateTruthmarkWriteLeaseChanges", () => {
|
|
it("accepts only files covered by allowed writes", () => {
|
|
expect(
|
|
validateTruthmarkWriteLeaseChanges(lease, [
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"./docs/truthmark/routes/areas/workflows.md",
|
|
]),
|
|
).toMatchObject({
|
|
allowedChanges: expect.arrayContaining([
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"docs/truthmark/routes/areas/workflows.md",
|
|
]),
|
|
forbiddenChanges: [],
|
|
offLeaseChanges: [],
|
|
});
|
|
});
|
|
|
|
it("reports off-lease and explicitly forbidden changes", () => {
|
|
expect(
|
|
validateTruthmarkWriteLeaseChanges(lease, [
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"src/agents/truth-sync.ts",
|
|
".opencode/agents/truth-doc-writer.md",
|
|
"docs/truthmark/truth/workflows/truth-document.md",
|
|
]),
|
|
).toEqual({
|
|
allowedChanges: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
forbiddenChanges: [
|
|
".opencode/agents/truth-doc-writer.md",
|
|
"src/agents/truth-sync.ts",
|
|
],
|
|
offLeaseChanges: [
|
|
".opencode/agents/truth-doc-writer.md",
|
|
"docs/truthmark/truth/workflows/truth-document.md",
|
|
"src/agents/truth-sync.ts",
|
|
],
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("validateTruthmarkWriteWorkerAcceptance", () => {
|
|
it("accepts a completed worker report only when the actual diff matches the lease", () => {
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
|
|
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
}),
|
|
).toMatchObject({
|
|
status: "accepted",
|
|
reasons: [],
|
|
changeValidation: {
|
|
allowedChanges: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
forbiddenChanges: [],
|
|
offLeaseChanges: [],
|
|
},
|
|
});
|
|
});
|
|
|
|
it("rejects worker success when the actual diff includes off-lease changes", () => {
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
|
|
actualChangedFiles: [
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"docs/truthmark/truth/workflows/truth-document.md",
|
|
],
|
|
}).reasons.map((reason) => reason.code),
|
|
).toEqual(["off-lease-actual-diff", "reported-files-mismatch"]);
|
|
});
|
|
|
|
it("rejects worker success when the actual diff includes forbidden changes", () => {
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: parseTruthmarkWriteWorkerReport(completedReport),
|
|
actualChangedFiles: [
|
|
"docs/truthmark/truth/workflows/truth-sync.md",
|
|
"src/agents/truth-sync.ts",
|
|
],
|
|
}).reasons.map((reason) => reason.code),
|
|
).toEqual([
|
|
"forbidden-actual-diff",
|
|
"off-lease-actual-diff",
|
|
"reported-files-mismatch",
|
|
]);
|
|
});
|
|
|
|
it("rejects reports that omit required fields", () => {
|
|
const report = parseTruthmarkWriteWorkerReport(`
|
|
status: completed
|
|
worker: truth_doc_writer
|
|
workflow: truthmark-sync
|
|
shard: workflow-sync-doc
|
|
filesChanged:
|
|
- docs/truthmark/truth/workflows/truth-sync.md
|
|
claimsChecked: []
|
|
evidenceChecked: []
|
|
offLeaseChanges: []
|
|
blockers: []
|
|
`);
|
|
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: report,
|
|
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
}).reasons.map((reason) => reason.code),
|
|
).toEqual(["missing-report-field"]);
|
|
});
|
|
|
|
it("rejects reports with invalid identity field types", () => {
|
|
const report = parseTruthmarkWriteWorkerReport(`
|
|
status: completed
|
|
worker: 7
|
|
workflow: truthmark-sync
|
|
shard: workflow-sync-doc
|
|
filesChanged:
|
|
- docs/truthmark/truth/workflows/truth-sync.md
|
|
claimsChecked: []
|
|
evidenceChecked: []
|
|
offLeaseChanges: []
|
|
blockers: []
|
|
notes: []
|
|
`);
|
|
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: report,
|
|
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
}).reasons.map((reason) => reason.code),
|
|
).toEqual(["invalid-report-field"]);
|
|
});
|
|
|
|
it("rejects completed reports that self-report off-lease changes or blockers", () => {
|
|
const report = parseTruthmarkWriteWorkerReport(`
|
|
status: completed
|
|
worker: truth_doc_writer
|
|
workflow: truthmark-sync
|
|
shard: workflow-sync-doc
|
|
filesChanged:
|
|
- docs/truthmark/truth/workflows/truth-sync.md
|
|
claimsChecked:
|
|
- Sync parent validation claim
|
|
evidenceChecked:
|
|
- src/agents/write-lease.ts
|
|
offLeaseChanges:
|
|
- docs/truthmark/truth/workflows/truth-document.md
|
|
blockers:
|
|
- ownership ambiguous
|
|
notes:
|
|
- Parent review required.
|
|
`);
|
|
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: report,
|
|
actualChangedFiles: ["docs/truthmark/truth/workflows/truth-sync.md"],
|
|
}).reasons.map((reason) => reason.code),
|
|
).toEqual([
|
|
"completed-with-reported-off-lease-changes",
|
|
"completed-with-blockers",
|
|
]);
|
|
});
|
|
|
|
it("blocks, rather than accepts, blocked worker reports with blockers", () => {
|
|
const report = parseTruthmarkWriteWorkerReport(`
|
|
status: blocked
|
|
worker: truth_doc_writer
|
|
workflow: truthmark-sync
|
|
shard: workflow-sync-doc
|
|
filesChanged: []
|
|
claimsChecked:
|
|
- Sync parent validation claim
|
|
evidenceChecked:
|
|
- src/agents/write-lease.ts
|
|
offLeaseChanges: []
|
|
blockers:
|
|
- ownership ambiguous
|
|
notes:
|
|
- Parent review required.
|
|
`);
|
|
|
|
expect(
|
|
validateTruthmarkWriteWorkerAcceptance({
|
|
lease,
|
|
workerReport: report,
|
|
actualChangedFiles: [],
|
|
}).status,
|
|
).toBe("blocked");
|
|
});
|
|
});
|