mirror of
https://github.com/merlinhu1/truthmark.git
synced 2026-08-25 07:53:25 +02:00
* feat: add workflow eval framework Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows. Move research notes under docs/research and migrate tests from Vitest to node:test. * Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: MerlinH <merlinh221@gmail.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
317 lines
7.2 KiB
TypeScript
317 lines
7.2 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import { expect } from "expect";
|
|
|
|
import { checkDecisionSections } from "../../src/checks/decisions.js";
|
|
import { createDefaultConfig } from "../../src/config/defaults.js";
|
|
import { createTempRepo } from "../helpers/temp-repo.js";
|
|
|
|
const decisionConfig = createDefaultConfig();
|
|
|
|
describe("checkDecisionSections", () => {
|
|
it("emits review diagnostics for current workflow docs missing engineering structure sections", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile(
|
|
"docs/truthmark/engineering/installed-workflows.md",
|
|
`# Installed Workflows
|
|
|
|
## Scope
|
|
|
|
Installed workflow truth.
|
|
|
|
## Triggers
|
|
|
|
Explicit workflow invocations.
|
|
|
|
## Execution Model
|
|
|
|
Agents inspect the checkout directly.
|
|
`,
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/truthmark/engineering/installed-workflows.md"],
|
|
[
|
|
{
|
|
path: "docs/truthmark/engineering/installed-workflows.md",
|
|
kind: "engineering-workflow",
|
|
kindSource: "explicit",
|
|
lane: "engineering",
|
|
laneSource: "inferred",
|
|
realizedBy: [],
|
|
realizes: [],
|
|
dependsOn: [],
|
|
},
|
|
],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([
|
|
expect.objectContaining({
|
|
category: "doc-structure",
|
|
severity: "review",
|
|
file: "docs/truthmark/engineering/installed-workflows.md",
|
|
message: expect.stringContaining("Purpose"),
|
|
}),
|
|
]);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
|
|
it("accepts canonical docs with active decisions and rationale", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile(
|
|
"docs/truthmark/engineering/installed-workflows.md",
|
|
`# Installed Workflows
|
|
|
|
## Scope
|
|
|
|
Installed workflow truth.
|
|
|
|
## Purpose
|
|
|
|
Defines installed workflow runtime behavior.
|
|
|
|
## Triggers
|
|
|
|
Explicit workflow invocations.
|
|
|
|
## Execution Model
|
|
|
|
Agents inspect the checkout directly.
|
|
|
|
## Current Implementation Behavior
|
|
|
|
Installed workflow surfaces provide the current runtime.
|
|
|
|
## Source References
|
|
|
|
- AGENTS.md
|
|
|
|
## Product Truth Links
|
|
|
|
None.
|
|
|
|
## Engineering Decisions
|
|
|
|
- Installed skills and AGENTS blocks are the workflow runtime.
|
|
|
|
## Rationale
|
|
|
|
This keeps installed repositories usable when the Truthmark package is unavailable.
|
|
|
|
## Maintenance Notes
|
|
|
|
Update when installed workflow surfaces change.
|
|
`,
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/truthmark/engineering/installed-workflows.md"],
|
|
[
|
|
{
|
|
path: "docs/truthmark/engineering/installed-workflows.md",
|
|
kind: "engineering-workflow",
|
|
kindSource: "explicit",
|
|
lane: "engineering",
|
|
laneSource: "inferred",
|
|
realizedBy: [],
|
|
realizes: [],
|
|
dependsOn: [],
|
|
},
|
|
],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([]);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
|
|
it("does not require decision sections in non-canonical notes or index files", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile("docs/notes/future.md", "# Future\n");
|
|
await repo.writeFile(
|
|
"docs/truthmark/engineering/README.md",
|
|
"# Current Feature Docs\n",
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/notes/future.md", "docs/truthmark/engineering/README.md"],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([]);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
|
|
it("emits review diagnostics when a routed behavior doc is missing scope and current behavior", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile(
|
|
"docs/truthmark/engineering/repository/overview.md",
|
|
`# Repository Overview
|
|
|
|
## Engineering Decisions
|
|
|
|
- Decision (2026-05-14): Keep routed truth bounded.
|
|
|
|
## Rationale
|
|
|
|
Bounded truth docs are easier to maintain.
|
|
`,
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/truthmark/engineering/repository/overview.md"],
|
|
[
|
|
{
|
|
path: "docs/truthmark/engineering/repository/overview.md",
|
|
kind: "engineering-behavior",
|
|
kindSource: "explicit",
|
|
lane: "engineering",
|
|
laneSource: "inferred",
|
|
realizedBy: [],
|
|
realizes: [],
|
|
dependsOn: [],
|
|
},
|
|
],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([
|
|
expect.objectContaining({
|
|
category: "doc-structure",
|
|
severity: "review",
|
|
file: "docs/truthmark/engineering/repository/overview.md",
|
|
message: expect.stringContaining("Scope"),
|
|
}),
|
|
]);
|
|
expect(diagnostics[0]?.message).toContain(
|
|
"Current Implementation Behavior",
|
|
);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
|
|
it("uses a valid frontmatter truth_kind when the doc is not routed", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile(
|
|
"docs/truthmark/engineering/contract-surface.md",
|
|
`---
|
|
truth_kind: engineering-contract
|
|
---
|
|
|
|
# Contract Surface
|
|
|
|
## Scope
|
|
|
|
Owns the CLI contract.
|
|
|
|
## Contract Surface
|
|
|
|
The CLI returns structured diagnostics.
|
|
|
|
## Inputs
|
|
|
|
Command arguments.
|
|
|
|
## Outputs
|
|
|
|
Structured JSON output.
|
|
`,
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/truthmark/engineering/contract-surface.md"],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([
|
|
expect.objectContaining({
|
|
category: "doc-structure",
|
|
severity: "review",
|
|
file: "docs/truthmark/engineering/contract-surface.md",
|
|
message: expect.stringContaining("Purpose"),
|
|
}),
|
|
]);
|
|
expect(diagnostics[0]?.message).toContain(
|
|
"Current Implementation Behavior",
|
|
);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
|
|
it("applies common engineering structure checks to defaulted routed docs", async () => {
|
|
const repo = await createTempRepo();
|
|
|
|
try {
|
|
await repo.writeFile(
|
|
"docs/README.md",
|
|
`# Docs Index
|
|
|
|
## Scope
|
|
|
|
Indexes the canonical docs tree.
|
|
|
|
## Engineering Decisions
|
|
|
|
- Keep the docs tree split by role.
|
|
|
|
## Rationale
|
|
|
|
This keeps onboarding and current truth separate.
|
|
`,
|
|
);
|
|
|
|
const diagnostics = await checkDecisionSections(
|
|
repo.rootDir,
|
|
decisionConfig,
|
|
["docs/README.md"],
|
|
[
|
|
{
|
|
path: "docs/README.md",
|
|
kind: "engineering-behavior",
|
|
kindSource: "defaulted",
|
|
lane: "engineering",
|
|
laneSource: "defaulted",
|
|
realizedBy: [],
|
|
realizes: [],
|
|
dependsOn: [],
|
|
},
|
|
],
|
|
);
|
|
|
|
expect(diagnostics).toEqual([
|
|
expect.objectContaining({
|
|
category: "doc-structure",
|
|
severity: "review",
|
|
file: "docs/README.md",
|
|
message: expect.stringContaining("Source References"),
|
|
}),
|
|
]);
|
|
} finally {
|
|
await repo.cleanup();
|
|
}
|
|
});
|
|
});
|