Files
Merlin's CatGitHubCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>MerlinHCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
9fa9ac25a4 feat: add workflow eval framework (#32)
* feat: add workflow eval framework

Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows.

Move research notes under docs/research and migrate tests from Vitest to node:test.

* Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

---------

Co-authored-by: MerlinH <merlinh221@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
2026-06-30 20:57:55 +10:00

115 lines
3.0 KiB
TypeScript

import { describe, it } from "node:test";
import { expect } from "expect";
import {
DIAGNOSTIC_CATEGORIES,
type CommandResult,
} from "../../src/output/diagnostic.js";
import { renderHuman, renderJson } from "../../src/output/render.js";
describe("output rendering", () => {
it("renders the command name and summary first for human output", () => {
const result: CommandResult = {
command: "check",
summary: "Found 2 issues to review.",
diagnostics: [
{
category: "config",
severity: "action",
message: "Created .truthmark/config.yml.",
},
],
};
const output = renderHuman(result);
const lines = output.split("\n");
expect(lines[0]).toBe("truthmark check");
expect(lines[1]).toBe("Found 2 issues to review.");
});
it("renders action, review, and error diagnostics consistently", () => {
const result: CommandResult = {
command: "check",
summary: "Found issues.",
diagnostics: [
{
category: "config",
severity: "action",
message: "Created .truthmark/config.yml.",
file: ".truthmark/config.yml",
},
{
category: "coverage",
severity: "review",
message: "No truth docs are mapped for src/auth/**.",
area: "Authentication",
},
{
category: "links",
severity: "error",
message: "Broken link to docs/api/authentication.md.",
},
],
};
expect(renderHuman(result)).toContain(
"[ACTION] config: Created .truthmark/config.yml. (file: .truthmark/config.yml)",
);
expect(renderHuman(result)).toContain(
"[REVIEW] coverage: No truth docs are mapped for src/auth/**. (area: Authentication)",
);
expect(renderHuman(result)).toContain(
"[ERROR] links: Broken link to docs/api/authentication.md.",
);
});
it("renders deterministic JSON that round-trips to the command result", () => {
const result: CommandResult = {
command: "init",
summary: "Created Truthmark files.",
diagnostics: [
{
category: "config",
severity: "action",
message: "Created .truthmark/config.yml.",
data: {
created: true,
},
},
],
data: {
files: [".truthmark/config.yml", "docs/truthmark/routes/areas.md"],
},
};
const parsed = JSON.parse(renderJson(result)) as CommandResult;
expect(parsed).toEqual(result);
});
it("exports the V1 diagnostic category list", () => {
expect(DIAGNOSTIC_CATEGORIES).toEqual([
"config",
"authority",
"frontmatter",
"links",
"area-index",
"coverage",
"truth-sync",
"realization",
"doc-structure",
"lane-shape",
"lane-drift",
"traceability",
"generated-surface",
"repo-index",
"impact",
"freshness",
"source-traceability",
"context-pack",
"workflow-state",
]);
});
});