mirror of
https://github.com/merlinhu1/truthmark.git
synced 2026-08-25 07:53:25 +02:00
* feat: add workflow eval framework Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows. Move research notes under docs/research and migrate tests from Vitest to node:test. * Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: MerlinH <merlinh221@gmail.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
105 lines
4.4 KiB
TypeScript
105 lines
4.4 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import { expect } from "expect";
|
|
|
|
import { createDefaultConfig } from "../../src/config/defaults.js";
|
|
import {
|
|
renderTruthCheckInstructions,
|
|
renderTruthStructureInstructions,
|
|
renderTruthSyncInstructions,
|
|
} from "../../src/agents/instructions.js";
|
|
|
|
describe("renderTruthSyncInstructions", () => {
|
|
it("renders a compact Truth Sync reminder for managed instruction files", () => {
|
|
const instructions = renderTruthSyncInstructions();
|
|
|
|
expect(instructions).toContain("### Truth Sync");
|
|
expect(instructions).toContain("Automatic finish-time trigger");
|
|
expect(instructions).toContain(
|
|
"use the truthmark-sync skill before finishing",
|
|
);
|
|
expect(instructions).toContain(
|
|
"OpenCode /skill truthmark-sync; Codex /truthmark-sync or $truthmark-sync; Claude Code /truthmark-sync; GitHub Copilot /truthmark-sync; Antigravity @truthmark-sync; Cursor /truthmark-sync",
|
|
);
|
|
expect(instructions).toContain(
|
|
"staged, unstaged, and untracked functional code files",
|
|
);
|
|
expect(instructions).toContain("Run relevant tests before finishing");
|
|
expect(instructions).toContain(
|
|
"Support new or changed behavior-bearing truth claims with checkout evidence",
|
|
);
|
|
expect(instructions).toContain("documentation-only change");
|
|
expect(instructions).toContain("Explicit invocation runs immediately");
|
|
expect(instructions).toContain(
|
|
"Later functional-code changes need a fresh finish-time review",
|
|
);
|
|
expect(instructions).toContain("must not rewrite functional code");
|
|
expect(instructions).toContain("host supports subagent dispatch");
|
|
expect(instructions).toContain(
|
|
"WorkflowState and ImpactSet are optional compact derived context",
|
|
);
|
|
expect(instructions).toContain(
|
|
"If routing is missing, stale, broad, overloaded, catch-all, or cannot map changed code to a bounded truth owner",
|
|
);
|
|
expect(instructions).toContain("run Truth Structure before syncing");
|
|
expect(instructions).toContain(
|
|
"otherwise stop and recommend Truth Structure",
|
|
);
|
|
expect(instructions).not.toContain(".truthmark/local.yml");
|
|
expect(instructions).not.toContain("truth_sync.sync_agent");
|
|
expect(instructions).not.toContain("Truth Sync: completed");
|
|
expect(instructions).not.toContain("Truth Sync: skipped");
|
|
expect(instructions).not.toContain("truthmark packet --changed");
|
|
expect(instructions).not.toContain(
|
|
"truthmark check --json --workflow truth-sync",
|
|
);
|
|
});
|
|
|
|
it("keeps the managed Truth Sync reminder small", () => {
|
|
const instructions = renderTruthSyncInstructions();
|
|
const lines = instructions.split("\n");
|
|
|
|
expect(lines.slice(0, 4).join("\n")).toContain("Truth Sync");
|
|
expect(lines.length).toBeLessThanOrEqual(24);
|
|
});
|
|
|
|
it("uses the configured route index in the compact Sync reminder", () => {
|
|
const config = createDefaultConfig();
|
|
config.truthmark.paths.routesIndex = "docs/routes/index.md";
|
|
config.truthmark.paths.routeAreasRoot = "docs/routes/areas";
|
|
|
|
const instructions = renderTruthSyncInstructions(config);
|
|
|
|
expect(instructions).toContain("docs/routes/index.md; docs/routes/areas/");
|
|
expect(instructions).toContain("only when present");
|
|
expect(instructions).not.toContain(
|
|
"docs/truthmark/routes/areas.md; docs/truthmark/routes/areas/",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("agent-native workflow instructions", () => {
|
|
it("renders Truth Structure and Truth Check summaries", () => {
|
|
expect(renderTruthStructureInstructions()).toContain("truthmark-structure");
|
|
expect(renderTruthStructureInstructions()).toContain(
|
|
"configured route files when present",
|
|
);
|
|
expect(renderTruthStructureInstructions()).toContain(
|
|
"missing, stale, broad, overloaded, catch-all, unrouteable",
|
|
);
|
|
expect(renderTruthStructureInstructions()).toContain(
|
|
"canonical current-truth destinations",
|
|
);
|
|
expect(renderTruthStructureInstructions()).toContain("topology pressure");
|
|
expect(renderTruthStructureInstructions()).toContain(
|
|
"If the skill is unavailable",
|
|
);
|
|
expect(renderTruthCheckInstructions()).toContain("truthmark-check");
|
|
expect(renderTruthCheckInstructions()).toContain(
|
|
"run the truthmark check command only when available for additional validation",
|
|
);
|
|
expect(renderTruthCheckInstructions()).toContain(
|
|
"inspect the checkout directly when the command is unavailable",
|
|
);
|
|
});
|
|
});
|