From 9fa9ac25a4843bfbfab1f3561e16aa7585147695 Mon Sep 17 00:00:00 2001 From: Merlin's Cat Date: Tue, 30 Jun 2026 20:57:55 +1000 Subject: [PATCH] feat: add workflow eval framework (#32) * feat: add workflow eval framework Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows. Move research notes under docs/research and migrate tests from Vitest to node:test. * Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: MerlinH Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .gitignore | 4 + docs/README.md | 24 +- ...-evidence-backed-truth-workflows-design.md | 0 ...poagent-ai-doc-gen-truthmark-comparison.md | 0 ...6-05-14-truth-doc-template-kinds-design.md | 2 +- ...2026-05-15-agent-skills-workflow-review.md | 0 ...write-capable-opencode-subagents-design.md | 0 ...-skill-helper-script-portability-design.md | 0 ...-05-25-truthmark-portal-workflow-design.md | 2 +- .../2026-06-01-openspec-comparison.md | 0 .../2026-06-01-openspec-improvement-passes.md | 2 +- ...6-15-claim-ledger-architecture-proposal.md | 0 ...nual-agent-skill-quality-eval-framework.md | 27 + ...t-workflow-literature-review-2026-05-16.md | 4 +- ...ll-agent-workflow-quality-uplift-review.md | 30 +- ..._competitive_research_report_2026-05-15.md | 0 .../workflows/installed-workflow-runtime.md | 23 +- docs/truthmark/routes/areas.md | 2 + .../routes/areas/installed-workflows.md | 2 + eslint.config.js | 2 +- examples/github-actions/truthmark-impact.yml | 61 - openspec/config.yaml | 20 + package-lock.json | 1551 +++++------------ package.json | 12 +- .../judge-agent-run.mjs | 45 + .../run-agent-scenario.mjs | 931 ++++++++++ tests/agents/instructions.test.ts | 47 +- tests/agents/prompts.test.ts | 19 +- tests/agents/truth-check.test.ts | 38 +- tests/agents/truth-document.test.ts | 80 +- tests/agents/truth-sync.test.ts | 58 +- tests/agents/workflow-helper-scripts.test.ts | 290 ++- tests/agents/write-lease.test.ts | 3 +- tests/checks/branch-scope.test.ts | 10 +- tests/checks/check-truth-kinds.test.ts | 3 +- tests/checks/check.test.ts | 19 +- tests/checks/decisions.test.ts | 3 +- tests/checks/frontmatter.test.ts | 3 +- tests/checks/scorecard.test.ts | 150 +- tests/cli/build-artifact.test.ts | 57 +- tests/cli/check-workflow.test.ts | 10 +- tests/cli/help.test.ts | 7 +- tests/cli/index-impact-context.test.ts | 107 +- tests/cli/program.test.ts | 7 +- tests/cli/validate.test.ts | 67 +- tests/config/config-command.test.ts | 39 +- tests/config/load.test.ts | 72 +- tests/evidence/validate.test.ts | 3 +- tests/freshness/check.test.ts | 7 +- tests/fs/paths.test.ts | 3 +- tests/generation/registry.test.ts | 14 +- tests/generation/truth-doc-update.test.ts | 7 +- tests/generation/validate.test.ts | 26 +- tests/git/repository.test.ts | 68 +- tests/helpers/temp-repo.test.ts | 5 +- tests/impact/build.test.ts | 5 +- tests/init/init-instructions.test.ts | 11 +- tests/init/truth-doc-templates.test.ts | 13 +- .../agent-workflow-contract.test.ts | 15 +- tests/integration/branch-scope.test.ts | 25 +- tests/integration/init-check-workflow.test.ts | 3 +- tests/markdown/hash.test.ts | 5 +- tests/markdown/parse.test.ts | 13 +- tests/output/render.test.ts | 3 +- tests/package-files.test.ts | 3 +- tests/product-boundary.test.ts | 3 +- tests/realize/report.test.ts | 9 +- tests/repo-index/build.test.ts | 3 +- tests/repo-index/route-map.test.ts | 3 +- tests/routing/area-resolver.test.ts | 23 +- tests/routing/areas-markdown-globs.test.ts | 3 +- tests/routing/areas.test.ts | 12 +- tests/sync/policy.test.ts | 3 +- tests/sync/report.test.ts | 43 +- tests/sync/surfaces.test.ts | 88 +- tests/templates/generated-surfaces.test.ts | 131 +- tests/templates/github-action.test.ts | 3 +- tests/truth/docs.test.ts | 3 +- tests/version.test.ts | 3 +- tests/workflow-state/build.test.ts | 185 +- vitest.config.ts | 11 - workflow-eval-framwork/README.md | 88 + workflow-eval-framwork/catalog.yaml | 59 + .../examples/human-review-accepted.yaml | 7 + .../examples/human-review-rejected.yaml | 9 + workflow-eval-framwork/failure-taxonomy.md | 20 + .../fake-agents/forbidden-write.mjs | 8 + .../fake-agents/invalid-report.mjs | 7 + .../fake-agents/judge-fail.mjs | 12 + .../fake-agents/judge-pass.mjs | 33 + .../fake-agents/no-usage-pass.mjs | 35 + workflow-eval-framwork/fake-agents/pass.mjs | 54 + workflow-eval-framwork/improvement-loop.md | 17 + workflow-eval-framwork/judges/diff-grader.md | 4 + .../judges/report-grader.md | 4 + workflow-eval-framwork/judges/trace-grader.md | 4 + .../judges/triggering-grader.md | 4 + .../rubrics/shared-write-boundaries.yaml | 18 + .../rubrics/truthmark-check.yaml | 17 + .../rubrics/truthmark-document.yaml | 18 + .../rubrics/truthmark-realize.yaml | 18 + .../rubrics/truthmark-structure.yaml | 17 + .../rubrics/truthmark-sync.yaml | 19 + workflow-eval-framwork/runs/.gitkeep | 0 .../general-question-no-write/expected.yaml | 26 + .../fixture/.truthmark/config.yml | 5 + .../fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../general-question-no-write/prompt.md | 10 + .../general-question-no-write/scenario.yaml | 21 + .../unsupported-claim-audit/expected.yaml | 27 + .../fixture/.truthmark/config.yml | 5 + .../unsupported-claim-audit/fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../unsupported-claim-audit/prompt.md | 11 + .../unsupported-claim-audit/scenario.yaml | 21 + .../expected.yaml | 28 + .../fixture/.truthmark/config.yml | 5 + .../fixture/README.md | 5 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../prompt.md | 12 + .../scenario.yaml | 21 + .../expected.yaml | 25 + .../fixture/.truthmark/config.yml | 5 + .../fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../prompt.md | 12 + .../scenario.yaml | 21 + .../broad-route-repair/expected.yaml | 33 + .../fixture/.truthmark/config.yml | 5 + .../broad-route-repair/fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 16 + .../broad-route-repair/fixture/package.json | 10 + .../broad-route-repair/fixture/src/api.ts | 8 + .../broad-route-repair/fixture/src/cache.ts | 19 + .../broad-route-repair/fixture/src/metrics.ts | 13 + .../broad-route-repair/fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../broad-route-repair/prompt.md | 11 + .../broad-route-repair/scenario.yaml | 21 + .../bootstrap-route-blocked/expected.yaml | 29 + .../fixture/.truthmark/config.yml | 5 + .../bootstrap-route-blocked/fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/bootstrap-routing.md | 11 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 13 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../bootstrap-route-blocked/prompt.md | 12 + .../bootstrap-route-blocked/scenario.yaml | 21 + .../docs-only-skip/expected.yaml | 24 + .../fixture/.truthmark/config.yml | 5 + .../docs-only-skip/fixture/AGENTS.md | 13 + .../docs-only-skip/fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../docs-only-skip/fixture/package.json | 10 + .../docs-only-skip/fixture/src/api.ts | 8 + .../docs-only-skip/fixture/src/cache.ts | 19 + .../docs-only-skip/fixture/src/metrics.ts | 13 + .../docs-only-skip/fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../truthmark-sync/docs-only-skip/prompt.md | 12 + .../docs-only-skip/scenario.yaml | 23 + .../truthmark-sync/docs-only-skip/setup.patch | 5 + .../stale-engineering-truth/expected.yaml | 29 + .../fixture/.truthmark/config.yml | 5 + .../stale-engineering-truth/fixture/README.md | 5 + .../engineering/behaviors/widget-cache.md | 19 + .../engineering/contracts/widget-api.md | 18 + .../operations/widget-observability.md | 16 + .../product/capabilities/widget-cache.md | 18 + .../fixture/docs/truthmark/routes/areas.md | 20 + .../docs/truthmark/routes/areas/fixture.md | 31 + .../fixture/package.json | 10 + .../fixture/src/api.ts | 8 + .../fixture/src/cache.ts | 19 + .../fixture/src/metrics.ts | 13 + .../fixture/src/widget.ts | 10 + .../fixture/tests/widget-cache.test.ts | 15 + .../stale-engineering-truth/prompt.md | 11 + .../stale-engineering-truth/scenario.yaml | 23 + .../stale-engineering-truth/setup.patch | 12 + .../schemas/catalog.schema.json | 33 + .../schemas/expected.schema.json | 32 + .../schemas/human-review.schema.json | 12 + .../schemas/judge-results.schema.json | 22 + .../schemas/run.schema.json | 27 + .../schemas/scenario.schema.json | 32 + 249 files changed, 5518 insertions(+), 1829 deletions(-) rename {research => docs/research}/2026-05-14-evidence-backed-truth-workflows-design.md (100%) rename {research => docs/research}/2026-05-14-repoagent-ai-doc-gen-truthmark-comparison.md (100%) rename {research => docs/research}/2026-05-14-truth-doc-template-kinds-design.md (99%) rename {research => docs/research}/2026-05-15-agent-skills-workflow-review.md (100%) rename {research => docs/research}/2026-05-16-write-capable-opencode-subagents-design.md (100%) rename {research => docs/research}/2026-05-18-skill-helper-script-portability-design.md (100%) rename {research => docs/research}/2026-05-25-truthmark-portal-workflow-design.md (99%) rename {research => docs/research}/2026-06-01-openspec-comparison.md (100%) rename {research => docs/research}/2026-06-01-openspec-improvement-passes.md (99%) rename {research => docs/research}/2026-06-15-claim-ledger-architecture-proposal.md (100%) create mode 100644 docs/research/2026-06-29-manual-agent-skill-quality-eval-framework.md rename {research => docs/research}/prompt-skill-agent-workflow-literature-review-2026-05-16.md (99%) rename {research => docs/research}/prompt-skill-agent-workflow-quality-uplift-review.md (94%) rename {research => docs/research}/truthmark_competitive_research_report_2026-05-15.md (100%) delete mode 100644 examples/github-actions/truthmark-impact.yml create mode 100644 openspec/config.yaml create mode 100644 scripts/workflow-eval-framwork/judge-agent-run.mjs create mode 100644 scripts/workflow-eval-framwork/run-agent-scenario.mjs delete mode 100644 vitest.config.ts create mode 100644 workflow-eval-framwork/README.md create mode 100644 workflow-eval-framwork/catalog.yaml create mode 100644 workflow-eval-framwork/examples/human-review-accepted.yaml create mode 100644 workflow-eval-framwork/examples/human-review-rejected.yaml create mode 100644 workflow-eval-framwork/failure-taxonomy.md create mode 100644 workflow-eval-framwork/fake-agents/forbidden-write.mjs create mode 100644 workflow-eval-framwork/fake-agents/invalid-report.mjs create mode 100644 workflow-eval-framwork/fake-agents/judge-fail.mjs create mode 100644 workflow-eval-framwork/fake-agents/judge-pass.mjs create mode 100644 workflow-eval-framwork/fake-agents/no-usage-pass.mjs create mode 100644 workflow-eval-framwork/fake-agents/pass.mjs create mode 100644 workflow-eval-framwork/improvement-loop.md create mode 100644 workflow-eval-framwork/judges/diff-grader.md create mode 100644 workflow-eval-framwork/judges/report-grader.md create mode 100644 workflow-eval-framwork/judges/trace-grader.md create mode 100644 workflow-eval-framwork/judges/triggering-grader.md create mode 100644 workflow-eval-framwork/rubrics/shared-write-boundaries.yaml create mode 100644 workflow-eval-framwork/rubrics/truthmark-check.yaml create mode 100644 workflow-eval-framwork/rubrics/truthmark-document.yaml create mode 100644 workflow-eval-framwork/rubrics/truthmark-realize.yaml create mode 100644 workflow-eval-framwork/rubrics/truthmark-structure.yaml create mode 100644 workflow-eval-framwork/rubrics/truthmark-sync.yaml create mode 100644 workflow-eval-framwork/runs/.gitkeep create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/bootstrap-routing.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/AGENTS.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/setup.patch create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/expected.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/.truthmark/config.yml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/README.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/behaviors/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas/fixture.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/package.json create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/api.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/cache.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/metrics.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/widget.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/tests/widget-cache.test.ts create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/prompt.md create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml create mode 100644 workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/setup.patch create mode 100644 workflow-eval-framwork/schemas/catalog.schema.json create mode 100644 workflow-eval-framwork/schemas/expected.schema.json create mode 100644 workflow-eval-framwork/schemas/human-review.schema.json create mode 100644 workflow-eval-framwork/schemas/judge-results.schema.json create mode 100644 workflow-eval-framwork/schemas/run.schema.json create mode 100644 workflow-eval-framwork/schemas/scenario.schema.json diff --git a/.gitignore b/.gitignore index 7bb3207..8c12f29 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ Thumbs.db .hermes/ .lean-ctx/graph.db .lean-ctx/graph.meta.json + +# Manual workflow eval run artifacts +workflow-eval-framwork/runs/* +!workflow-eval-framwork/runs/.gitkeep diff --git a/docs/README.md b/docs/README.md index 78ef744..f421dc6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,7 @@ source_of_truth: ## Purpose -`docs/` is Truthmark's canonical repository documentation tree. It keeps repository-wide agent rules, reusable standards, current architecture, and current lane-separated truth docs separate from onboarding copy and historical planning notes. +`docs/` is Truthmark's repository documentation tree. It keeps repository-wide agent rules, reusable standards, current architecture, current lane-separated truth docs, and non-canonical research notes separate from onboarding copy. `AGENTS.md` is the agent entry point, but it delegates repository-wide rules to [docs/ai/repo-rules.md](ai/repo-rules.md). [README.md](../README.md) remains the human onboarding and product entry point. [docs/user-guide.md](user-guide.md) owns detailed human-facing command, workflow, and configuration guidance. `.truthmark/config.yml` defines the committed hierarchy contract. @@ -35,6 +35,7 @@ Repository-wide conflict order and completion policy live in [docs/ai/repo-rules - `docs/truthmark/product/` for product capability promises, boundaries, and acceptance criteria - `docs/truthmark/engineering/` for current implementation behavior, architecture, contracts, workflows, and operations - `docs/architecture/` for repository-specific architecture guardrails +- `docs/research/` for non-canonical research and planning notes - `docs/truthmark/templates/` for editable scaffold templates used to create new docs - `docs/truthmark/engineering/contracts/` for stable contracts the CLI exposes @@ -45,16 +46,17 @@ Repository-wide conflict order and completion policy live in [docs/ai/repo-rules ## Directory Map -| Path | Type | Primary audience | Purpose | -| --- | --- | --- | --- | -| `docs/ai/` | agent rules | agent | Repository-wide rules and fast onboarding | -| `docs/truthmark/routes/` | routing | both | Truth-routing metadata such as `areas.md` and `areas/**/*.md` | -| `docs/standards/` | standard | agent | Reusable constraints, verification rules, completion gates | -| `docs/truthmark/product/` | product truth | agent | Product capability promises, boundaries, decisions, and acceptance criteria | -| `docs/truthmark/engineering/` | engineering truth | agent | Current implementation behavior for init, check, contracts, workflows, and operations | -| `docs/architecture/` | architecture | agent | Repository-specific architecture guardrails | -| `docs/truthmark/templates/` | template | both | Editable templates for scaffolded docs; templates are not Truth Sync targets | -| `docs/user-guide.md` | guide | human | Detailed usage guide kept out of the root README so onboarding stays concise | +| Path | Type | Primary audience | Purpose | +| ----------------------------- | ----------------- | ---------------- | ------------------------------------------------------------------------------------- | +| `docs/ai/` | agent rules | agent | Repository-wide rules and fast onboarding | +| `docs/truthmark/routes/` | routing | both | Truth-routing metadata such as `areas.md` and `areas/**/*.md` | +| `docs/standards/` | standard | agent | Reusable constraints, verification rules, completion gates | +| `docs/truthmark/product/` | product truth | agent | Product capability promises, boundaries, decisions, and acceptance criteria | +| `docs/truthmark/engineering/` | engineering truth | agent | Current implementation behavior for init, check, contracts, workflows, and operations | +| `docs/architecture/` | architecture | agent | Repository-specific architecture guardrails | +| `docs/research/` | research | human | Non-canonical research, comparison, and planning notes | +| `docs/truthmark/templates/` | template | both | Editable templates for scaffolded docs; templates are not Truth Sync targets | +| `docs/user-guide.md` | guide | human | Detailed usage guide kept out of the root README so onboarding stays concise | ## Frontmatter Policy diff --git a/research/2026-05-14-evidence-backed-truth-workflows-design.md b/docs/research/2026-05-14-evidence-backed-truth-workflows-design.md similarity index 100% rename from research/2026-05-14-evidence-backed-truth-workflows-design.md rename to docs/research/2026-05-14-evidence-backed-truth-workflows-design.md diff --git a/research/2026-05-14-repoagent-ai-doc-gen-truthmark-comparison.md b/docs/research/2026-05-14-repoagent-ai-doc-gen-truthmark-comparison.md similarity index 100% rename from research/2026-05-14-repoagent-ai-doc-gen-truthmark-comparison.md rename to docs/research/2026-05-14-repoagent-ai-doc-gen-truthmark-comparison.md diff --git a/research/2026-05-14-truth-doc-template-kinds-design.md b/docs/research/2026-05-14-truth-doc-template-kinds-design.md similarity index 99% rename from research/2026-05-14-truth-doc-template-kinds-design.md rename to docs/research/2026-05-14-truth-doc-template-kinds-design.md index c656b65..90609c8 100644 --- a/research/2026-05-14-truth-doc-template-kinds-design.md +++ b/docs/research/2026-05-14-truth-doc-template-kinds-design.md @@ -2,7 +2,7 @@ Status: draft for review -This is a non-canonical design note. It is intentionally kept under `research/` so it does not redefine current Truthmark behavior until a reviewed decision is promoted into canonical docs and implementation. +This is a non-canonical design note. It is intentionally kept under `docs/research/` so it does not redefine current Truthmark behavior until a reviewed decision is promoted into canonical docs and implementation. ## Problem diff --git a/research/2026-05-15-agent-skills-workflow-review.md b/docs/research/2026-05-15-agent-skills-workflow-review.md similarity index 100% rename from research/2026-05-15-agent-skills-workflow-review.md rename to docs/research/2026-05-15-agent-skills-workflow-review.md diff --git a/research/2026-05-16-write-capable-opencode-subagents-design.md b/docs/research/2026-05-16-write-capable-opencode-subagents-design.md similarity index 100% rename from research/2026-05-16-write-capable-opencode-subagents-design.md rename to docs/research/2026-05-16-write-capable-opencode-subagents-design.md diff --git a/research/2026-05-18-skill-helper-script-portability-design.md b/docs/research/2026-05-18-skill-helper-script-portability-design.md similarity index 100% rename from research/2026-05-18-skill-helper-script-portability-design.md rename to docs/research/2026-05-18-skill-helper-script-portability-design.md diff --git a/research/2026-05-25-truthmark-portal-workflow-design.md b/docs/research/2026-05-25-truthmark-portal-workflow-design.md similarity index 99% rename from research/2026-05-25-truthmark-portal-workflow-design.md rename to docs/research/2026-05-25-truthmark-portal-workflow-design.md index 2194054..36f9469 100644 --- a/research/2026-05-25-truthmark-portal-workflow-design.md +++ b/docs/research/2026-05-25-truthmark-portal-workflow-design.md @@ -2,7 +2,7 @@ Status: draft for review -This is a non-canonical design note. It is intentionally kept under `research/` so it does not redefine current Truthmark behavior until a reviewed decision is promoted into canonical docs and implementation. +This is a non-canonical design note. It is intentionally kept under `docs/research/` so it does not redefine current Truthmark behavior until a reviewed decision is promoted into canonical docs and implementation. ## Decision Summary diff --git a/research/2026-06-01-openspec-comparison.md b/docs/research/2026-06-01-openspec-comparison.md similarity index 100% rename from research/2026-06-01-openspec-comparison.md rename to docs/research/2026-06-01-openspec-comparison.md diff --git a/research/2026-06-01-openspec-improvement-passes.md b/docs/research/2026-06-01-openspec-improvement-passes.md similarity index 99% rename from research/2026-06-01-openspec-improvement-passes.md rename to docs/research/2026-06-01-openspec-improvement-passes.md index 256b395..375ecad 100644 --- a/research/2026-06-01-openspec-improvement-passes.md +++ b/docs/research/2026-06-01-openspec-improvement-passes.md @@ -3,7 +3,7 @@ > **For Hermes:** Use `subagent-driven-development` to implement these passes one pass at a time. Each pass should finish with focused tests, `npm run check` when feasible, and a Truthmark Sync review if functional source changes were made. Date: 2026-06-01 -Source research: `research/2026-06-01-openspec-comparison.md` +Source research: `docs/research/2026-06-01-openspec-comparison.md` Scope: Truthmark implementation, generated agent surfaces, CLI contract, validation/reporting UX. Non-goal: do **not** turn Truthmark into OpenSpec, a spec-driver library, a proposal/task lifecycle, or an arbitrary workflow DAG engine. diff --git a/research/2026-06-15-claim-ledger-architecture-proposal.md b/docs/research/2026-06-15-claim-ledger-architecture-proposal.md similarity index 100% rename from research/2026-06-15-claim-ledger-architecture-proposal.md rename to docs/research/2026-06-15-claim-ledger-architecture-proposal.md diff --git a/docs/research/2026-06-29-manual-agent-skill-quality-eval-framework.md b/docs/research/2026-06-29-manual-agent-skill-quality-eval-framework.md new file mode 100644 index 0000000..a939c41 --- /dev/null +++ b/docs/research/2026-06-29-manual-agent-skill-quality-eval-framework.md @@ -0,0 +1,27 @@ +--- +status: draft +doc_type: research-index +last_reviewed: 2026-06-29 +source_of_truth: + - ../../workflow-eval-framwork/README.md + - ../../workflow-eval-framwork/catalog.yaml + - ../../scripts/workflow-eval-framwork/run-agent-scenario.mjs + - ../../tests/evals/workflow-eval-framwork.test.ts +--- + +# Manual Agent Skill And Prompt Quality Eval Framework + +The implemented manual evaluation framework lives under `workflow-eval-framwork/`. + +Use these artifacts: + +- Framework README: [`../../workflow-eval-framwork/README.md`](../../workflow-eval-framwork/README.md) +- Catalog: [`../../workflow-eval-framwork/catalog.yaml`](../../workflow-eval-framwork/catalog.yaml) +- Manual runner: [`../../scripts/workflow-eval-framwork/run-agent-scenario.mjs`](../../scripts/workflow-eval-framwork/run-agent-scenario.mjs) +- Schema and fake-runner tests: [`../../tests/evals/workflow-eval-framwork.test.ts`](../../tests/evals/workflow-eval-framwork.test.ts) + +The requested framework folder name is intentionally spelled `workflow-eval-framwork`. + +The key invariant is that Truthmark's generated-surface tests prove injection and freshness, while this framework tests actual agent behavior when workflow skills and prompts are used. + +The framework is manual and token-expensive. It must not become a default CI gate, hosted service, daemon, database, hidden memory layer, or required downstream runtime. diff --git a/research/prompt-skill-agent-workflow-literature-review-2026-05-16.md b/docs/research/prompt-skill-agent-workflow-literature-review-2026-05-16.md similarity index 99% rename from research/prompt-skill-agent-workflow-literature-review-2026-05-16.md rename to docs/research/prompt-skill-agent-workflow-literature-review-2026-05-16.md index 1108602..89de539 100644 --- a/research/prompt-skill-agent-workflow-literature-review-2026-05-16.md +++ b/docs/research/prompt-skill-agent-workflow-literature-review-2026-05-16.md @@ -101,7 +101,7 @@ Gemini generated surfaces are command files such as `.gemini/commands/truthmark/ ### Research folder -The research folder contains non-canonical design references. The 2026-05-15 review describes skills as context packages, descriptions as routing logic, progressive disclosure, evals, and security considerations. It explicitly states that behavior-bearing findings must be promoted into truth docs, the manifest, renderers, generated surfaces, and tests before becoming behavior. Source: [research/2026-05-15-agent-skills-workflow-review.md](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). +The docs/research folder contains non-canonical design references. The 2026-05-15 review describes skills as context packages, descriptions as routing logic, progressive disclosure, evals, and security considerations. It explicitly states that behavior-bearing findings must be promoted into truth docs, the manifest, renderers, generated surfaces, and tests before becoming behavior. Source: [research/2026-05-15-agent-skills-workflow-review.md](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). The 2026-05-16 write-capable OpenCode subagents design proposes parent-issued write leases, parent-owned final acceptance, and possible workers such as `truth-doc-writer`, `truth-route-structurer`, and `truth-realize-writer`. The research file is explicitly non-canonical. Source: [research/2026-05-16-write-capable-opencode-subagents-design.md](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md). @@ -805,7 +805,7 @@ The literature raises several questions that are not fully answered for this rep - Host surfaces include skills, prompts, commands, managed instruction blocks, and agents depending on host. - Current generated surfaces include read-only verifiers plus leased `truth-doc-writer` agents where hosts support agents. - Gemini generated surfaces in the inspected branch are command files rather than generated Gemini subagents. -- The research folder contains non-canonical design references and should not be treated as behavior-bearing unless promoted. +- The docs/research folder contains non-canonical design references and should not be treated as behavior-bearing unless promoted. - The write-capable OpenCode subagents research file proposes a broader write-worker design, while the current promoted worker in source/generated surfaces is the doc writer. ## Appendix C: Search Queries Used diff --git a/research/prompt-skill-agent-workflow-quality-uplift-review.md b/docs/research/prompt-skill-agent-workflow-quality-uplift-review.md similarity index 94% rename from research/prompt-skill-agent-workflow-quality-uplift-review.md rename to docs/research/prompt-skill-agent-workflow-quality-uplift-review.md index 2cc1e5a..b2513df 100644 --- a/research/prompt-skill-agent-workflow-quality-uplift-review.md +++ b/docs/research/prompt-skill-agent-workflow-quality-uplift-review.md @@ -1,6 +1,6 @@ # Truthmark Prompt, Skill, Agent, and Workflow Quality Uplift Review -Intended commit path: `research/prompt-skill-agent-workflow-quality-uplift-review.md` +Intended commit path: `docs/research/prompt-skill-agent-workflow-quality-uplift-review.md` ## Status @@ -11,7 +11,7 @@ It does not by itself change Truthmark workflows, generated surfaces, skills, ag This review is based on: - the current repository branch: `subagent-improvement`; -- the current Truthmark research folder; +- the current Truthmark docs/research folder; - the current generated host surfaces; - the literature review corpus on prompt standards, skills, agents/subagents, workflow implementation, permissions, routing, evaluation, security, and maintenance. @@ -136,11 +136,11 @@ Truthmark also has a write-capable `truth-doc-writer`. It requires a parent-issu ### Research folder -The research folder contains non-canonical design references. +The docs/research folder contains non-canonical design references. -`research/2026-05-15-agent-skills-workflow-review.md` describes skills as context packages, descriptions as routing logic, progressive disclosure, and eval needs. Repository evidence: [`research/2026-05-15-agent-skills-workflow-review.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). +`docs/research/2026-05-15-agent-skills-workflow-review.md` describes skills as context packages, descriptions as routing logic, progressive disclosure, and eval needs. Repository evidence: [`docs/research/2026-05-15-agent-skills-workflow-review.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). -`research/2026-05-16-write-capable-opencode-subagents-design.md` proposes write-capable OpenCode subagents under explicit parent-issued leases. It is non-canonical. Current source has already promoted the `truth-doc-writer` pattern beyond OpenCode to several host surfaces; broader workers such as route structurer and realize writer remain research-only ideas. Repository evidence: [`research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md). +`docs/research/2026-05-16-write-capable-opencode-subagents-design.md` proposes write-capable OpenCode subagents under explicit parent-issued leases. It is non-canonical. Current source has already promoted the `truth-doc-writer` pattern beyond OpenCode to several host surfaces; broader workers such as route structurer and realize writer remain research-only ideas. Repository evidence: [`docs/research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md). ### Current tests @@ -321,13 +321,13 @@ Severity levels: | Already strong | Sync-only automatic workflow | Truth Sync is the only automatic finish-time workflow; Structure, Document, Realize, and Check are explicit/manual. | Workflow overview, manifest, root blocks, and generated surfaces all state this. Sources: [`docs/truth/workflows/overview.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/docs/truth/workflows/overview.md), [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts), [`AGENTS.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/AGENTS.md). | Microsoft distinguishes AI-selected skills from deterministic workflows for side effects/approvals; Claude supports manual-only skill modes; GitHub distinguishes automatic and manual surfaces. Sources: [Microsoft Agent Skills](https://learn.microsoft.com/en-us/agent-framework/agents/skills), [Claude Code skills](https://code.claude.com/docs/en/slash-commands), [GitHub customization cheat sheet](https://docs.github.com/en/copilot/reference/customization-cheat-sheet). | Automatic side-effecting workflows should be narrowly scoped. Truthmark’s current automatic behavior is conservative. | | Already strong | Read-only verifier subagents | Route auditor, claim verifier, and doc reviewer are scoped as read-only verifier agents with parent-owned decisions. | `workflow-surfaces.ts` defines read-only context boundaries and JSON report expectations. Generated host agents reflect this. Source: [`src/templates/workflow-surfaces.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/templates/workflow-surfaces.ts). | Claude, Codex, OpenCode, and Gemini describe subagents as specialists with separate context and tool scopes. Sources: [Claude Code subagents](https://code.claude.com/docs/en/sub-agents), [OpenAI Codex Subagents](https://developers.openai.com/codex/subagents), [OpenCode Agents](https://opencode.ai/docs/agents/), [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/). | Verifier agents are a low-risk, high-fit subagent pattern for Truthmark. | | Already strong | Write lease concept | `truth-doc-writer` requires a parent-issued lease and parent diff validation. | `workflow-surfaces.ts`, generated doc-writer agents, and write-lease tests encode lease fields, off-lease behavior, and path validation. Sources: [`src/templates/workflow-surfaces.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/templates/workflow-surfaces.ts), [`.opencode/agents/truth-doc-writer.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/.opencode/agents/truth-doc-writer.md), [`tests/agents/write-lease.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/write-lease.test.ts). | OpenCode supports least-privilege permissions, Microsoft and MCP stress sandbox/allowlist controls, and NCSC stresses deterministic safeguards because prompt injection cannot be fully eliminated. Sources: [OpenCode Permissions](https://opencode.ai/docs/permissions/), [Microsoft Agent Skills](https://learn.microsoft.com/en-us/agent-framework/agents/skills), [MCP Security Best Practices](https://modelcontextprotocol.io/docs/tutorials/security/security_best_practices), [NCSC prompt injection guidance](https://www.ncsc.gov.uk/blog-post/prompt-injection-is-not-sql-injection). | The lease model is a practical bridge between static host permissions and runtime workflow intent. | -| Critical | Future write-worker expansion | Adding `truth-route-structurer`, `truth-realize-writer`, or other write-capable workers before complete leases, conflict policy, diff validation, report schemas, and evals would be a critical risk. | The research file proposes additional write-capable workers but labels itself non-canonical. Current source promotes only `truth-doc-writer`. Sources: [`research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts). | Gemini cautions that heavy code-edit subagents can conflict or overwrite each other; Anthropic recommends adding complexity only when measurement shows improvement; NCSC recommends deterministic safeguards and privilege reduction. Sources: [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/), [Anthropic Building Effective Agents](https://www.anthropic.com/engineering/building-effective-agents), [NCSC prompt injection guidance](https://www.ncsc.gov.uk/blog-post/prompt-injection-is-not-sql-injection). | Code-writing or route-rewriting workers can create overlapping edits, hidden policy drift, and false acceptance if not strictly controlled. | +| Critical | Future write-worker expansion | Adding `truth-route-structurer`, `truth-realize-writer`, or other write-capable workers before complete leases, conflict policy, diff validation, report schemas, and evals would be a critical risk. | The research file proposes additional write-capable workers but labels itself non-canonical. Current source promotes only `truth-doc-writer`. Sources: [`docs/research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts). | Gemini cautions that heavy code-edit subagents can conflict or overwrite each other; Anthropic recommends adding complexity only when measurement shows improvement; NCSC recommends deterministic safeguards and privilege reduction. Sources: [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/), [Anthropic Building Effective Agents](https://www.anthropic.com/engineering/building-effective-agents), [NCSC prompt injection guidance](https://www.ncsc.gov.uk/blog-post/prompt-injection-is-not-sql-injection). | Code-writing or route-rewriting workers can create overlapping edits, hidden policy drift, and false acceptance if not strictly controlled. | | High | Empirical prompt/workflow eval coverage | The repo has strong deterministic shape tests, but inspected evidence does not show a complete prompt-style eval suite for real workflow routing, negative prompts, forbidden adjacency, report compliance, and delegation behavior. | Tests enforce manifest shape and generated surfaces; they do not appear to exercise a broad model-facing prompt corpus. Sources: [`tests/agents/workflow-manifest.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/workflow-manifest.test.ts), [`tests/init/init.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/init/init.test.ts), [`tests/agents/prompts.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/prompts.test.ts). | OpenAI recommends explicit, implicit, contextual, and negative-control eval prompts; Perplexity recommends evals before skill changes and neighbor-confusion tests; Anthropic says prompt engineering should begin from success criteria and empirical tests. Sources: [OpenAI Eval Skills](https://developers.openai.com/blog/eval-skills), [Perplexity](https://research.perplexity.ai/articles/designing-refining-and-maintaining-agent-skills-at-perplexity), [Anthropic Prompt Engineering Overview](https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/overview). | Without behavioral evals, prompt and routing quality cannot be claimed as world-class. | | High | Delegation policy not fully first-class | The manifest lists subagents but does not appear to own detailed `delegateWhen`, `inlineWhen`, conflict policy, parallelism policy, or host fallback policy. | `workflow-manifest.ts` has read-only and write subagent assignment but no structured delegation-policy fields. Source: [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts). | Claude distinguishes when to use subagents versus the main conversation; Codex notes subagents consume more tokens; Gemini cautions about parallel code-edit conflicts; Anthropic recommends simple workflows and measured complexity. Sources: [Claude Code subagents](https://code.claude.com/docs/en/sub-agents), [OpenAI Codex Subagents](https://developers.openai.com/codex/subagents), [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/), [Anthropic Building Effective Agents](https://www.anthropic.com/engineering/building-effective-agents). | Subagent dispatch is a product behavior, not a documentation detail. It should be manifest-owned and testable. | | High | Cross-host permission equivalence cannot be assumed | The same logical doc writer exists across several hosts, but host permission systems differ; Gemini currently has commands only. | Generated surfaces include doc writers for Codex, OpenCode, Claude, and GitHub Copilot, while Gemini receives commands. OpenCode has explicit granular permissions. Sources: [`src/templates/generated-surfaces.ts`](https://github.com/merlinhu1/truthmark/blob/subagent-improvement/src/templates/generated-surfaces.ts), [`.gemini/commands/truthmark/sync.toml`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/.gemini/commands/truthmark/sync.toml), [`.opencode/agents/truth-doc-writer.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/.opencode/agents/truth-doc-writer.md). | GitHub docs show feature support varies across surfaces; Claude states skill `allowed-tools` grants permission but does not necessarily restrict other tools; OpenCode permissions are granular but host-specific; Gemini subagents are separate from command files. Sources: [GitHub customization cheat sheet](https://docs.github.com/en/copilot/reference/customization-cheat-sheet), [Claude Code skills](https://code.claude.com/docs/en/slash-commands), [OpenCode Permissions](https://opencode.ai/docs/permissions/), [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/). | Protocol parity and permission parity are different. Truthmark should not imply identical safety across hosts without conformance evidence. | | High | Parent diff validation is not yet proven end-to-end | The lease pattern and path validator are strong, but inspected tests do not show complete parent-orchestration simulations that compare worker reports, actual diffs, workflow status, and final acceptance. | `write-lease.test.ts` validates allowed/off-lease paths; generated prompts instruct parent diff validation. Sources: [`tests/agents/write-lease.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/write-lease.test.ts), [`src/templates/workflow-surfaces.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/templates/workflow-surfaces.ts). | OpenAI agent evals evaluate traces, handoffs, and violations; Goalkeeper’s judge gate inspects diff/log against a Definition of Done after validators pass. Sources: [OpenAI Agent Evals Guide](https://developers.openai.com/api/docs/guides/agent-evals), [Goalkeeper](https://github.com/itsuzef/goalkeeper). | The most important safety property for write-capable workers is acceptance based on actual diff, not self-report. | | High | Report contracts are not fully schema-owned across all surfaces | The manifest includes report sections and generated surfaces contain report instructions, but inspected evidence does not show a complete schema validator for every workflow report and every worker report across hosts. | Manifest has `reportSections`; write worker returns YAML; Sync has structured parser references; tests check some report sections. Sources: [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts), [`src/templates/workflow-surfaces.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/templates/workflow-surfaces.ts), [`tests/agents/prompts.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/prompts.test.ts). | OpenAI evals and Goalkeeper both treat structured outputs and checkable definitions of done as evaluation artifacts. Sources: [OpenAI Eval Skills](https://developers.openai.com/blog/eval-skills), [OpenAI Agent Evals Guide](https://developers.openai.com/api/docs/guides/agent-evals), [Goalkeeper](https://github.com/itsuzef/goalkeeper). | Report shape is part of workflow reliability and should be machine-checkable where possible. | -| High | Stale research/canonical drift risk | The write-capable subagent research doc describes a proposed OpenCode-first direction, but current source has already promoted `truth-doc-writer` across several hosts. | Research file is non-canonical and proposes broader workers; canonical source/generated surfaces include doc writer across Codex/OpenCode/Claude/GitHub. Sources: [`research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/templates/generated-surfaces.ts`](https://github.com/merlinhu1/truthmark/blob/subagent-improvement/src/templates/generated-surfaces.ts). | Anthropic cautions against time-sensitive stale skill content; Perplexity says skill descriptions and changes need maintenance/evals because small wording changes affect routing. Sources: [Anthropic Skill Best Practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices), [Perplexity](https://research.perplexity.ai/articles/designing-refining-and-maintaining-agent-skills-at-perplexity). | Agents may treat research as current behavior unless the canonical boundary is repeatedly clear. | +| High | Stale research/canonical drift risk | The write-capable subagent research doc describes a proposed OpenCode-first direction, but current source has already promoted `truth-doc-writer` across several hosts. | Research file is non-canonical and proposes broader workers; canonical source/generated surfaces include doc writer across Codex/OpenCode/Claude/GitHub. Sources: [`docs/research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/templates/generated-surfaces.ts`](https://github.com/merlinhu1/truthmark/blob/subagent-improvement/src/templates/generated-surfaces.ts). | Anthropic cautions against time-sensitive stale skill content; Perplexity says skill descriptions and changes need maintenance/evals because small wording changes affect routing. Sources: [Anthropic Skill Best Practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices), [Perplexity](https://research.perplexity.ai/articles/designing-refining-and-maintaining-agent-skills-at-perplexity). | Agents may treat research as current behavior unless the canonical boundary is repeatedly clear. | | Medium | Skill body bloat and progressive disclosure | Generated workflow skills are necessarily procedural, but the repository does not yet appear to enforce broad body-size or duplication budgets across all generated host surfaces. | Existing tests enforce compact root blocks and description length, but inspected tests do not appear to enforce skill body budget or duplicated-procedure budget across every generated skill. Sources: [`tests/init/init.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/init/init.test.ts), [`tests/agents/workflow-manifest.test.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/tests/agents/workflow-manifest.test.ts). | Anthropic recommends keeping `SKILL.md` concise and splitting details; Perplexity says every skill is a context tax; Agent Skills spec recommends progressive disclosure. Sources: [Anthropic Skill Best Practices](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices), [Perplexity](https://research.perplexity.ai/articles/designing-refining-and-maintaining-agent-skills-at-perplexity), [Agent Skills Specification](https://agentskills.io/specification). | Workflow skills can become too large to route well or too expensive to load. | | Medium | Positive/negative routing examples are manifest data but not enough by themselves | The manifest has positive, negative, and forbidden-adjacent examples, but model-facing prompt behavior needs an eval corpus. | Manifest includes these arrays and tests require them. Source: [`src/agents/workflow-manifest.ts`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/src/agents/workflow-manifest.ts). | OpenAI and Perplexity both recommend explicit prompt sets, negative controls, and neighbor-confusion evals. Sources: [OpenAI Eval Skills](https://developers.openai.com/blog/eval-skills), [Perplexity](https://research.perplexity.ai/articles/designing-refining-and-maintaining-agent-skills-at-perplexity). | Metadata can be correct while actual model routing still fails. | | Medium | Gemini capability gap | Truthmark currently generates Gemini commands, while 2026 Gemini CLI also supports subagents. This is not necessarily wrong, but it is a capability difference that should remain explicit. | Gemini generated surfaces are command TOMLs under `.gemini/commands/truthmark`. Source: [`.gemini/commands/truthmark/sync.toml`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/.gemini/commands/truthmark/sync.toml). | Gemini CLI supports project/global subagents under `.gemini/agents` with descriptions, tools, and separate context; Google cautions about heavy code-edit conflicts. Source: [Gemini CLI subagents](https://developers.googleblog.com/subagents-have-arrived-in-gemini-cli/). | Users may assume all hosts have the same generated subagent behavior. | @@ -469,15 +469,15 @@ Every recommendation below is traceable to both repository evidence and reviewed These are small, low-risk changes that improve workflow quality immediately. -#### 1. Add this research document under `research/` +#### 1. Add this research document under `docs/research/` Change: -- Add `research/prompt-skill-agent-workflow-quality-uplift-review.md`. +- Add `docs/research/prompt-skill-agent-workflow-quality-uplift-review.md`. Repository evidence: -- The research folder already contains non-canonical design references and explicitly separates research from canonical behavior. Source: [`research/2026-05-15-agent-skills-workflow-review.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). +- The docs/research folder already contains non-canonical design references and explicitly separates research from canonical behavior. Source: [`docs/research/2026-05-15-agent-skills-workflow-review.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-15-agent-skills-workflow-review.md). External evidence: @@ -491,14 +491,14 @@ Rationale: Change: -- Add a short note near the top of `research/2026-05-16-write-capable-opencode-subagents-design.md`: +- Add a short note near the top of `docs/research/2026-05-16-write-capable-opencode-subagents-design.md`: - `truth-doc-writer` has been promoted into source/generated surfaces for several hosts. - Additional workers such as `truth-route-structurer` and `truth-realize-writer` remain research-only. - Behavior-bearing rules live in source/generated surfaces/tests, not this research file. Repository evidence: -- The research file proposes write-capable OpenCode subagents, while current source/generated surfaces include `truth-doc-writer` across more hosts. Sources: [`research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/templates/generated-surfaces.ts`](https://github.com/merlinhu1/truthmark/blob/subagent-improvement/src/templates/generated-surfaces.ts). +- The research file proposes write-capable OpenCode subagents, while current source/generated surfaces include `truth-doc-writer` across more hosts. Sources: [`docs/research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md), [`src/templates/generated-surfaces.ts`](https://github.com/merlinhu1/truthmark/blob/subagent-improvement/src/templates/generated-surfaces.ts). External evidence: @@ -816,7 +816,7 @@ Research question: Repository evidence: -- The research file proposes broader write-capable workers, but source currently promotes only `truth-doc-writer`. Source: [`research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md). +- The research file proposes broader write-capable workers, but source currently promotes only `truth-doc-writer`. Source: [`docs/research/2026-05-16-write-capable-opencode-subagents-design.md`](https://raw.githubusercontent.com/merlinhu1/truthmark/subagent-improvement/research/2026-05-16-write-capable-opencode-subagents-design.md). External evidence: @@ -1098,8 +1098,8 @@ Likely files to change for the “Ship now” and “Build next” items: ### Research files -- `research/prompt-skill-agent-workflow-quality-uplift-review.md` -- `research/2026-05-16-write-capable-opencode-subagents-design.md` +- `docs/research/prompt-skill-agent-workflow-quality-uplift-review.md` +- `docs/research/2026-05-16-write-capable-opencode-subagents-design.md` ### Canonical truth docs diff --git a/research/truthmark_competitive_research_report_2026-05-15.md b/docs/research/truthmark_competitive_research_report_2026-05-15.md similarity index 100% rename from research/truthmark_competitive_research_report_2026-05-15.md rename to docs/research/truthmark_competitive_research_report_2026-05-15.md diff --git a/docs/truthmark/engineering/workflows/installed-workflow-runtime.md b/docs/truthmark/engineering/workflows/installed-workflow-runtime.md index d58eef1..0c79ea3 100644 --- a/docs/truthmark/engineering/workflows/installed-workflow-runtime.md +++ b/docs/truthmark/engineering/workflows/installed-workflow-runtime.md @@ -1,7 +1,7 @@ --- status: active truth_kind: engineering-workflow -last_reviewed: 2026-06-26 +last_reviewed: 2026-06-29 --- # Installed Workflow Runtime @@ -151,6 +151,25 @@ Optional CLI repository-intelligence helpers are compact advisory cards: - `impact` exposes branch-diff routing data. - Neither helper emits source-file or truth-doc body contents. +- The repository includes a maintainer-only workflow evaluation framework under `workflow-eval-framwork/`; it is not part of the published npm package and normal Truthmark users do not need it: + +- It tests agent behavior for installed workflow skills and prompts using cataloged scenarios, rubrics, realistic multi-file development fixtures, fake agents, deterministic grading, required manual LLM judge commands, and human review. +- Scenarios include source files, tests, route ownership, product truth, engineering behavior truth, API contract truth, and operations truth so workflow agents must choose the correct lane and bounded owner instead of solving a one-file toy case. +- A catalog run executes registered scenarios across Truth Sync, Truth Document, Truth Realize, Truth Check, and Truth Structure, then persists only `final-report.md` and `audit.json` for the full suite. +- Per-scenario command output, patches, reports, and judge scratch files are temporary in catalog mode and are discarded after aggregation. +- Scenario setup patches are applied after the fixture baseline commit so agents evaluate real pre-existing diffs. +- Deterministic expectations are prompt-visible or semantically forced by the workflow; exact command requirements and required file targets are not hidden arbitrary grader traps. +- Catalog runs without judge results are `not_evaluable`; deterministic-only output is only a harness smoke test. +- Judge failures produce failed runs; `not_evaluable` is reserved for missing or malformed judge output. +- Explicit `--agent-model` and `--judge-model` labels are preserved in `audit.json`, `final-report.md`, and wrapper environment variables for reproducible model comparisons. +- The runner exports `TRUTHMARK_EVAL_AGENT_USAGE` and `TRUTHMARK_EVAL_JUDGE_USAGE` sidecar paths so wrappers can report provider token usage without coupling the runner to one provider event format. +- Scenario results, suite `audit.json`, and `final-report.md` record measured agent tokens, judge tokens, and suite totals for prompt/workflow-surface usage comparisons. +- `--require-usage` makes missing or invalid usage sidecars non-passing for real model-comparison runs. +- The framework does not enforce token budgets or estimate dollar cost; raw token counts are the durable telemetry signal. +- Focused single-scenario runs require an explicit debug flag and are not sufficient for workflow-quality claims across the product surface. +- Manual run artifacts stay ignored by default so expensive or stale eval history is not committed accidentally. +- The framework is not part of normal `truthmark check`, `truthmark init`, generated host surfaces, package install, downstream workflow execution, or default CI agent execution. + Truth Sync workflow status separates impacted primary truth docs from candidate stale truth docs and route files. This keeps stale repository-truth correction available without making every indexed doc look like the starting scope. @@ -252,6 +271,8 @@ Update this doc when workflow manifest fields, generated surface paths, optional - ../../../../src/templates/workflow-surfaces.ts - ../../../../src/templates/generated-surfaces.ts - ../../../../src/checks/generated-surfaces.ts +- ../../../../workflow-eval-framwork/catalog.yaml +- ../../../../scripts/workflow-eval-framwork/run-agent-scenario.mjs - `src/agents/workflow-manifest.ts` - `src/templates/workflow-surfaces.ts` - `src/templates/generated-surfaces.ts` diff --git a/docs/truthmark/routes/areas.md b/docs/truthmark/routes/areas.md index fa185a8..840f3d8 100644 --- a/docs/truthmark/routes/areas.md +++ b/docs/truthmark/routes/areas.md @@ -87,6 +87,8 @@ Code surface: - src/checks/generated-surfaces.ts - src/truth/\*\* - src/version.ts +- workflow-eval-framwork/\*\* +- scripts/workflow-eval-framwork/\*\* - tests/evals/\*\* - tests/templates/\*\* diff --git a/docs/truthmark/routes/areas/installed-workflows.md b/docs/truthmark/routes/areas/installed-workflows.md index c4152f0..355981a 100644 --- a/docs/truthmark/routes/areas/installed-workflows.md +++ b/docs/truthmark/routes/areas/installed-workflows.md @@ -42,6 +42,8 @@ Code surface: - src/checks/generated-surfaces.ts - src/truth/\*\* - src/version.ts +- workflow-eval-framwork/\*\* +- scripts/workflow-eval-framwork/\*\* - tests/evals/\*\* - tests/templates/\*\* diff --git a/eslint.config.js b/eslint.config.js index 59f91de..71749ba 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -14,7 +14,7 @@ export default [ js.configs.recommended, ...tseslint.configs.recommended, { - files: ["**/*.ts"], + files: ["**/*.ts", "**/*.mjs"], rules: { "no-undef": "off", }, diff --git a/examples/github-actions/truthmark-impact.yml b/examples/github-actions/truthmark-impact.yml deleted file mode 100644 index 06d5594..0000000 --- a/examples/github-actions/truthmark-impact.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Truthmark Impact - -on: - pull_request: - -permissions: - contents: read - pull-requests: write - -jobs: - truthmark-impact: - runs-on: ubuntu-latest - env: - TRUTHMARK_BLOCKING: "false" - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - - run: npm ci - - name: Generate ImpactSet - run: | - npx truthmark impact --base origin/${{ github.base_ref }} --json > impactset.json - npx truthmark check --base origin/${{ github.base_ref }} --json > truthmark-check.json - - name: Upload Truthmark reports - uses: actions/upload-artifact@v4 - with: - name: truthmark-impact - path: | - impactset.json - truthmark-check.json - - name: Comment on PR - uses: actions/github-script@v7 - with: - script: | - const fs = require("node:fs"); - const impact = JSON.parse(fs.readFileSync("impactset.json", "utf8")); - const check = JSON.parse(fs.readFileSync("truthmark-check.json", "utf8")); - const docs = impact.data.impactSet.affectedTruthDocs; - const body = [ - "## Truthmark Impact", - "", - impact.summary, - check.summary, - "", - "Affected truth docs:", - ...(docs.length === 0 ? ["- none"] : docs.map((path) => `- ${path}`)), - ].join("\n"); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - - name: Enforce blocking mode - if: env.TRUTHMARK_BLOCKING == 'true' - run: | - node -e "const check=require('./truthmark-check.json'); const bad=check.diagnostics.filter((d)=>d.severity==='error'||d.category==='freshness'); if (bad.length) { console.error(JSON.stringify(bad, null, 2)); process.exit(1); }" diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000..392946c --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,20 @@ +schema: spec-driven + +# Project context (optional) +# This is shown to AI when creating artifacts. +# Add your tech stack, conventions, style guides, domain knowledge, etc. +# Example: +# context: | +# Tech stack: TypeScript, React, Node.js +# We use conventional commits +# Domain: e-commerce platform + +# Per-artifact rules (optional) +# Add custom rules for specific artifacts. +# Example: +# rules: +# proposal: +# - Keep proposals under 500 words +# - Always include a "Non-goals" section +# tasks: +# - Break tasks into chunks of max 2 hours diff --git a/package-lock.json b/package-lock.json index 0cc8bec..8917095 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,48 +27,39 @@ "@eslint/js": "^9.39.4", "@types/node": "^24.9.1", "eslint": "^9.39.4", + "expect": "^30.2.0", "prettier": "^3.8.3", "tsup": "^8.5.0", "tsx": "^4.20.6", - "typescript-eslint": "^8.59.2", - "vitest": "^4.0.8" + "typescript-eslint": "^8.59.2" }, "engines": { "node": ">=24" } }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", - "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" } }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" + "engines": { + "node": ">=6.9.0" } }, "node_modules/@esbuild/aix-ppc64": { @@ -747,6 +738,85 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@jest/diff-sequences": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.4.0.tgz", + "integrity": "sha512-zOpzlfUs45l6u7jm39qr87JCHUDsaeCtvL+kQe/Vn9jSnRB4/5IPXISm0h9I1vZW/o00Kn4UTJ2MOlhnUGwv3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.4.1.tgz", + "integrity": "sha512-ZBn5CglH8fBsQsvs4VWNzD4aWfUYks+IdOOQU3MEK71ol/BcVm+P+rtb1KpiFBpSWSCE27uOahyyf1vfqOVbcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/get-type": "30.1.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/get-type": { + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/pattern": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.4.0.tgz", + "integrity": "sha512-RAWn3+f9u8BsHijKJ71uHcFp6vmyEt6VvoWXkl6hKF3qVIuWNmudVjg12DlBPGup/frIl5UcUlH5HfEuvHpEXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-regex-util": "30.4.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.4.1.tgz", + "integrity": "sha512-i6b4qw5qnP8c5FEeBJg/uZQ4ddrkN6Ca8qISJh0pr7a5hfn3h3v5x60BEbOC7OYAGZNMs1LfFLwnW2CuK8F57Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/types": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.4.1.tgz", + "integrity": "sha512-f1x/vJXIfjOlEmejYpbkbgw1gOqpPECwMvMEtBqe47j7H2Hg8h8w3o3ikhSXq3MI15kg+oQ0exWO0uCtTNJLoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/pattern": "30.4.0", + "@jest/schemas": "30.4.1", + "@types/istanbul-lib-coverage": "^2.0.6", + "@types/istanbul-reports": "^3.0.4", + "@types/node": "*", + "@types/yargs": "^17.0.33", + "chalk": "^4.1.2" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -786,25 +856,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", - "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -840,280 +891,6 @@ "node": ">= 8" } }, - "node_modules/@oxc-project/types": { - "version": "0.133.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", - "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", - "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", - "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", - "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", - "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", - "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", - "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", - "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", - "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", - "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", - "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", - "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", - "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", - "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", - "@napi-rs/wasm-runtime": "^1.1.4" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", - "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", - "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "dev": true, - "license": "MIT" - }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.62.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.0.tgz", @@ -1470,6 +1247,13 @@ "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", "license": "MIT" }, + "node_modules/@sinclair/typebox": { + "version": "0.34.49", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", + "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", + "dev": true, + "license": "MIT" + }, "node_modules/@sindresorhus/merge-streams": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", @@ -1482,35 +1266,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", - "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/chai": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", - "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/deep-eql": "*", - "assertion-error": "^2.0.1" - } - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -1520,13 +1275,6 @@ "@types/ms": "*" } }, - "node_modules/@types/deep-eql": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", - "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -1534,6 +1282,33 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -1566,12 +1341,36 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/@types/yargs": { + "version": "17.0.35", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", + "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.61.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.61.0.tgz", @@ -1854,119 +1653,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/@vitest/expect": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz", - "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.9", - "@vitest/utils": "4.1.9", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", - "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.9", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz", - "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz", - "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.9", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/snapshot": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz", - "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.9", - "@vitest/utils": "4.1.9", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/spy": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz", - "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz", - "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.9", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/acorn": { "version": "8.17.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", @@ -2036,16 +1722,6 @@ "dev": true, "license": "Python-2.0" }, - "node_modules/assertion-error": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", - "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/bail": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", @@ -2122,16 +1798,6 @@ "node": ">=6" } }, - "node_modules/chai": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", - "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -2175,6 +1841,22 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/ci-info": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", + "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2228,13 +1910,6 @@ "node": "^14.18.0 || >=16.10.0" } }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2295,16 +1970,6 @@ "node": ">=6" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, "node_modules/devlop": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", @@ -2318,13 +1983,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/es-module-lexer": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", - "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", - "dev": true, - "license": "MIT" - }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -2548,16 +2206,6 @@ "node": ">=4.0" } }, - "node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -2594,14 +2242,22 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, - "node_modules/expect-type": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", - "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "node_modules/expect": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.4.1.tgz", + "integrity": "sha512-PMARsyh/JtqC20HoGqlFcIlQAyqUtW4PlI1rup1uhYJtKuwAjbvWi3GQMAn+STdHum/dk8xrKfUM1+5SAwpolA==", "dev": true, - "license": "Apache-2.0", + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "30.4.1", + "@jest/get-type": "30.1.0", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-util": "30.4.1" + }, "engines": { - "node": ">=12.0.0" + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/extend": { @@ -2830,6 +2486,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -2958,6 +2621,129 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/jest-diff": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.4.1.tgz", + "integrity": "sha512-CRpFK0RtLriVDGcPPAnR6HMVI8bSR2jnUIgralhauzYQZIb4RH9AtEInTuQr65LmmGggGcRT6HIASxwqsVsmlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/diff-sequences": "30.4.0", + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "pretty-format": "30.4.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.4.1.tgz", + "integrity": "sha512-zvYfX5CaeEkFrrLS9suWe9rvJrm9J1Iv3ua8kIBv9GEPzcnsfBf0bob37la7s67fs0nlBC3EuvkOLnXQKxtx4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "jest-diff": "30.4.1", + "pretty-format": "30.4.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.4.1.tgz", + "integrity": "sha512-kwCKIvq0MCW1HzLoGola9Te6JUdzgV0loyKJ3Qghrkz9i5/RRIHsL95BMQc2HBBhlBKC4j22K9p11TGHH8RBpQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@jest/types": "30.4.1", + "@types/stack-utils": "^2.0.3", + "chalk": "^4.1.2", + "graceful-fs": "^4.2.11", + "jest-util": "30.4.1", + "picomatch": "^4.0.3", + "pretty-format": "30.4.1", + "slash": "^3.0.0", + "stack-utils": "^2.0.6" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-message-util/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/jest-mock": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.4.1.tgz", + "integrity": "sha512-/i8SVb8/NSB7RfNi8gfqu8gxLV23KaL5EpAttyb9iz8qWRIqXRLflycz/32wXsYkOnaUlx8NAKnJYtpsmXUmfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.4.1", + "@types/node": "*", + "jest-util": "30.4.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-regex-util": { + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.4.0.tgz", + "integrity": "sha512-mWlvLviKIgIQ8VCuM1xRdD0TWp3zlzionlmDBjuXVBs+VkmXq6FgW9T4Emr7oGz/Rk6feDCGyiugolcQEyp3mg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-util": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.4.1.tgz", + "integrity": "sha512-vjQb1sACEiv13DKJMDToJpzVW0joCsIQrmbg0fi7CyOOt+g9jTuQl2A216pWRBYhOVt53XbL/2LbMKg1BECWOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "30.4.1", + "@types/node": "*", + "chalk": "^4.1.2", + "ci-info": "^4.2.0", + "graceful-fs": "^4.2.11", + "picomatch": "^4.0.3" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-util/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/joycon": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", @@ -2968,6 +2754,13 @@ "node": ">=10" } }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/js-yaml": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", @@ -3035,267 +2828,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lightningcss": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", - "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", - "dev": true, - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.32.0", - "lightningcss-darwin-arm64": "1.32.0", - "lightningcss-darwin-x64": "1.32.0", - "lightningcss-freebsd-x64": "1.32.0", - "lightningcss-linux-arm-gnueabihf": "1.32.0", - "lightningcss-linux-arm64-gnu": "1.32.0", - "lightningcss-linux-arm64-musl": "1.32.0", - "lightningcss-linux-x64-gnu": "1.32.0", - "lightningcss-linux-x64-musl": "1.32.0", - "lightningcss-win32-arm64-msvc": "1.32.0", - "lightningcss-win32-x64-msvc": "1.32.0" - } - }, - "node_modules/lightningcss-android-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", - "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", - "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", - "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", - "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", - "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", - "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", - "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", - "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", - "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", - "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", - "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/lilconfig": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", @@ -3904,25 +3436,6 @@ "thenify-all": "^1.0.0" } }, - "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3968,20 +3481,6 @@ "node": ">=0.10.0" } }, - "node_modules/obug": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", - "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", - "dev": true, - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT", - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -4124,35 +3623,6 @@ "pathe": "^2.0.1" } }, - "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.12", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, "node_modules/postcss-load-config": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", @@ -4222,6 +3692,35 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/pretty-format": { + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.4.1.tgz", + "integrity": "sha512-K6KiKMHTL4jjX4u3Kir2EW07nRfcqVTXIImx50wbjHQTcZPgg+gjVeNTIT3l3L1Rd4UefxfogquC9J37SoFyyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.4.1", + "ansi-styles": "^5.2.0", + "react-is-18": "npm:react-is@^18.3.1", + "react-is-19": "npm:react-is@^19.2.5" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/pretty-ms": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-9.3.0.tgz", @@ -4267,6 +3766,22 @@ ], "license": "MIT" }, + "node_modules/react-is-18": { + "name": "react-is", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/react-is-19": { + "name": "react-is", + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.7.tgz", + "integrity": "sha512-kZFnouyVv7eP/Phmrlo9FK+zcAdriZJvzxXHF1Sl1P377WSGe2G/JxVolhTrB/jeV47lKImhNUsijjHAAbcl/A==", + "dev": true, + "license": "MIT" + }, "node_modules/readdirp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", @@ -4326,40 +3841,6 @@ "node": ">=0.10.0" } }, - "node_modules/rolldown": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", - "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.133.0", - "@rolldown/pluginutils": "^1.0.0" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.3", - "@rolldown/binding-darwin-arm64": "1.0.3", - "@rolldown/binding-darwin-x64": "1.0.3", - "@rolldown/binding-freebsd-x64": "1.0.3", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", - "@rolldown/binding-linux-arm64-gnu": "1.0.3", - "@rolldown/binding-linux-arm64-musl": "1.0.3", - "@rolldown/binding-linux-ppc64-gnu": "1.0.3", - "@rolldown/binding-linux-s390x-gnu": "1.0.3", - "@rolldown/binding-linux-x64-gnu": "1.0.3", - "@rolldown/binding-linux-x64-musl": "1.0.3", - "@rolldown/binding-openharmony-arm64": "1.0.3", - "@rolldown/binding-wasm32-wasi": "1.0.3", - "@rolldown/binding-win32-arm64-msvc": "1.0.3", - "@rolldown/binding-win32-x64-msvc": "1.0.3" - } - }, "node_modules/rollup": { "version": "4.62.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.0.tgz", @@ -4462,13 +3943,6 @@ "node": ">=8" } }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true, - "license": "ISC" - }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -4481,6 +3955,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/source-map": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", @@ -4491,29 +3975,28 @@ "node": ">= 12" } }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", "dev": true, - "license": "BSD-3-Clause", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, "engines": { - "node": ">=0.10.0" + "node": ">=10" } }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", "dev": true, - "license": "MIT" - }, - "node_modules/std-env": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", - "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", - "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=8" + } }, "node_modules/strip-final-newline": { "version": "4.0.0", @@ -4609,13 +4092,6 @@ "node": ">=0.8" } }, - "node_modules/tinybench": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", - "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", - "dev": true, - "license": "MIT" - }, "node_modules/tinyexec": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", @@ -4671,16 +4147,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/tinyrainbow": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", - "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -4733,14 +4199,6 @@ "dev": true, "license": "Apache-2.0" }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, "node_modules/tsup": { "version": "8.5.1", "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", @@ -5011,210 +4469,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/vite": { - "version": "8.0.16", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", - "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "lightningcss": "^1.32.0", - "picomatch": "^4.0.4", - "postcss": "^8.5.15", - "rolldown": "1.0.3", - "tinyglobby": "^0.2.17" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.18", - "esbuild": "^0.27.0 || ^0.28.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "@vitejs/devtools": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vite/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/vitest": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz", - "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.9", - "@vitest/mocker": "4.1.9", - "@vitest/pretty-format": "4.1.9", - "@vitest/runner": "4.1.9", - "@vitest/snapshot": "4.1.9", - "@vitest/spy": "4.1.9", - "@vitest/utils": "4.1.9", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.9", - "@vitest/browser-preview": "4.1.9", - "@vitest/browser-webdriverio": "4.1.9", - "@vitest/coverage-istanbul": "4.1.9", - "@vitest/coverage-v8": "4.1.9", - "@vitest/ui": "4.1.9", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "node_modules/vitest/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/vitest/node_modules/tinyexec": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", - "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5230,23 +4484,6 @@ "node": ">= 8" } }, - "node_modules/why-is-node-running": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", - "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/package.json b/package.json index 08f9183..9872e6a 100644 --- a/package.json +++ b/package.json @@ -50,12 +50,12 @@ "build": "tsup", "prepack": "npm run build", "dev": "tsx src/cli/main.ts", - "format:check": "prettier --check eslint.config.js package.json package-lock.json tests/package-files.test.ts tsconfig.json vitest.config.ts", + "format:check": "prettier --check eslint.config.js package.json package-lock.json tests/package-files.test.ts tsconfig.json", "lint": "eslint .", - "package:check": "vitest run tests/package-files.test.ts", + "package:check": "node --import tsx --test --test-concurrency=1 tests/package-files.test.ts", "release:check": "npm run lint && npm run format:check && npm run typecheck && npm run test && npm run build && npm run package:check && npm audit --omit=dev", - "test": "vitest run", - "test:watch": "vitest", + "test": "node --import tsx --test --test-concurrency=1 \"tests/**/*.test.ts\"", + "test:watch": "node --watch --import tsx --test --test-concurrency=1 \"tests/**/*.test.ts\"", "typecheck": "tsc --noEmit", "check": "npm run lint && npm run typecheck && npm run test && npm run build" }, @@ -75,11 +75,11 @@ "@eslint/js": "^9.39.4", "@types/node": "^24.9.1", "eslint": "^9.39.4", + "expect": "^30.2.0", "prettier": "^3.8.3", "tsup": "^8.5.0", "tsx": "^4.20.6", - "typescript-eslint": "^8.59.2", - "vitest": "^4.0.8" + "typescript-eslint": "^8.59.2" }, "overrides": { "esbuild": "0.28.1" diff --git a/scripts/workflow-eval-framwork/judge-agent-run.mjs b/scripts/workflow-eval-framwork/judge-agent-run.mjs new file mode 100644 index 0000000..8323633 --- /dev/null +++ b/scripts/workflow-eval-framwork/judge-agent-run.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +function parseArgs(argv) { + const args = new Map(); + for (let index = 0; index < argv.length; index += 1) { + const item = argv[index]; + if (!item.startsWith("--")) { + throw new Error(`Unexpected positional argument: ${item}`); + } + const key = item.slice(2); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for --${key}`); + } + args.set(key, value); + index += 1; + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const run = args.get("run"); + if (!run) { + throw new Error("Missing required --run"); + } + const runDir = path.resolve(process.cwd(), run); + const result = JSON.parse(await readFile(path.join(runDir, "result.json"), "utf8")); + const judgeResult = { + status: "not_evaluable", + judges: [], + sourceStatus: result.status, + note: "Configure a maintainer-local LLM judge provider before using semantic judging.", + }; + const outputPath = path.join(runDir, "judge-results.json"); + await writeFile(outputPath, `${JSON.stringify(judgeResult, null, 2)}\n`, "utf8"); + console.log(`judgeResults=${outputPath}`); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/workflow-eval-framwork/run-agent-scenario.mjs b/scripts/workflow-eval-framwork/run-agent-scenario.mjs new file mode 100644 index 0000000..2864d7e --- /dev/null +++ b/scripts/workflow-eval-framwork/run-agent-scenario.mjs @@ -0,0 +1,931 @@ +#!/usr/bin/env node +import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { execa, execaCommand } from "execa"; +import micromatch from "micromatch"; +import { parse as parseYaml } from "yaml"; + +const REQUIRED_WORKFLOWS = [ + "truthmark-check", + "truthmark-document", + "truthmark-realize", + "truthmark-structure", + "truthmark-sync", +]; + +function parseArgs(argv) { + const args = new Map(); + const flags = new Set(["debug-scenario", "require-usage"]); + for (let index = 0; index < argv.length; index += 1) { + const item = argv[index]; + if (!item.startsWith("--")) { + throw new Error(`Unexpected positional argument: ${item}`); + } + const key = item.slice(2); + if (flags.has(key)) { + args.set(key, true); + continue; + } + if (key === "judge") { + throw new Error("--judge was removed because deterministic-only judging is not meaningful. Use --judge-command with an explicit LLM judge command."); + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for --${key}`); + } + args.set(key, value); + index += 1; + } + return args; +} + +function requireArg(args, name) { + const value = args.get(name); + if (typeof value !== "string" || value.length === 0) { + throw new Error(`Missing required --${name}`); + } + return value; +} + +function optionalString(args, name) { + const value = args.get(name); + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function safeSegment(value) { + return value.replace(/[^a-zA-Z0-9._-]+/gu, "-").replace(/^-|-$/gu, ""); +} + +function timestampSegment(date = new Date()) { + return date.toISOString().replace(/[:.]/gu, ""); +} + +async function readYaml(filePath) { + return parseYaml(await readFile(filePath, "utf8")); +} + +async function git(args, cwd, options = {}) { + return execa("git", args, { cwd, reject: false, ...options }); +} + +async function initializeFixtureGit(tempRepo) { + await git(["init"], tempRepo); + await git(["config", "user.email", "workflow-eval@example.invalid"], tempRepo); + await git(["config", "user.name", "Workflow Eval"], tempRepo); + await git(["add", "."], tempRepo); + await git(["commit", "-m", "baseline"], tempRepo, { + env: { GIT_AUTHOR_DATE: "2026-01-01T00:00:00Z", GIT_COMMITTER_DATE: "2026-01-01T00:00:00Z" }, + }); +} + +function parseChangedFiles(statusOutput) { + return statusOutput + .split("\n") + .map((line) => line.trimEnd()) + .filter(Boolean) + .map((line) => line.slice(3).trim()) + .map((file) => file.replace(/^"|"$/gu, "")); +} + +function asStringArray(value) { + return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; +} + +function matchAny(file, patterns) { + return micromatch.isMatch(file, patterns, { dot: true }); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + +function commandWasRunOrExplained(evidence, command) { + if (evidence.includes(command)) { + return true; + } + const escaped = escapeRegExp(command); + const explanationPattern = new RegExp( + `(?:skip|skipped|cannot|can't|unable|did not run|not run|not available|would fail|no package|missing dependency)[\\s\\S]{0,200}${escaped}|${escaped}[\\s\\S]{0,200}(?:skip|skipped|cannot|can't|unable|did not run|not run|not available|would fail|no package|missing dependency)`, + "iu", + ); + return explanationPattern.test(evidence); +} + +async function collectTraceEvidence(runDir, commandOutput) { + const chunks = [commandOutput]; + let bytes = commandOutput.length; + const maxBytes = 1_000_000; + + async function visit(directory) { + if (bytes >= maxBytes) { + return; + } + let entries; + try { + entries = await readdir(directory, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (bytes >= maxBytes) { + return; + } + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) { + await visit(absolute); + continue; + } + if (!entry.isFile()) { + continue; + } + try { + const text = await readFile(absolute, "utf8"); + const relative = path.relative(runDir, absolute); + const chunk = `\n--- ${relative} ---\n${text}`; + bytes += chunk.length; + chunks.push(chunk); + } catch { + // Binary or unreadable artifacts are not trace evidence. + } + } + } + + await visit(runDir); + return chunks.join("\n"); +} + +function gradeDeterministic({ changedFiles, agentChangedFiles, expected, commandOutput, traceEvidence, finalReportExists }) { + const failures = []; + const mustRead = asStringArray(expected.must_read); + const mustNotChange = asStringArray(expected.must_not_change); + const mustChange = asStringArray(expected.must_change); + const mustRunOrExplain = asStringArray(expected.must_run_or_explain); + const report = expected.report && typeof expected.report === "object" ? expected.report : {}; + const evidence = `${commandOutput}\n${traceEvidence}`; + + for (const file of mustRead) { + if (!evidence.includes(file)) { + failures.push({ id: "required-read-not-recorded", message: `${file} was not recorded in trace evidence` }); + } + } + + for (const file of agentChangedFiles) { + if (mustNotChange.length > 0 && matchAny(file, mustNotChange)) { + failures.push({ id: "forbidden-write", message: `${file} matched forbidden write patterns` }); + } + } + + for (const pattern of mustChange) { + if (!agentChangedFiles.some((file) => matchAny(file, [pattern]))) { + failures.push({ id: "missing-required-change", message: `${pattern} was not changed` }); + } + } + + for (const command of mustRunOrExplain) { + if (!commandWasRunOrExplained(evidence, command)) { + failures.push({ id: "verification-not-recorded", message: `${command} was not run or specifically explained` }); + } + } + + if (report.required === true && !finalReportExists) { + failures.push({ id: "missing-report", message: "expected report was not produced" }); + } + + return { + status: failures.length === 0 ? "passed" : "failed", + failures, + changedFiles, + agentChangedFiles, + requiredReads: mustRead, + }; +} + +function normalizeJudgeResult(candidate, fallback) { + if (!candidate || typeof candidate !== "object") { + return fallback; + } + const status = typeof candidate.status === "string" ? candidate.status : "not_evaluable"; + if (!["passed", "failed", "not_evaluable", "skipped"].includes(status)) { + return { + status: "not_evaluable", + judges: [], + note: `Judge returned unsupported status ${JSON.stringify(status)}.` , + }; + } + return { + ...candidate, + status, + judges: Array.isArray(candidate.judges) ? candidate.judges : [], + }; +} + +const usageTokenFields = ["inputTokens", "cachedInputTokens", "outputTokens", "reasoningOutputTokens", "totalTokens"]; + +function emptyTokenTotals() { + return { + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + totalTokens: 0, + }; +} + +function addTokenTotals(left, right) { + return { + inputTokens: left.inputTokens + right.inputTokens, + cachedInputTokens: left.cachedInputTokens + right.cachedInputTokens, + outputTokens: left.outputTokens + right.outputTokens, + reasoningOutputTokens: left.reasoningOutputTokens + right.reasoningOutputTokens, + totalTokens: left.totalTokens + right.totalTokens, + }; +} + +function hasUsageCount(record) { + return usageTokenFields.some((field) => Object.hasOwn(record, field)); +} + +function normalizeUsageRecord(record, phase, index) { + if (!record || typeof record !== "object" || Array.isArray(record)) { + throw new Error(`${phase} usage record ${index + 1} is not an object`); + } + if (!hasUsageCount(record)) { + throw new Error(`${phase} usage record ${index + 1} does not include token counts`); + } + + const normalized = { + schemaVersion: record.schemaVersion ?? 1, + source: typeof record.source === "string" ? record.source : "usage-sidecar", + model: typeof record.model === "string" ? record.model : undefined, + ...emptyTokenTotals(), + }; + + for (const field of usageTokenFields) { + if (!Object.hasOwn(record, field)) { + continue; + } + const value = record[field]; + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${phase} usage record ${index + 1} has invalid ${field}`); + } + normalized[field] = value; + } + + if (!Object.hasOwn(record, "totalTokens")) { + normalized.totalTokens = normalized.inputTokens + normalized.outputTokens + normalized.reasoningOutputTokens; + } + + return normalized; +} + +async function readUsageSidecar(filePath, phase) { + let text; + try { + text = await readFile(filePath, "utf8"); + } catch (error) { + if (error && error.code === "ENOENT") { + return { + status: "unavailable", + phase, + path: filePath, + warnings: [`${phase} command did not write usage sidecar ${filePath}`], + records: [], + ...emptyTokenTotals(), + }; + } + return { + status: "invalid", + phase, + path: filePath, + warnings: [`Could not read ${phase} usage sidecar ${filePath}: ${error instanceof Error ? error.message : String(error)}`], + records: [], + ...emptyTokenTotals(), + }; + } + + let parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + return { + status: "invalid", + phase, + path: filePath, + warnings: [`${phase} usage sidecar ${filePath} did not contain valid JSON: ${error instanceof Error ? error.message : String(error)}`], + records: [], + ...emptyTokenTotals(), + }; + } + + const candidates = Array.isArray(parsed) ? parsed : [parsed]; + if (candidates.length === 0) { + return { + status: "invalid", + phase, + path: filePath, + warnings: [`${phase} usage sidecar ${filePath} contained no usage records`], + records: [], + ...emptyTokenTotals(), + }; + } + + try { + const records = candidates.map((record, index) => normalizeUsageRecord(record, phase, index)); + const totals = records.reduce((sum, record) => addTokenTotals(sum, record), emptyTokenTotals()); + return { + status: "measured", + phase, + path: filePath, + warnings: [], + records, + ...totals, + }; + } catch (error) { + return { + status: "invalid", + phase, + path: filePath, + warnings: [error instanceof Error ? error.message : String(error)], + records: [], + ...emptyTokenTotals(), + }; + } +} + +function usageStatusForPhases(agent, judge) { + if (agent.status === "invalid" || judge.status === "invalid") { + return "invalid"; + } + if (agent.status === "unavailable" || judge.status === "unavailable") { + return "unavailable"; + } + return "measured"; +} + +function compactUsagePhase(phaseUsage) { + return { + status: phaseUsage.status, + inputTokens: phaseUsage.inputTokens, + cachedInputTokens: phaseUsage.cachedInputTokens, + outputTokens: phaseUsage.outputTokens, + reasoningOutputTokens: phaseUsage.reasoningOutputTokens, + totalTokens: phaseUsage.totalTokens, + records: phaseUsage.records, + warnings: phaseUsage.warnings, + }; +} + +function evaluateRequiredUsage(usage, { requireUsage }) { + const failures = []; + + if (requireUsage) { + for (const phase of ["agent", "judge"]) { + if (usage[phase].status !== "measured") { + failures.push({ id: `${phase}-usage-required`, message: `${phase} usage is required but status was ${usage[phase].status}` }); + } + } + } + + return { + status: failures.length > 0 ? "failed" : "passed", + failures, + }; +} + +function buildUsage({ agentUsage, judgeUsage, requireUsage }) { + const agent = compactUsagePhase(agentUsage); + const judge = compactUsagePhase(judgeUsage); + const total = addTokenTotals(agentUsage, judgeUsage); + const usage = { + status: usageStatusForPhases(agentUsage, judgeUsage), + agent, + judge, + total, + }; + usage.requirement = evaluateRequiredUsage(usage, { requireUsage }); + return usage; +} + +function aggregateSuiteUsage(results) { + const agent = results.reduce((sum, result) => addTokenTotals(sum, result.usage.agent), emptyTokenTotals()); + const judge = results.reduce((sum, result) => addTokenTotals(sum, result.usage.judge), emptyTokenTotals()); + const total = addTokenTotals(agent, judge); + const statuses = results.map((result) => result.usage.status); + return { + status: statuses.every((status) => status === "measured") ? "measured" : statuses.some((status) => status === "invalid") ? "invalid" : "unavailable", + agent, + judge, + total, + requirement: { + status: results.some((result) => result.usage.requirement.status === "failed") ? "failed" : "passed", + failures: results.flatMap((result) => result.usage.requirement.failures.map((failure) => ({ ...failure, workflow: result.workflow, scenario: result.scenario }))), + }, + }; +} + +async function runJudge({ judgeCommand, judgeModel, judgeUsagePath, runDir, repoRoot, scenario, scenarioPath, expectedPath, deterministic }) { + const outputPath = path.join(runDir, "judge-results.json"); + + if (deterministic.status !== "passed") { + const skipped = { + status: "skipped", + judges: [], + note: "Deterministic grading failed; LLM judge spending is blocked until objective failures are fixed.", + }; + await writeFile(outputPath, `${JSON.stringify(skipped, null, 2)}\n`, "utf8"); + return skipped; + } + + if (!judgeCommand) { + const missing = { + status: "not_evaluable", + judges: [], + note: "No --judge-command was supplied. Deterministic-only runs are harness smoke tests, not meaningful workflow-quality evals.", + }; + await writeFile(outputPath, `${JSON.stringify(missing, null, 2)}\n`, "utf8"); + return missing; + } + + const judge = await execaCommand(judgeCommand, { + cwd: repoRoot, + reject: false, + timeout: 300_000, + env: { + TRUTHMARK_EVAL_EXPECTED: expectedPath, + TRUTHMARK_EVAL_RUN_DIR: runDir, + TRUTHMARK_EVAL_SCENARIO: scenarioPath, + TRUTHMARK_EVAL_WORKFLOW: scenario.workflow, + TRUTHMARK_EVAL_JUDGE_MODEL: judgeModel ?? "", + TRUTHMARK_EVAL_JUDGE_USAGE: judgeUsagePath, + }, + }); + await writeFile(path.join(runDir, "judge-output.txt"), `${judge.stdout}\n`, "utf8"); + await writeFile(path.join(runDir, "judge-error.txt"), `${judge.stderr}\n`, "utf8"); + + let parsed; + try { + parsed = JSON.parse(judge.stdout); + } catch { + parsed = { + status: "not_evaluable", + judges: [], + note: "Judge command did not emit strict JSON on stdout.", + }; + } + + const normalized = normalizeJudgeResult(parsed, { + status: "not_evaluable", + judges: [], + note: "Judge command output was not an object.", + }); + const finalJudge = judge.exitCode === 0 + ? normalized + : { + ...normalized, + status: normalized.status === "passed" ? "not_evaluable" : normalized.status, + note: normalized.note ?? `Judge command exited with ${judge.exitCode}.`, + }; + await writeFile(outputPath, `${JSON.stringify(finalJudge, null, 2)}\n`, "utf8"); + return finalJudge; +} + +function resultStatus({ agentExitCode, deterministic, judgeResult }) { + if (agentExitCode !== 0 || deterministic.status === "failed" || judgeResult.status === "failed") { + return "failed"; + } + return judgeResult.status === "passed" ? "passed" : "not_evaluable"; +} + +function renderHumanReport(result) { + const lines = [ + `# Workflow Eval Report`, + "", + `- Status: ${result.status}`, + `- Workflow: ${result.workflow}`, + `- Scenario: ${result.scenario}`, + `- Host: ${result.host}`, + `- Agent model: ${result.agentModel ?? ""}`, + `- Judge model: ${result.judgeModel ?? ""}`, + `- Run directory: ${result.runDir}`, + "", + "## Deterministic grading", + "", + `- Status: ${result.deterministic.status}`, + "", + "## LLM judging", + "", + `- Status: ${result.judge.status}`, + "", + "## Token Usage Summary", + "", + `- Usage status: ${result.usage.status}`, + `- Agent tokens: ${result.usage.agent.totalTokens} total (${result.usage.agent.inputTokens} input, ${result.usage.agent.cachedInputTokens} cached input, ${result.usage.agent.outputTokens} output, ${result.usage.agent.reasoningOutputTokens} reasoning output)`, + `- Judge tokens: ${result.usage.judge.totalTokens} total (${result.usage.judge.inputTokens} input, ${result.usage.judge.cachedInputTokens} cached input, ${result.usage.judge.outputTokens} output, ${result.usage.judge.reasoningOutputTokens} reasoning output)`, + `- Suite tokens: ${result.usage.total.totalTokens} total`, + `- Required usage status: ${result.usage.requirement.status}`, + ]; + + if (result.deterministic.failures.length > 0) { + lines.push("", "### Deterministic failures", ""); + for (const failure of result.deterministic.failures) { + lines.push(`- ${failure.id}: ${failure.message}`); + } + } + + lines.push("", "## Artifacts", "", "- `result.json`", "- `deterministic-results.json`", "- `judge-results.json`", "- `command-output.txt`", "- `command-error.txt`", "- `changed-files.txt`", "- `after.patch`", "- `final-report.md`", "- `human-review.yaml`"); + return `${lines.join("\n")}\n`; +} + +function scenarioRunId({ scenario, host }) { + return `${timestampSegment()}-${safeSegment(scenario.workflow)}-${safeSegment(scenario.id)}-${safeSegment(host)}`; +} + +async function runScenario({ repoRoot, scenarioArg, host, agentCommand, judgeCommand, agentModel, judgeModel, outputBase, timeout, requireUsage }) { + const scenarioPath = path.resolve(repoRoot, scenarioArg); + const scenarioDir = path.dirname(scenarioPath); + const scenario = await readYaml(scenarioPath); + const expectedPath = path.join(scenarioDir, scenario.expected); + const expected = await readYaml(expectedPath); + const promptPath = path.join(scenarioDir, scenario.prompt); + const fixtureDir = path.join(scenarioDir, scenario.fixture); + const runDir = path.join(outputBase, scenarioRunId({ scenario, host })); + const finalReportPath = path.join(runDir, "final-report.md"); + const agentUsagePath = path.join(runDir, "agent-usage.json"); + const judgeUsagePath = path.join(runDir, "judge-usage.json"); + const tempRepo = await mkdtemp(path.join(os.tmpdir(), "truthmark-workflow-eval-fixture-")); + + await mkdir(runDir, { recursive: true }); + console.log(`runDir=${runDir}`); + + try { + await cp(fixtureDir, tempRepo, { recursive: true }); + await initializeFixtureGit(tempRepo); + + const setup = scenario.setup && typeof scenario.setup === "object" ? scenario.setup : {}; + const setupPatch = typeof setup.patch === "string" ? setup.patch : undefined; + if (setupPatch) { + const setupPatchPath = path.join(scenarioDir, setupPatch); + await writeFile(path.join(runDir, "setup.patch"), await readFile(setupPatchPath, "utf8"), "utf8"); + const applyResult = await git(["apply", setupPatchPath], tempRepo); + if (applyResult.exitCode !== 0) { + throw new Error(`Failed to apply scenario setup patch ${setupPatch}: ${applyResult.stderr}`); + } + } + + const beforeAgentStatus = await git(["status", "--porcelain", "-uall"], tempRepo); + const beforeAgentChangedFiles = parseChangedFiles(beforeAgentStatus.stdout); + await writeFile(path.join(runDir, "before-agent-changed-files.txt"), `${beforeAgentChangedFiles.join("\n")}\n`, "utf8"); + const beforeAgentDiff = await git(["diff", "--binary"], tempRepo); + await writeFile(path.join(runDir, "before-agent.patch"), `${beforeAgentDiff.stdout}\n`, "utf8"); + + await writeFile( + path.join(runDir, "trace-summary.txt"), + [ + `scenario=${scenarioArg}`, + `workflow=${scenario.workflow}`, + `prompt=${path.relative(repoRoot, promptPath)}`, + `host=${host}`, + `agentCommand=${agentCommand}`, + `judgeCommand=${judgeCommand ?? ""}`, + `agentModel=${agentModel ?? ""}`, + `judgeModel=${judgeModel ?? ""}`, + `requireUsage=${requireUsage ? "true" : "false"}`, + `agentUsage=${agentUsagePath}`, + `judgeUsage=${judgeUsagePath}`, + `setupPatch=${setupPatch ?? ""}`, + "trace=host command stdout/stderr plus changed-file, patch, report, and judge artifacts", + ].join("\n") + "\n", + "utf8", + ); + + const agentResult = await execaCommand(agentCommand, { + cwd: tempRepo, + reject: false, + timeout: Number(timeout ?? 900_000), + env: { + TRUTHMARK_EVAL_EXPECTED: expectedPath, + TRUTHMARK_EVAL_PROMPT: promptPath, + TRUTHMARK_EVAL_REPORT: finalReportPath, + TRUTHMARK_EVAL_RUN_DIR: runDir, + TRUTHMARK_EVAL_SCENARIO: scenarioPath, + TRUTHMARK_EVAL_WORKFLOW: scenario.workflow, + TRUTHMARK_EVAL_AGENT_MODEL: agentModel ?? "", + TRUTHMARK_EVAL_JUDGE_MODEL: judgeModel ?? "", + TRUTHMARK_EVAL_AGENT_USAGE: agentUsagePath, + }, + }); + + const combinedOutput = `${agentResult.stdout}\n${agentResult.stderr}`; + await writeFile(path.join(runDir, "command-output.txt"), `${agentResult.stdout}\n`, "utf8"); + await writeFile(path.join(runDir, "command-error.txt"), `${agentResult.stderr}\n`, "utf8"); + + const status = await git(["status", "--porcelain", "-uall"], tempRepo); + const changedFiles = parseChangedFiles(status.stdout); + const beforeAgentChangedSet = new Set(beforeAgentChangedFiles); + const agentChangedFiles = changedFiles.filter((file) => !beforeAgentChangedSet.has(file)); + await writeFile(path.join(runDir, "changed-files.txt"), `${changedFiles.join("\n")}\n`, "utf8"); + await writeFile(path.join(runDir, "agent-changed-files.txt"), `${agentChangedFiles.join("\n")}\n`, "utf8"); + + const diff = await git(["diff", "--binary"], tempRepo); + await writeFile(path.join(runDir, "before.patch"), `${beforeAgentDiff.stdout}\n`, "utf8"); + await writeFile(path.join(runDir, "after.patch"), `${diff.stdout}\n`, "utf8"); + + let finalReportExists = true; + try { + await readFile(finalReportPath, "utf8"); + } catch { + finalReportExists = false; + await writeFile(finalReportPath, "# Final Report Missing\n\nThe agent did not produce a final workflow report artifact.\n", "utf8"); + } + + const traceEvidence = await collectTraceEvidence(runDir, combinedOutput); + await writeFile(path.join(runDir, "trace-evidence.txt"), traceEvidence, "utf8"); + + const baseDeterministic = gradeDeterministic({ + changedFiles, + agentChangedFiles, + expected, + commandOutput: combinedOutput, + traceEvidence, + finalReportExists, + }); + + const judgeResult = await runJudge({ + judgeCommand, + judgeUsagePath, + runDir, + repoRoot, + scenario, + scenarioPath, + expectedPath, + deterministic: baseDeterministic, + judgeModel, + }); + + const agentUsage = await readUsageSidecar(agentUsagePath, "agent"); + const judgeUsage = await readUsageSidecar(judgeUsagePath, "judge"); + const usage = buildUsage({ + agentUsage, + judgeUsage, + requireUsage, + }); + const deterministic = { + ...baseDeterministic, + failures: [...baseDeterministic.failures, ...usage.requirement.failures], + }; + deterministic.status = deterministic.failures.length === 0 ? "passed" : "failed"; + await writeFile(path.join(runDir, "deterministic-results.json"), `${JSON.stringify(deterministic, null, 2)}\n`, "utf8"); + + const statusValue = resultStatus({ + agentExitCode: agentResult.exitCode, + deterministic, + judgeResult, + }); + const result = { + status: statusValue, + runDir, + workflow: scenario.workflow, + scenario: scenario.id, + host, + agentModel, + judgeModel, + agentExitCode: agentResult.exitCode, + deterministic, + judge: judgeResult, + usage, + }; + await writeFile(path.join(runDir, "result.json"), `${JSON.stringify(result, null, 2)}\n`, "utf8"); + await writeFile(path.join(runDir, "report.md"), renderHumanReport(result), "utf8"); + await writeFile( + path.join(runDir, "human-review.yaml"), + "status: not-evaluable\nreviewer: \"\"\nreviewed_at: \"\"\nnotes: []\nfollow_up: []\n", + "utf8", + ); + + return result; + } finally { + await rm(tempRepo, { force: true, recursive: true }); + } +} + +function makeAuditScenario(result) { + return { + workflow: result.workflow, + scenario: result.scenario, + status: result.status, + host: result.host, + agentModel: result.agentModel, + judgeModel: result.judgeModel, + agentExitCode: result.agentExitCode, + deterministic: { + status: result.deterministic.status, + failures: result.deterministic.failures, + changedFiles: result.deterministic.changedFiles, + agentChangedFiles: result.deterministic.agentChangedFiles, + requiredReads: result.deterministic.requiredReads, + }, + judge: result.judge, + usage: result.usage, + }; +} + +function renderCatalogReport(audit) { + const lines = [ + "# Workflow Evaluation Report", + "", + `- Status: ${audit.status}`, + `- Host: ${audit.host}`, + `- Agent model: \`${audit.agentModel ?? ""}\``, + `- Judge model: \`${audit.judgeModel ?? ""}\``, + `- Total scenarios: ${audit.total}`, + `- Passed: ${audit.passed}`, + `- Failed: ${audit.failed}`, + `- Not evaluable: ${audit.notEvaluable}`, + `- Audit data: \`${audit.audit}\``, + "", + "## Token Usage Summary", + "", + `- Usage status: ${audit.usage.status}`, + `- Agent tokens: ${audit.usage.agent.totalTokens} total (${audit.usage.agent.inputTokens} input, ${audit.usage.agent.cachedInputTokens} cached input, ${audit.usage.agent.outputTokens} output, ${audit.usage.agent.reasoningOutputTokens} reasoning output)`, + `- Judge tokens: ${audit.usage.judge.totalTokens} total (${audit.usage.judge.inputTokens} input, ${audit.usage.judge.cachedInputTokens} cached input, ${audit.usage.judge.outputTokens} output, ${audit.usage.judge.reasoningOutputTokens} reasoning output)`, + `- Suite tokens: ${audit.usage.total.totalTokens} total (${audit.usage.total.inputTokens} input, ${audit.usage.total.cachedInputTokens} cached input, ${audit.usage.total.outputTokens} output, ${audit.usage.total.reasoningOutputTokens} reasoning output)`, + `- Required usage status: ${audit.usage.requirement.status}`, + "", + "## Scenario Results", + "", + "| Workflow | Scenario | Status | Deterministic | Judge | Notes |", + "|---|---|---:|---:|---:|---|", + ]; + + for (const scenario of audit.scenarios) { + const notes = []; + for (const failure of scenario.deterministic.failures ?? []) { + notes.push(`${failure.id}: ${failure.message}`); + } + if (scenario.judge?.note) { + notes.push(scenario.judge.note); + } + for (const judge of scenario.judge?.judges ?? []) { + if (judge.summary) { + notes.push(judge.summary); + } + } + if (scenario.usage?.requirement?.status && scenario.usage.requirement.status !== "passed") { + notes.push(`required usage ${scenario.usage.requirement.status}`); + } + for (const warning of [...(scenario.usage?.agent?.warnings ?? []), ...(scenario.usage?.judge?.warnings ?? [])]) { + notes.push(warning); + } + const note = notes.join("; ").replace(/\n/gu, " ").slice(0, 900); + lines.push(`| \`${scenario.workflow}\` | \`${scenario.scenario}\` | ${scenario.status} | ${scenario.deterministic.status} | ${scenario.judge.status} | ${note} |`); + } + + lines.push( + "", + "## Persistent Artifacts", + "", + "- `final-report.md`: this human-facing report.", + "- `audit.json`: compact machine-readable audit data for the run.", + "", + "Detailed per-scenario command traces, patches, and intermediate files were temporary and discarded after aggregation.", + ); + return `${lines.join("\n")}\n`; +} + +function validateCatalogCoverage(catalog) { + const workflows = Array.isArray(catalog.workflows) ? catalog.workflows : []; + const ids = workflows.map((workflow) => workflow.id).sort(); + const missing = REQUIRED_WORKFLOWS.filter((workflow) => !ids.includes(workflow)); + if (missing.length > 0) { + throw new Error(`Catalog is missing required workflow coverage: ${missing.join(", ")}`); + } + for (const workflow of workflows) { + if (!Array.isArray(workflow.scenarios) || workflow.scenarios.length === 0) { + throw new Error(`Workflow ${workflow.id} must have at least one meaningful scenario.`); + } + } +} + +async function runCatalog({ repoRoot, args }) { + const catalogArg = requireArg(args, "catalog"); + const host = requireArg(args, "host"); + const agentCommand = requireArg(args, "agent-command"); + const judgeCommand = optionalString(args, "judge-command"); + const agentModel = optionalString(args, "agent-model"); + const judgeModel = optionalString(args, "judge-model"); + const requireUsage = args.get("require-usage") === true; + const catalogPath = path.resolve(repoRoot, catalogArg); + const catalog = await readYaml(catalogPath); + validateCatalogCoverage(catalog); + const scenarios = catalog.workflows.flatMap((workflow) => workflow.scenarios ?? []); + + const suiteDir = path.resolve( + repoRoot, + args.get("output") ?? path.join("workflow-eval-framwork", "runs", `${timestampSegment()}-all-workflows-${safeSegment(host)}`), + ); + const reportPath = path.join(suiteDir, "final-report.md"); + const auditPath = path.join(suiteDir, "audit.json"); + const tempSuiteDir = await mkdtemp(path.join(os.tmpdir(), "truthmark-workflow-eval-suite-")); + await mkdir(suiteDir, { recursive: true }); + console.log(`suiteDir=${suiteDir}`); + + try { + const results = []; + for (const scenario of scenarios) { + results.push( + await runScenario({ + repoRoot, + scenarioArg: scenario, + host, + agentCommand, + judgeCommand, + agentModel, + judgeModel, + outputBase: tempSuiteDir, + timeout: args.get("timeout"), + requireUsage, + }), + ); + } + + const passed = results.filter((result) => result.status === "passed").length; + const failed = results.filter((result) => result.status === "failed").length; + const notEvaluable = results.filter((result) => result.status === "not_evaluable").length; + const summaryStatus = failed > 0 ? "failed" : notEvaluable > 0 ? "not_evaluable" : "passed"; + const suiteUsage = aggregateSuiteUsage(results); + const audit = { + status: summaryStatus, + suiteDir, + report: reportPath, + audit: auditPath, + host, + judgeRequired: true, + judgeCommandSupplied: judgeCommand !== undefined, + agentModel, + judgeModel, + total: results.length, + passed, + failed, + notEvaluable, + usage: suiteUsage, + scenarios: results.map(makeAuditScenario), + persistentArtifacts: ["final-report.md", "audit.json"], + temporaryArtifactsDiscarded: true, + }; + await writeFile(auditPath, `${JSON.stringify(audit, null, 2)} +`, "utf8"); + await writeFile(reportPath, renderCatalogReport(audit), "utf8"); + console.log(`report=${reportPath}`); + console.log(`audit=${auditPath}`); + return audit.status; + } finally { + await rm(tempSuiteDir, { force: true, recursive: true }); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.has("pricing-file")) { + throw new Error("--pricing-file was removed; workflow eval reports token usage only, not dollar estimates."); + } + const repoRoot = process.cwd(); + + if (args.has("catalog")) { + const status = await runCatalog({ repoRoot, args }); + process.exitCode = status === "passed" ? 0 : 1; + return; + } + + if (args.has("scenario") && args.get("debug-scenario") !== true) { + throw new Error("Use --catalog for evaluation runs. Single --scenario runs are debug-only and must pass --debug-scenario."); + } + + const host = requireArg(args, "host"); + const agentCommand = requireArg(args, "agent-command"); + const judgeCommand = optionalString(args, "judge-command"); + const agentModel = optionalString(args, "agent-model"); + const judgeModel = optionalString(args, "judge-model"); + const requireUsage = args.get("require-usage") === true; + const scenarioArg = requireArg(args, "scenario"); + const outputBase = path.resolve(repoRoot, args.get("output") ?? path.join("workflow-eval-framwork", "runs", "debug")); + const result = await runScenario({ + repoRoot, + scenarioArg, + host, + agentCommand, + judgeCommand, + agentModel, + judgeModel, + outputBase, + timeout: args.get("timeout"), + requireUsage, + }); + process.exitCode = result.status === "passed" ? 0 : 1; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/tests/agents/instructions.test.ts b/tests/agents/instructions.test.ts index 26c13dd..78fc600 100644 --- a/tests/agents/instructions.test.ts +++ b/tests/agents/instructions.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createDefaultConfig } from "../../src/config/defaults.js"; import { @@ -13,32 +14,44 @@ describe("renderTruthSyncInstructions", () => { expect(instructions).toContain("### Truth Sync"); expect(instructions).toContain("Automatic finish-time trigger"); - expect(instructions).toContain("use the truthmark-sync skill before finishing"); + expect(instructions).toContain( + "use the truthmark-sync skill before finishing", + ); expect(instructions).toContain( "OpenCode /skill truthmark-sync; Codex /truthmark-sync or $truthmark-sync; Claude Code /truthmark-sync; GitHub Copilot /truthmark-sync; Antigravity @truthmark-sync; Cursor /truthmark-sync", ); - expect(instructions).toContain("staged, unstaged, and untracked functional code files"); + expect(instructions).toContain( + "staged, unstaged, and untracked functional code files", + ); expect(instructions).toContain("Run relevant tests before finishing"); expect(instructions).toContain( "Support new or changed behavior-bearing truth claims with checkout evidence", ); expect(instructions).toContain("documentation-only change"); expect(instructions).toContain("Explicit invocation runs immediately"); - expect(instructions).toContain("Later functional-code changes need a fresh finish-time review"); + expect(instructions).toContain( + "Later functional-code changes need a fresh finish-time review", + ); expect(instructions).toContain("must not rewrite functional code"); expect(instructions).toContain("host supports subagent dispatch"); - expect(instructions).toContain("WorkflowState and ImpactSet are optional compact derived context"); + expect(instructions).toContain( + "WorkflowState and ImpactSet are optional compact derived context", + ); expect(instructions).toContain( "If routing is missing, stale, broad, overloaded, catch-all, or cannot map changed code to a bounded truth owner", ); expect(instructions).toContain("run Truth Structure before syncing"); - expect(instructions).toContain("otherwise stop and recommend Truth Structure"); + expect(instructions).toContain( + "otherwise stop and recommend Truth Structure", + ); expect(instructions).not.toContain(".truthmark/local.yml"); expect(instructions).not.toContain("truth_sync.sync_agent"); expect(instructions).not.toContain("Truth Sync: completed"); expect(instructions).not.toContain("Truth Sync: skipped"); expect(instructions).not.toContain("truthmark packet --changed"); - expect(instructions).not.toContain("truthmark check --json --workflow truth-sync"); + expect(instructions).not.toContain( + "truthmark check --json --workflow truth-sync", + ); }); it("keeps the managed Truth Sync reminder small", () => { @@ -58,18 +71,28 @@ describe("renderTruthSyncInstructions", () => { expect(instructions).toContain("docs/routes/index.md; docs/routes/areas/"); expect(instructions).toContain("only when present"); - expect(instructions).not.toContain("docs/truthmark/routes/areas.md; docs/truthmark/routes/areas/"); + expect(instructions).not.toContain( + "docs/truthmark/routes/areas.md; docs/truthmark/routes/areas/", + ); }); }); describe("agent-native workflow instructions", () => { it("renders Truth Structure and Truth Check summaries", () => { expect(renderTruthStructureInstructions()).toContain("truthmark-structure"); - expect(renderTruthStructureInstructions()).toContain("configured route files when present"); - expect(renderTruthStructureInstructions()).toContain("missing, stale, broad, overloaded, catch-all, unrouteable"); - expect(renderTruthStructureInstructions()).toContain("canonical current-truth destinations"); + expect(renderTruthStructureInstructions()).toContain( + "configured route files when present", + ); + expect(renderTruthStructureInstructions()).toContain( + "missing, stale, broad, overloaded, catch-all, unrouteable", + ); + expect(renderTruthStructureInstructions()).toContain( + "canonical current-truth destinations", + ); expect(renderTruthStructureInstructions()).toContain("topology pressure"); - expect(renderTruthStructureInstructions()).toContain("If the skill is unavailable"); + expect(renderTruthStructureInstructions()).toContain( + "If the skill is unavailable", + ); expect(renderTruthCheckInstructions()).toContain("truthmark-check"); expect(renderTruthCheckInstructions()).toContain( "run the truthmark check command only when available for additional validation", diff --git a/tests/agents/prompts.test.ts b/tests/agents/prompts.test.ts index 012e816..67c2622 100644 --- a/tests/agents/prompts.test.ts +++ b/tests/agents/prompts.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createDefaultConfig } from "../../src/config/defaults.js"; import { renderTruthRealizePrompt } from "../../src/agents/prompts.js"; @@ -20,7 +21,9 @@ describe("renderTruthRealizePrompt", () => { expect(prompt).toContain( "implement only bounded, current truth claims from the source docs", ); - expect(prompt).toContain("RepoIndex, RouteMap, ImpactSet, and WorkflowState/action context"); + expect(prompt).toContain( + "RepoIndex, RouteMap, ImpactSet, and WorkflowState/action context", + ); expect(prompt).toContain("workflow write boundaries"); expect(prompt).toContain( "read product truth under docs/truthmark/product as requirements", @@ -37,7 +40,9 @@ describe("renderTruthRealizePrompt", () => { expect(prompt).not.toContain("Truth-doc restructure gate"); expect(prompt).not.toContain("Truth-doc shape repair review"); expect(prompt).not.toContain("restructure truth docs"); - expect(prompt).toContain("Report changed code files and verification steps"); + expect(prompt).toContain( + "Report changed code files and verification steps", + ); expect(prompt).toContain("installed instruction or skill"); expect(prompt).not.toContain("Invocations:"); expect(prompt).not.toContain("OpenCode /skill truthmark-realize"); @@ -53,11 +58,15 @@ describe("renderTruthRealizePrompt", () => { const prompt = renderTruthRealizePrompt(config); - expect(prompt).toContain("docs/product/product/capabilities/authentication-session.md"); + expect(prompt).toContain( + "docs/product/product/capabilities/authentication-session.md", + ); expect(prompt).toContain( "read engineering truth under docs/product/engineering as implementation context", ); - expect(prompt).not.toContain("docs/truthmark/product/capabilities/authentication-session.md"); + expect(prompt).not.toContain( + "docs/truthmark/product/capabilities/authentication-session.md", + ); }); it("renders adjacent-workflow exclusions in generated skill metadata", () => { diff --git a/tests/agents/truth-check.test.ts b/tests/agents/truth-check.test.ts index df116e3..6187548 100644 --- a/tests/agents/truth-check.test.ts +++ b/tests/agents/truth-check.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { TRUTH_CHECK_EXPLICIT_INVOCATIONS, @@ -32,7 +33,9 @@ describe("renderTruthCheckSkillBody", () => { it("renders the agent-led truth audit workflow", () => { const skill = renderTruthCheckSkillBody(); - expect(TRUTH_CHECK_EXPLICIT_INVOCATIONS).toContain("Cursor /truthmark-check"); + expect(TRUTH_CHECK_EXPLICIT_INVOCATIONS).toContain( + "Cursor /truthmark-check", + ); expect(skill).toContain("name: truthmark-check"); expect(skill).toContain( "description: Use when the user asks to audit repository truth health, routing, ownership, or canonical docs.", @@ -98,7 +101,9 @@ describe("Truth Check generated surfaces", () => { expect(renderTruthmarkCheckSkill()).toContain( "Parent agent owns the final Truth Check report", ); - expect(renderTruthmarkCheckLocalSkill()).not.toContain("Codex subagent mode:"); + expect(renderTruthmarkCheckLocalSkill()).not.toContain( + "Codex subagent mode:", + ); expect(renderTruthmarkCheckClaudeSkill()).toContain( "Claude Code subagent mode:", ); @@ -111,8 +116,12 @@ describe("Truth Check generated surfaces", () => { expect(renderTruthmarkCheckClaudeSkill()).toContain( "truth-doc-reviewer subagent", ); - expect(renderTruthmarkCheckLocalSkill()).not.toContain("OpenCode /skill truthmark-check"); - expect(renderTruthmarkCheckLocalSkill()).not.toContain("Cursor @truthmark-check"); + expect(renderTruthmarkCheckLocalSkill()).not.toContain( + "OpenCode /skill truthmark-check", + ); + expect(renderTruthmarkCheckLocalSkill()).not.toContain( + "Cursor @truthmark-check", + ); expect(renderTruthmarkCheckSkillMetadata()).toContain( 'display_name: "Truthmark Check"', ); @@ -162,7 +171,9 @@ describe("Truth Check generated surfaces", () => { const docReviewer = renderTruthmarkClaudeDocReviewerAgent(); expect(routeAuditor).toContain("name: truth-route-auditor"); expect(routeAuditor).toContain("tools: Read, Grep, Glob, LS"); - expect(routeAuditor).toContain("Manual invocation: use the truth-route-auditor subagent"); + expect(routeAuditor).toContain( + "Manual invocation: use the truth-route-auditor subagent", + ); expect(routeAuditor).toContain(readOnlyContextBoundary); expect(routeAuditor).toContain("Do not edit files"); expect(routeAuditor).toContain("Return JSON only"); @@ -231,7 +242,8 @@ describe("Truth Check generated surfaces", () => { customConfig.truthmark.paths.engineeringTruthRoot = "product/engineering"; customConfig.truthmark.paths.routesIndex = "product/routes/index.md"; customConfig.truthmark.paths.routeAreasRoot = "product/routes/areas"; - const customOpenCodeWriter = renderTruthmarkOpenCodeDocWriterAgent(customConfig); + const customOpenCodeWriter = + renderTruthmarkOpenCodeDocWriterAgent(customConfig); expect(codexWriter).toContain('name = "truth_doc_writer"'); expect(codexWriter).toContain('sandbox_mode = "workspace-write"'); @@ -243,15 +255,17 @@ describe("Truth Check generated surfaces", () => { expect(openCodeWriter).toContain('"docs/truthmark/routes/areas.md": allow'); expect(openCodeWriter).toContain("@truth-doc-writer"); expect(customOpenCodeWriter).toContain('"product/engineering/**": allow'); - expect(customOpenCodeWriter).toContain( - '"product/routes/index.md": allow', - ); + expect(customOpenCodeWriter).toContain('"product/routes/index.md": allow'); expect(customOpenCodeWriter).toContain( '"product/routes/areas/**/*.md": allow', ); - expect(customOpenCodeWriter).not.toContain('"docs/truthmark/engineering/**": allow'); + expect(customOpenCodeWriter).not.toContain( + '"docs/truthmark/engineering/**": allow', + ); expect(claudeWriter).toContain("name: truth-doc-writer"); - expect(claudeWriter).toContain("tools: Read, Grep, Glob, LS, Edit, MultiEdit"); + expect(claudeWriter).toContain( + "tools: Read, Grep, Glob, LS, Edit, MultiEdit", + ); expect(copilotWriter).toContain("name: truth-doc-writer"); expect(copilotWriter).toContain("tools: [read, search, edit]"); expect(copilotWriter).toContain("offLeaseChanges"); diff --git a/tests/agents/truth-document.test.ts b/tests/agents/truth-document.test.ts index bc34bf3..90c4f82 100644 --- a/tests/agents/truth-document.test.ts +++ b/tests/agents/truth-document.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseFrontmatter } from "../../src/markdown/frontmatter.js"; import { createDefaultConfig } from "../../src/config/defaults.js"; @@ -37,7 +38,9 @@ describe("renderTruthDocumentSkillBody", () => { it("renders the manual existing-implementation documentation workflow", () => { const skill = renderTruthDocumentSkillBody(); - expect(TRUTH_DOCUMENT_EXPLICIT_INVOCATIONS).toContain("Cursor /truthmark-document"); + expect(TRUTH_DOCUMENT_EXPLICIT_INVOCATIONS).toContain( + "Cursor /truthmark-document", + ); expect(skill).toContain("name: truthmark-document"); expect(skill).toContain("manual and implementation-first"); expect(skill).toContain("existing implemented behavior"); @@ -47,15 +50,23 @@ describe("renderTruthDocumentSkillBody", () => { expect(skill).toContain("When creating or updating a truth doc"); expect(skill).toContain("Truth-doc prose style:"); expect(skill).toContain("Use professional, plain technical prose"); - expect(skill).toContain("Prefer specific current-state claims over promotional, symbolic, or generic significance language"); + expect(skill).toContain( + "Prefer specific current-state claims over promotional, symbolic, or generic significance language", + ); expect(skill).toContain("Avoid common AI-writing tells"); expect(skill).toContain( "one durable claim per bullet or line; paragraphs should be no longer than one or two short sentences", ); - expect(skill).toContain("Do not add personality, rhetorical flourish, first-person commentary, or marketing tone"); - expect(skill).toContain("without removing scope, evidence, decisions, or source references"); + expect(skill).toContain( + "Do not add personality, rhetorical flourish, first-person commentary, or marketing tone", + ); + expect(skill).toContain( + "without removing scope, evidence, decisions, or source references", + ); expect(skill).not.toContain("PERSONALITY AND SOUL"); - expect(skill).not.toContain("What makes the below so obviously AI generated?"); + expect(skill).not.toContain( + "What makes the below so obviously AI generated?", + ); expect(skill).toContain("HTML comments under each template section"); expect(skill).toContain("normative authoring guidance"); expect(skill).toContain("Truth-doc ownership review"); @@ -65,9 +76,7 @@ describe("renderTruthDocumentSkillBody", () => { expect(skill).toContain( "report Ownership reviewed, Structure required, Truth docs split, Truth docs restructured, or Manual handoff reason", ); - expect(skill).toContain( - "Decision/Rationale preservation review", - ); + expect(skill).toContain("Decision/Rationale preservation review"); expect(skill).toContain( "before any truth-doc split, restructure, or shape repair, inventory existing Product Decisions, Engineering Decisions, and Rationale sections", ); @@ -101,15 +110,17 @@ describe("renderTruthDocumentSkillBody", () => { expect(skill).toContain( "Repository instruction files and explicitly configured policy docs remain instruction authority when present; do not assume a repository uses any particular policy path.", ); - expect(skill).toContain("RepoIndex, RouteMap, ImpactSet, and WorkflowState/action context"); - expect(skill).toContain("repository-intelligence artifacts were not generated"); + expect(skill).toContain( + "RepoIndex, RouteMap, ImpactSet, and WorkflowState/action context", + ); + expect(skill).toContain( + "repository-intelligence artifacts were not generated", + ); expect(skill).toContain("Evidence checklist"); expect(skill).toContain( "route-first: map the documented behavior to bounded route owners and primary canonical docs", ); - expect(skill).toContain( - "support claims with primary checkout evidence", - ); + expect(skill).toContain("support claims with primary checkout evidence"); expect(skill).toContain( "remove, narrow, or record unsupported claims for manual handoff", ); @@ -141,14 +152,18 @@ describe("renderTruthDocumentSkillBody", () => { const skill = renderTruthDocumentSkillBody(config); expect(skill).toContain("docs/truthmark/engineering/contracts/routing.md"); - expect(skill).toContain("docs/truthmark/engineering/behaviors/check-diagnostics.md"); + expect(skill).toContain( + "docs/truthmark/engineering/behaviors/check-diagnostics.md", + ); expect(skill).toContain("docs/routes/index.md"); }); }); describe("Truth Document generated surfaces", () => { it("renders Codex metadata and OpenCode skill content", () => { - expect(renderTruthmarkDocumentSkill()).toContain("name: truthmark-document"); + expect(renderTruthmarkDocumentSkill()).toContain( + "name: truthmark-document", + ); expect(renderTruthmarkDocumentSkill()).toContain("Codex subagent mode:"); expect(renderTruthmarkDocumentSkill()).toContain( "use automatically when this workflow runs in Codex", @@ -186,8 +201,12 @@ describe("Truth Document generated surfaces", () => { expect(renderTruthmarkDocumentLocalSkill()).not.toContain( "Claude Code subagent mode:", ); - expect(renderTruthmarkDocumentLocalSkill()).not.toContain("OpenCode /skill truthmark-document"); - expect(renderTruthmarkDocumentLocalSkill()).not.toContain("Cursor /truthmark-document"); + expect(renderTruthmarkDocumentLocalSkill()).not.toContain( + "OpenCode /skill truthmark-document", + ); + expect(renderTruthmarkDocumentLocalSkill()).not.toContain( + "Cursor /truthmark-document", + ); expect(renderTruthmarkDocumentSkillMetadata()).toContain( 'display_name: "Truthmark Document"', ); @@ -203,13 +222,16 @@ describe("Truth Document generated surfaces", () => { host: "cursor", }); const cursorDocumentSkill = - cursorDocumentPackage.find((file) => file.path.endsWith("/SKILL.md"))?.content ?? ""; + cursorDocumentPackage.find((file) => file.path.endsWith("/SKILL.md")) + ?.content ?? ""; const cursorDocumentProcedure = - cursorDocumentPackage.find((file) => file.path.endsWith("/support/procedure.md")) - ?.content ?? ""; + cursorDocumentPackage.find((file) => + file.path.endsWith("/support/procedure.md"), + )?.content ?? ""; const cursorDocumentReportTemplate = - cursorDocumentPackage.find((file) => file.path.endsWith("/support/report-template.md")) - ?.content ?? ""; + cursorDocumentPackage.find((file) => + file.path.endsWith("/support/report-template.md"), + )?.content ?? ""; expect(cursorDocumentSkill).toContain("Use as a Cursor Agent Skill."); expect(cursorDocumentSkill).toContain(".cursor/skills/"); @@ -220,7 +242,9 @@ describe("Truth Document generated surfaces", () => { "This prompt is the GitHub Copilot entrypoint for Truthmark Document.", ); for (const surface of [renderTruthmarkCopilotDocumentPrompt()]) { - expect(surface).toContain("Do not invoke another Truthmark command from here."); + expect(surface).toContain( + "Do not invoke another Truthmark command from here.", + ); expect(surface).toContain( "If skill entrypoints are unavailable, use the host's direct evidence-first manual fallback procedure.", ); @@ -234,7 +258,11 @@ describe("Truth Document generated surfaces", () => { "support/report-template.md", ); expect(cursorDocumentProcedure).toContain("Truthmark Document Procedure"); - expect(cursorDocumentProcedure).not.toContain("Report completion in this shape:"); - expect(cursorDocumentReportTemplate).toContain("Report completion in this shape:"); + expect(cursorDocumentProcedure).not.toContain( + "Report completion in this shape:", + ); + expect(cursorDocumentReportTemplate).toContain( + "Report completion in this shape:", + ); }); }); diff --git a/tests/agents/truth-sync.test.ts b/tests/agents/truth-sync.test.ts index 481fb53..2d0988f 100644 --- a/tests/agents/truth-sync.test.ts +++ b/tests/agents/truth-sync.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseFrontmatter } from "../../src/markdown/frontmatter.js"; import { createDefaultConfig } from "../../src/config/defaults.js"; @@ -126,7 +127,9 @@ describe("renderTruthSyncSkillBody", () => { expect(skillBody).toContain("Affected route/truth owner"); expect(skillBody).toContain("User-provided decisions/rationale"); expect(skillBody).toContain("Decision/rationale captured"); - expect(skillBody).toContain("Preserve concise user-provided decision rationale"); + expect(skillBody).toContain( + "Preserve concise user-provided decision rationale", + ); expect(skillBody).toContain("No-update-needed rationale"); expect(skillBody).toContain( "Only edit allowed truth docs/routes after Sync Intent is clear", @@ -138,9 +141,13 @@ describe("renderTruthSyncSkillBody", () => { expect(skillBody).toContain( "one durable claim per bullet or line; paragraphs should be no longer than one or two short sentences", ); - expect(skillBody).toContain("Do not add personality, rhetorical flourish, first-person commentary, or marketing tone"); + expect(skillBody).toContain( + "Do not add personality, rhetorical flourish, first-person commentary, or marketing tone", + ); expect(skillBody).not.toContain("PERSONALITY AND SOUL"); - expect(skillBody).not.toContain("What makes the below so obviously AI generated?"); + expect(skillBody).not.toContain( + "What makes the below so obviously AI generated?", + ); expect(skillBody).toContain("Claim:"); expect(skillBody).toContain("Result: supported"); expect(skillBody).toContain("structured Truth Sync report contract"); @@ -235,18 +242,12 @@ describe("Truth Sync generated metadata", () => { file.path.endsWith("/support/report-template.md"), )?.content; - expect(entrypoint, workflowId).toContain("Progressive disclosure:"); - expect(entrypoint, workflowId).toContain("support/procedure.md"); - expect(entrypoint, workflowId).toContain("support/report-template.md"); - expect(procedure, workflowId).toContain( - "Truthmark-managed generated file", - ); - expect(procedure, workflowId).not.toContain( - "Report completion in this shape:", - ); - expect(reportTemplate, workflowId).toContain( - "Report completion in this shape:", - ); + expect(entrypoint).toContain("Progressive disclosure:"); + expect(entrypoint).toContain("support/procedure.md"); + expect(entrypoint).toContain("support/report-template.md"); + expect(procedure).toContain("Truthmark-managed generated file"); + expect(procedure).not.toContain("Report completion in this shape:"); + expect(reportTemplate).toContain("Report completion in this shape:"); } }); @@ -294,13 +295,15 @@ describe("Truth Sync generated metadata", () => { host: "cursor", }); const cursorSyncSkill = - cursorSyncPackage.find((file) => file.path.endsWith("/SKILL.md"))?.content ?? ""; - const cursorSyncProcedure = - cursorSyncPackage.find((file) => file.path.endsWith("/support/procedure.md")) + cursorSyncPackage.find((file) => file.path.endsWith("/SKILL.md")) ?.content ?? ""; + const cursorSyncProcedure = + cursorSyncPackage.find((file) => + file.path.endsWith("/support/procedure.md"), + )?.content ?? ""; expect(cursorSyncSkill).toContain( - 'description: Use automatically at finish-time after functional code changes', + "description: Use automatically at finish-time after functional code changes", ); expect(cursorSyncSkill).toContain("Use as a Cursor Agent Skill."); expect(cursorSyncSkill).toContain("support/procedure.md"); @@ -323,12 +326,16 @@ describe("Truth Sync generated metadata", () => { expect(surface).not.toContain("helper package unavailable"); } - expect(renderTruthmarkCopilotSyncPrompt()).toContain("support/procedure.md"); + expect(renderTruthmarkCopilotSyncPrompt()).toContain( + "support/procedure.md", + ); expect(renderTruthmarkCopilotSyncPrompt()).toContain( "support/report-template.md", ); expect(cursorSyncProcedure).toContain("Truthmark Sync Procedure"); - expect(cursorSyncProcedure).not.toContain("Report completion in this shape:"); + expect(cursorSyncProcedure).not.toContain( + "Report completion in this shape:", + ); }); it("adds host-specific subagent guidance without changing generic surfaces", () => { @@ -374,11 +381,10 @@ describe("Truth Sync generated metadata", () => { skillPath: ".cursor/skills/truthmark-sync/SKILL.md", workflowId: "truthmark-sync", host: "cursor", - }).find((file) => file.path.endsWith("/support/procedure.md"))?.content ?? ""; + }).find((file) => file.path.endsWith("/support/procedure.md"))?.content ?? + ""; - expect(cursorSyncProcedure).not.toContain( - "Codex subagent mode:", - ); + expect(cursorSyncProcedure).not.toContain("Codex subagent mode:"); expect(renderTruthmarkCopilotSyncPrompt()).not.toContain( "Codex subagent mode:", ); diff --git a/tests/agents/workflow-helper-scripts.test.ts b/tests/agents/workflow-helper-scripts.test.ts index c1cb03a..a2127a2 100644 --- a/tests/agents/workflow-helper-scripts.test.ts +++ b/tests/agents/workflow-helper-scripts.test.ts @@ -1,12 +1,29 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { renderTruthmarkSkillPackage } from "../../src/templates/workflow-surfaces.js"; import { runCli } from "../helpers/run-cli.js"; import { createTempRepo } from "../helpers/temp-repo.js"; +type TestCase = readonly unknown[]; + +const formatCaseName = (name: string, values: TestCase): string => + values.reduce( + (result, value) => result.replace(/%[sdifoOpj]/u, String(value)), + name, + ); + +const itEach = + (cases: readonly T[]) => + (name: string, fn: (...values: T) => void | Promise): void => { + for (const values of cases) { + it(formatCaseName(name, values), () => fn(...values)); + } + }; + type HelperResult = { exitCode: number; stdout: string; @@ -33,7 +50,9 @@ const parseValidationEnvelope = (stdout: string): HelperResult["json"] => { const validation = parsed.data?.validation; if (validation === undefined) { - throw new Error(`missing data.validation in helper JSON\nstdout:\n${stdout}`); + throw new Error( + `missing data.validation in helper JSON\nstdout:\n${stdout}`, + ); } return validation; @@ -41,7 +60,9 @@ const parseValidationEnvelope = (stdout: string): HelperResult["json"] => { const tempRepos: Array>> = []; -const snapshotFiles = async (rootDir: string): Promise> => { +const snapshotFiles = async ( + rootDir: string, +): Promise> => { const entries = await fs.readdir(rootDir, { recursive: true, withFileTypes: true, @@ -49,7 +70,9 @@ const snapshotFiles = async (rootDir: string): Promise> = const filePaths = entries .filter((entry) => entry.isFile()) - .map((entry) => path.relative(rootDir, path.join(entry.parentPath, entry.name))) + .map((entry) => + path.relative(rootDir, path.join(entry.parentPath, entry.name)), + ) .sort(); return Object.fromEntries( @@ -94,13 +117,17 @@ const runCliHelper = async ({ }; }; -const getGeneratedReportExample = (workflowId: "truthmark-document" | "truthmark-sync"): string => { +const getGeneratedReportExample = ( + workflowId: "truthmark-document" | "truthmark-sync", +): string => { const files = renderTruthmarkSkillPackage({ skillPath: `.agents/skills/${workflowId}/SKILL.md`, workflowId, host: "codex", }); - const reportTemplate = files.find((file) => file.path.endsWith("/support/report-template.md")); + const reportTemplate = files.find((file) => + file.path.endsWith("/support/report-template.md"), + ); const match = reportTemplate?.content.match(/```md\n([\s\S]*?)\n```/u); if (match === null || match === undefined) { @@ -110,7 +137,9 @@ const getGeneratedReportExample = (workflowId: "truthmark-document" | "truthmark return match[1]; }; -const materializeSkillPackage = async (workflowId: "truthmark-document" | "truthmark-sync") => { +const materializeSkillPackage = async ( + workflowId: "truthmark-document" | "truthmark-sync", +) => { const repo = await createTempRepo(); tempRepos.push(repo); @@ -125,7 +154,9 @@ const materializeSkillPackage = async (workflowId: "truthmark-document" | "truth return repo; }; -const syncReportWithEvidence = (evidenceChecked: string): string => `Truth Sync: completed +const syncReportWithEvidence = ( + evidenceChecked: string, +): string => `Truth Sync: completed Changed code reviewed: - src/init/init.ts @@ -150,7 +181,9 @@ Notes: - Complete. `; -const documentReportWithEvidence = (evidenceChecked: string): string => `Truth Document: completed +const documentReportWithEvidence = ( + evidenceChecked: string, +): string => `Truth Document: completed Implementation reviewed: - src/templates/workflow-surfaces.ts @@ -196,7 +229,13 @@ const runWriteLease = async ({ "lease.yml": lease, "changed-files.txt": changedFiles, }, - args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"], + args: [ + "validate", + "write-lease", + "lease.yml", + "changed-files.txt", + "--json", + ], }); afterEach(async () => { @@ -227,7 +266,9 @@ describe("workflow helper scripts", () => { const helperPolicy = files.find((file) => file.path.endsWith("/support/helper-policy.md"), ); - const entrypoint = files.find((file) => file.path.endsWith("/SKILL.md")); + const entrypoint = files.find((file) => + file.path.endsWith("/SKILL.md"), + ); expect(manifest).toBeUndefined(); expect(helperPolicy).toBeUndefined(); @@ -239,7 +280,10 @@ describe("workflow helper scripts", () => { it("keeps report validators available without generated helper manifests", async () => { const repo = await materializeSkillPackage("truthmark-sync"); - const skillDirectory = path.join(repo.rootDir, ".agents/skills/truthmark-sync"); + const skillDirectory = path.join( + repo.rootDir, + ".agents/skills/truthmark-sync", + ); await expect( fs.access(path.join(skillDirectory, "helper-manifest.yml")), @@ -254,9 +298,12 @@ describe("workflow helper scripts", () => { "utf8", ); - const result = await runCli(["validate", "sync-report", "report.md", "--json"], { - cwd: skillDirectory, - }); + const result = await runCli( + ["validate", "sync-report", "report.md", "--json"], + { + cwd: skillDirectory, + }, + ); expect(result.exitCode).toBe(0); expect(JSON.parse(result.stdout)).toMatchObject({ @@ -315,14 +362,18 @@ Notes: }); it("accepts Truth Sync helper status bullets with flexible whitespace", async () => { - const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files. + const report = + syncReportWithEvidence(`- Claim: Init writes generated workflow files. Evidence: src/init/init.ts Result: supported`) - .replace("- validate-sync-report: ran, passed", "- validate-sync-report: ran, passed") - .replace( - "- validate-write-lease: skipped, no write lease used", - "-\tvalidate-write-lease: skipped, no write lease used", - ); + .replace( + "- validate-sync-report: ran, passed", + "- validate-sync-report: ran, passed", + ) + .replace( + "- validate-write-lease: skipped, no write lease used", + "-\tvalidate-write-lease: skipped, no write lease used", + ); const result = await runSyncReport(report); expect(result.exitCode).toBe(0); @@ -374,16 +425,19 @@ Notes: expect(result.json.ok).toBe(true); }); - it.each([ + itEach([ ["blocked", "Reason"], ["skipped", "Reason"], - ])("rejects a Truth Sync %s report with no required body", async (status, expectedError) => { - const result = await runSyncReport(`Truth Sync: ${status}\n`); + ])( + "rejects a Truth Sync %s report with no required body", + async (status, expectedError) => { + const result = await runSyncReport(`Truth Sync: ${status}\n`); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain(expectedError); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain(expectedError); + }, + ); it("rejects a blocked Truth Sync report missing manual-review files", async () => { const result = await runSyncReport(`Truth Sync: blocked @@ -397,18 +451,24 @@ Next action: expect(result.exitCode).toBe(1); expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("Files requiring manual review"); + expect(result.json.errors?.join("\n")).toContain( + "Files requiring manual review", + ); }); - it.each([ + itEach([ ["Changed code reviewed"], ["Ownership reviewed"], ["Truth docs updated"], ["Notes"], ])("rejects a completed Truth Sync report with empty %s", async (label) => { - const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files. + const report = + syncReportWithEvidence(`- Claim: Init writes generated workflow files. Evidence: src/init/init.ts - Result: supported`).replace(new RegExp(`${label}:\\n- [^\\n]+`, "u"), `${label}:`); + Result: supported`).replace( + new RegExp(`${label}:\\n- [^\\n]+`, "u"), + `${label}:`, + ); const result = await runSyncReport(report); expect(result.exitCode).toBe(1); @@ -479,7 +539,7 @@ Notes: expect(result.json.errors?.join("\n")).toContain("Evidence checked"); }); - it.each([ + itEach([ ["empty Evidence checked", ""], [ "missing Result", @@ -498,31 +558,37 @@ Notes: Evidence: src/init/init.ts Result: supported`, ], - ])("rejects a completed Truth Sync report with %s", async (_name, evidenceChecked) => { - const result = await runSyncReport(syncReportWithEvidence(evidenceChecked)); + ])( + "rejects a completed Truth Sync report with %s", + async (_name, evidenceChecked) => { + const result = await runSyncReport( + syncReportWithEvidence(evidenceChecked), + ); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("Evidence checked"); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain("Evidence checked"); + }, + ); - it.each([ - ["validate-sync-report"], - ["validate-write-lease"], - ])("rejects a completed Truth Sync report when %s ran and failed", async (helperId) => { - const report = syncReportWithEvidence(`- Claim: Init writes generated workflow files. + itEach([["validate-sync-report"], ["validate-write-lease"]])( + "rejects a completed Truth Sync report when %s ran and failed", + async (helperId) => { + const report = + syncReportWithEvidence(`- Claim: Init writes generated workflow files. Evidence: src/init/init.ts Result: supported`).replace( - `- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`, - `- ${helperId}: ran, failed`, - ); + `- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`, + `- ${helperId}: ran, failed`, + ); - const result = await runSyncReport(report); + const result = await runSyncReport(report); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("ran, failed"); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain("ran, failed"); + }, + ); it("accepts a valid completed Truth Document report", async () => { const result = await runCliHelper({ @@ -608,21 +674,27 @@ Notes: expect(result.json.errors?.join("\n")).toContain("Reason"); }); - it.each([ + itEach([ ["Implementation reviewed"], ["Ownership reviewed"], ["Truth docs created"], ["Notes"], - ])("rejects a completed Truth Document report with empty %s", async (label) => { - const report = documentReportWithEvidence(`- Claim: Helpers are optional. + ])( + "rejects a completed Truth Document report with empty %s", + async (label) => { + const report = documentReportWithEvidence(`- Claim: Helpers are optional. Evidence: src/agents/workflow-manifest.ts - Result: supported`).replace(new RegExp(`${label}:\\n- [^\\n]+`, "u"), `${label}:`); - const result = await runDocumentReport(report); + Result: supported`).replace( + new RegExp(`${label}:\\n- [^\\n]+`, "u"), + `${label}:`, + ); + const result = await runDocumentReport(report); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain(label); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain(label); + }, + ); it("rejects a completed Truth Document report with malformed Evidence checked entries", async () => { const result = await runCliHelper({ @@ -653,7 +725,7 @@ Notes: expect(result.json.errors?.join("\n")).toContain("Evidence checked"); }); - it.each([ + itEach([ ["empty Evidence checked", ""], [ "labels outside Evidence checked", @@ -681,33 +753,38 @@ Notes: Evidence: src/agents/workflow-manifest.ts Result: supported`, ], - ])("rejects a completed Truth Document report with %s", async (_name, evidenceChecked) => { - const result = await runDocumentReport(documentReportWithEvidence(evidenceChecked)); + ])( + "rejects a completed Truth Document report with %s", + async (_name, evidenceChecked) => { + const result = await runDocumentReport( + documentReportWithEvidence(evidenceChecked), + ); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("Evidence checked"); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain("Evidence checked"); + }, + ); - it.each([ - ["validate-document-report"], - ["validate-write-lease"], - ])("rejects a completed Truth Document report when %s ran and failed", async (helperId) => { - const report = documentReportWithEvidence(`- Claim: Helpers are optional. + itEach([["validate-document-report"], ["validate-write-lease"]])( + "rejects a completed Truth Document report when %s ran and failed", + async (helperId) => { + const report = documentReportWithEvidence(`- Claim: Helpers are optional. Evidence: src/agents/workflow-manifest.ts Result: supported`).replace( - `- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`, - `- ${helperId}: ran, failed`, - ); + `- ${helperId}: ${helperId === "validate-write-lease" ? "skipped, no write lease used" : "ran, passed"}`, + `- ${helperId}: ran, failed`, + ); - const result = await runDocumentReport(report); + const result = await runDocumentReport(report); - expect(result.exitCode).toBe(1); - expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("ran, failed"); - }); + expect(result.exitCode).toBe(1); + expect(result.json.ok).toBe(false); + expect(result.json.errors?.join("\n")).toContain("ran, failed"); + }, + ); - it.each([ + itEach([ [ "block list", `allowedWrites: @@ -716,7 +793,10 @@ forbiddenWrites: - src/** `, ], - ["flow list", "allowedWrites: [docs/truthmark/truth/**]\nforbiddenWrites: []\n"], + [ + "flow list", + "allowedWrites: [docs/truthmark/truth/**]\nforbiddenWrites: []\n", + ], [ "quoted paths and comments", `# parent-issued lease @@ -749,7 +829,7 @@ filesChanged: expect(result.json.ok).toBe(true); }); - it.each([ + itEach([ [ "invalid YAML", `allowedWrites: @@ -802,7 +882,13 @@ forbiddenWrites: `, "changed-files.txt": "src/init/init.ts\n", }, - args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"], + args: [ + "validate", + "write-lease", + "lease.yml", + "changed-files.txt", + "--json", + ], }); expect(result.exitCode).toBe(1); @@ -810,10 +896,22 @@ forbiddenWrites: expect(result.json.errors?.join("\n")).toContain("outside allowedWrites"); }); - it.each([ - ["parent-directory changed file", "../outside.md", "invalid changed file path"], - ["absolute changed file", "/docs/truthmark/truth/workflows/overview.md", "invalid changed file path"], - ["normalized-outside changed file", "docs/truthmark/truth/../../src/init.ts", "invalid changed file path"], + itEach([ + [ + "parent-directory changed file", + "../outside.md", + "invalid changed file path", + ], + [ + "absolute changed file", + "/docs/truthmark/truth/workflows/overview.md", + "invalid changed file path", + ], + [ + "normalized-outside changed file", + "docs/truthmark/truth/../../src/init.ts", + "invalid changed file path", + ], ])("rejects write-lease %s", async (_name, changedFiles, expectedError) => { const result = await runWriteLease({ lease: `allowedWrites: @@ -829,7 +927,7 @@ forbiddenWrites: expect(result.json.errors?.join("\n")).toContain(expectedError); }); - it.each([ + itEach([ ["parent-directory allowedWrites", "../docs/truthmark/truth/**"], ["absolute allowedWrites", "/docs/truthmark/truth/**"], ["normalized-outside allowedWrites", "docs/truthmark/truth/../../src/**"], @@ -844,7 +942,9 @@ forbiddenWrites: [] expect(result.exitCode).toBe(1); expect(result.json.ok).toBe(false); - expect(result.json.errors?.join("\n")).toContain("invalid allowedWrites path"); + expect(result.json.errors?.join("\n")).toContain( + "invalid allowedWrites path", + ); }); it("rejects unsupported write-lease glob patterns with manual-validation guidance", async () => { @@ -856,7 +956,13 @@ forbiddenWrites: [] `, "changed-files.txt": "docs/truthmark/truth/workflows/overview.md\n", }, - args: ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"], + args: [ + "validate", + "write-lease", + "lease.yml", + "changed-files.txt", + "--json", + ], }); expect(result.exitCode).toBe(1); diff --git a/tests/agents/write-lease.test.ts b/tests/agents/write-lease.test.ts index 8fb145e..1f9e838 100644 --- a/tests/agents/write-lease.test.ts +++ b/tests/agents/write-lease.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { TRUTHMARK_WRITE_WORKER_REPORT_FIELDS, diff --git a/tests/checks/branch-scope.test.ts b/tests/checks/branch-scope.test.ts index c1db3db..637cfbb 100644 --- a/tests/checks/branch-scope.test.ts +++ b/tests/checks/branch-scope.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { getBranchScopeData } from "../../src/checks/branch-scope.js"; @@ -24,7 +25,8 @@ describe("getBranchScopeData", () => { expect(branchScope.relevantFileHashes).toEqual( expect.objectContaining({ ".truthmark/config.yml": expect.stringMatching(/^[0-9a-f]{64}$/), - "docs/truthmark/routes/areas.md": expect.stringMatching(/^[0-9a-f]{64}$/), + "docs/truthmark/routes/areas.md": + expect.stringMatching(/^[0-9a-f]{64}$/), }), ); expect(Object.keys(branchScope.relevantFileHashes).sort()).toEqual([ @@ -74,7 +76,9 @@ describe("runCheck branch scope", () => { headSha: string | null; } | undefined; - const categories = result.diagnostics.map((diagnostic) => diagnostic.category as string); + const categories = result.diagnostics.map( + (diagnostic) => diagnostic.category as string, + ); expect(branchScope?.identity).toBe("unborn:main"); expect(branchScope?.headSha).toBeNull(); diff --git a/tests/checks/check-truth-kinds.test.ts b/tests/checks/check-truth-kinds.test.ts index 96afdca..6deccf4 100644 --- a/tests/checks/check-truth-kinds.test.ts +++ b/tests/checks/check-truth-kinds.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runCheck } from "../../src/checks/check.js"; import { runConfig } from "../../src/config/command.js"; diff --git a/tests/checks/check.test.ts b/tests/checks/check.test.ts index c3ddb68..0b8a713 100644 --- a/tests/checks/check.test.ts +++ b/tests/checks/check.test.ts @@ -1,7 +1,8 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runInit } from "../../src/init/init.js"; import { runCheck } from "../../src/checks/check.js"; @@ -236,15 +237,13 @@ Local stale edit. ".gemini/agents/truth-doc-writer.md", ".gemini/commands/truthmark/sync.toml", ]) { - expect( - result.diagnostics.filter( - (diagnostic) => - diagnostic.category === "generated-surface" && - diagnostic.file === retiredPath && - diagnostic.message.includes("obsolete"), - ), - retiredPath, - ).toHaveLength(1); + const obsoleteDiagnostics = result.diagnostics.filter( + (diagnostic) => + diagnostic.category === "generated-surface" && + diagnostic.file === retiredPath && + diagnostic.message.includes("obsolete"), + ); + expect(obsoleteDiagnostics).toHaveLength(1); } expect( result.diagnostics.filter( diff --git a/tests/checks/decisions.test.ts b/tests/checks/decisions.test.ts index cb06a51..5d787db 100644 --- a/tests/checks/decisions.test.ts +++ b/tests/checks/decisions.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { checkDecisionSections } from "../../src/checks/decisions.js"; import { createDefaultConfig } from "../../src/config/defaults.js"; diff --git a/tests/checks/frontmatter.test.ts b/tests/checks/frontmatter.test.ts index a251fc6..853c831 100644 --- a/tests/checks/frontmatter.test.ts +++ b/tests/checks/frontmatter.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { checkFrontmatter } from "../../src/checks/frontmatter.js"; import { createDefaultConfig } from "../../src/config/defaults.js"; diff --git a/tests/checks/scorecard.test.ts b/tests/checks/scorecard.test.ts index f99a615..6f591ad 100644 --- a/tests/checks/scorecard.test.ts +++ b/tests/checks/scorecard.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { buildTruthHealthScorecard, @@ -7,7 +8,9 @@ import { } from "../../src/checks/scorecard.js"; import type { Diagnostic } from "../../src/output/diagnostic.js"; -const diagnostic = (partial: Partial & Pick): Diagnostic => ({ +const diagnostic = ( + partial: Partial & Pick, +): Diagnostic => ({ severity: "review", message: `${partial.category} diagnostic`, ...partial, @@ -24,47 +27,93 @@ const dimension = ( describe("buildTruthHealthScorecard", () => { it("returns the compact truthmark-scorecard/v0 shape with the seven dimensions", () => { - const scorecard = buildTruthHealthScorecard([], { branchFreshnessRan: false }); + const scorecard = buildTruthHealthScorecard([], { + branchFreshnessRan: false, + }); expect(scorecard.schemaVersion).toBe("truthmark-scorecard/v0"); - expect(scorecard.dimensions.map((item) => item.id)).toEqual([...TRUTH_HEALTH_DIMENSION_IDS]); + expect(scorecard.dimensions.map((item) => item.id)).toEqual([ + ...TRUTH_HEALTH_DIMENSION_IDS, + ]); for (const item of scorecard.dimensions) { - expect(Object.keys(item).sort()).toEqual(["diagnosticIndexes", "evidence", "id", "status"].filter((key) => key in item).sort()); + expect(Object.keys(item).sort()).toEqual( + ["diagnosticIndexes", "evidence", "id", "status"] + .filter((key) => key in item) + .sort(), + ); expect(item.diagnosticIndexes).toEqual([]); } - expect(dimension(scorecard.dimensions, "branch-freshness").status).toBe("not-run"); - expect(dimension(scorecard.dimensions, "branch-freshness").evidence).toEqual([ - "base not supplied", - ]); + expect(dimension(scorecard.dimensions, "branch-freshness").status).toBe( + "not-run", + ); + expect( + dimension(scorecard.dimensions, "branch-freshness").evidence, + ).toEqual(["base not supplied"]); }); it("maps diagnostic severity to fail, warn, pass, and not-run statuses", () => { const diagnostics: Diagnostic[] = [ - diagnostic({ category: "coverage", severity: "review", file: "src/unmapped.ts" }), - diagnostic({ category: "frontmatter", severity: "error", file: "docs/truth.md" }), + diagnostic({ + category: "coverage", + severity: "review", + file: "src/unmapped.ts", + }), + diagnostic({ + category: "frontmatter", + severity: "error", + file: "docs/truth.md", + }), ]; - const scorecard = buildTruthHealthScorecard(diagnostics, { branchFreshnessRan: false }); + const scorecard = buildTruthHealthScorecard(diagnostics, { + branchFreshnessRan: false, + }); - expect(dimension(scorecard.dimensions, "routing-coverage").status).toBe("warn"); - expect(dimension(scorecard.dimensions, "truth-doc-structure").status).toBe("fail"); - expect(dimension(scorecard.dimensions, "generated-surface-freshness").status).toBe("pass"); - expect(dimension(scorecard.dimensions, "branch-freshness").status).toBe("not-run"); + expect(dimension(scorecard.dimensions, "routing-coverage").status).toBe( + "warn", + ); + expect(dimension(scorecard.dimensions, "truth-doc-structure").status).toBe( + "fail", + ); + expect( + dimension(scorecard.dimensions, "generated-surface-freshness").status, + ).toBe("pass"); + expect(dimension(scorecard.dimensions, "branch-freshness").status).toBe( + "not-run", + ); }); it("maps categories to the expected dimensions and caps compact evidence", () => { const diagnostics: Diagnostic[] = [ - diagnostic({ category: "area-index", file: "docs/truthmark/routes/areas.md" }), - diagnostic({ category: "source-traceability", file: "docs/truthmark/truth/api.md" }), + diagnostic({ + category: "area-index", + file: "docs/truthmark/routes/areas.md", + }), + diagnostic({ + category: "source-traceability", + file: "docs/truthmark/truth/api.md", + }), diagnostic({ category: "freshness", message: "Changed file src/api.ts is not routed to truth ownership.", file: "src/api.ts", }), diagnostic({ category: "generated-surface", file: "AGENTS.md" }), - diagnostic({ category: "doc-structure", message: "Missing Product Decisions section", file: "docs/architecture/overview.md" }), - diagnostic({ category: "doc-structure", message: "Missing Rationale section", file: "docs/architecture/overview.md" }), - diagnostic({ category: "doc-structure", message: "Missing Scope section", file: "docs/architecture/overview.md" }), + diagnostic({ + category: "doc-structure", + message: "Missing Product Decisions section", + file: "docs/architecture/overview.md", + }), + diagnostic({ + category: "doc-structure", + message: "Missing Rationale section", + file: "docs/architecture/overview.md", + }), + diagnostic({ + category: "doc-structure", + message: "Missing Scope section", + file: "docs/architecture/overview.md", + }), diagnostic({ category: "doc-structure", message: @@ -73,20 +122,36 @@ describe("buildTruthHealthScorecard", () => { }), ]; - const scorecard = buildTruthHealthScorecard(diagnostics, { branchFreshnessRan: true }); + const scorecard = buildTruthHealthScorecard(diagnostics, { + branchFreshnessRan: true, + }); - expect(dimension(scorecard.dimensions, "routing-coverage").diagnosticIndexes).toEqual([0]); - expect(dimension(scorecard.dimensions, "ownership-clarity").diagnosticIndexes).toEqual([0, 2]); - expect(dimension(scorecard.dimensions, "source-traceability").diagnosticIndexes).toEqual([1]); - expect(dimension(scorecard.dimensions, "branch-freshness").diagnosticIndexes).toEqual([2]); - expect(dimension(scorecard.dimensions, "generated-surface-freshness").diagnosticIndexes).toEqual([3]); - expect(dimension(scorecard.dimensions, "truth-doc-structure").diagnosticIndexes).toEqual([ - 4, 5, 6, 7, - ]); - expect(dimension(scorecard.dimensions, "truth-doc-structure").evidence).toHaveLength(2); - expect(dimension(scorecard.dimensions, "decision-rationale-preservation").diagnosticIndexes).toEqual([ - 4, 5, - ]); + expect( + dimension(scorecard.dimensions, "routing-coverage").diagnosticIndexes, + ).toEqual([0]); + expect( + dimension(scorecard.dimensions, "ownership-clarity").diagnosticIndexes, + ).toEqual([0, 2]); + expect( + dimension(scorecard.dimensions, "source-traceability").diagnosticIndexes, + ).toEqual([1]); + expect( + dimension(scorecard.dimensions, "branch-freshness").diagnosticIndexes, + ).toEqual([2]); + expect( + dimension(scorecard.dimensions, "generated-surface-freshness") + .diagnosticIndexes, + ).toEqual([3]); + expect( + dimension(scorecard.dimensions, "truth-doc-structure").diagnosticIndexes, + ).toEqual([4, 5, 6, 7]); + expect( + dimension(scorecard.dimensions, "truth-doc-structure").evidence, + ).toHaveLength(2); + expect( + dimension(scorecard.dimensions, "decision-rationale-preservation") + .diagnosticIndexes, + ).toEqual([4, 5]); }); it("keeps diagnostic indexes tied to the raw diagnostics array when one diagnostic maps to multiple dimensions", () => { @@ -95,10 +160,19 @@ describe("buildTruthHealthScorecard", () => { diagnostic({ category: "coverage", file: "src/unmapped.ts" }), ]; - const scorecard = buildTruthHealthScorecard(diagnostics, { branchFreshnessRan: false }); + const scorecard = buildTruthHealthScorecard(diagnostics, { + branchFreshnessRan: false, + }); - expect(dimension(scorecard.dimensions, "routing-coverage").diagnosticIndexes).toEqual([1]); - expect(dimension(scorecard.dimensions, "ownership-clarity").diagnosticIndexes).toEqual([1]); - expect(dimension(scorecard.dimensions, "generated-surface-freshness").diagnosticIndexes).toEqual([0]); + expect( + dimension(scorecard.dimensions, "routing-coverage").diagnosticIndexes, + ).toEqual([1]); + expect( + dimension(scorecard.dimensions, "ownership-clarity").diagnosticIndexes, + ).toEqual([1]); + expect( + dimension(scorecard.dimensions, "generated-surface-freshness") + .diagnosticIndexes, + ).toEqual([0]); }); }); diff --git a/tests/cli/build-artifact.test.ts b/tests/cli/build-artifact.test.ts index 0a137be..f2bc142 100644 --- a/tests/cli/build-artifact.test.ts +++ b/tests/cli/build-artifact.test.ts @@ -3,13 +3,16 @@ import os from "node:os"; import { execa } from "execa"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { runInit } from "../../src/init/init.js"; import { createTempRepo } from "../helpers/temp-repo.js"; -const workspaceRoot = path.resolve(fileURLToPath(new URL("../../", import.meta.url))); +const workspaceRoot = path.resolve( + fileURLToPath(new URL("../../", import.meta.url)), +); const builtCliEntrypoint = path.resolve( fileURLToPath(new URL("../../dist/main.js", import.meta.url)), ); @@ -23,10 +26,14 @@ describe("built truthmark CLI", () => { expect(buildResult.exitCode).toBe(0); - const result = await execa(process.execPath, [builtCliEntrypoint, "--help"], { - cwd: workspaceRoot, - reject: false, - }); + const result = await execa( + process.execPath, + [builtCliEntrypoint, "--help"], + { + cwd: workspaceRoot, + reject: false, + }, + ); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("Usage: truthmark"); @@ -46,10 +53,14 @@ describe("built truthmark CLI", () => { await fs.symlink(builtCliEntrypoint, linkedCliEntrypoint); try { - const result = await execa(process.execPath, [linkedCliEntrypoint, "--help"], { - cwd: workspaceRoot, - reject: false, - }); + const result = await execa( + process.execPath, + [linkedCliEntrypoint, "--help"], + { + cwd: workspaceRoot, + reject: false, + }, + ); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("Usage: truthmark"); @@ -66,10 +77,14 @@ describe("built truthmark CLI", () => { expect(buildResult.exitCode).toBe(0); - const result = await execa(process.execPath, [builtCliEntrypoint, "check", "--help"], { - cwd: workspaceRoot, - reject: false, - }); + const result = await execa( + process.execPath, + [builtCliEntrypoint, "check", "--help"], + { + cwd: workspaceRoot, + reject: false, + }, + ); expect(result.exitCode).toBe(0); expect(result.stdout).not.toContain("--workflow"); @@ -121,7 +136,14 @@ describe("built truthmark CLI", () => { await runInit(repo.rootDir); const result = await execa( process.execPath, - [builtCliEntrypoint, "workflow", "status", "--workflow", "truthmark-check", "--json"], + [ + builtCliEntrypoint, + "workflow", + "status", + "--workflow", + "truthmark-check", + "--json", + ], { cwd: repo.rootDir, reject: false, @@ -134,11 +156,12 @@ describe("built truthmark CLI", () => { expect(result.exitCode).toBe(0); expect(output.command).toBe("workflow status"); - expect(output.data.workflowState.schemaVersion).toBe("truthmark-workflow/v0"); + expect(output.data.workflowState.schemaVersion).toBe( + "truthmark-workflow/v0", + ); expect(output.data.workflowState.workflow).toBe("truthmark-check"); } finally { await repo.cleanup(); } }); - }); diff --git a/tests/cli/check-workflow.test.ts b/tests/cli/check-workflow.test.ts index 87c59c7..9e98137 100644 --- a/tests/cli/check-workflow.test.ts +++ b/tests/cli/check-workflow.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runCli } from "../helpers/run-cli.js"; @@ -11,7 +12,12 @@ describe("truthmark check workflow options", () => { }); it("rejects old workflow helper invocations as unsupported", async () => { - const result = await runCli(["check", "--json", "--workflow", "truth-sync"]); + const result = await runCli([ + "check", + "--json", + "--workflow", + "truth-sync", + ]); expect(result.exitCode).not.toBe(0); expect(result.stdout).toBe(""); diff --git a/tests/cli/help.test.ts b/tests/cli/help.test.ts index 7c671f0..42321f8 100644 --- a/tests/cli/help.test.ts +++ b/tests/cli/help.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runCli } from "../helpers/run-cli.js"; @@ -75,6 +76,8 @@ describe("truthmark CLI", () => { expect(typeof payload.summary).toBe("string"); expect(payload.summary.length).toBeGreaterThan(0); expect(Array.isArray(payload.diagnostics)).toBe(true); - expect(payload.data?.scorecard?.schemaVersion).toBe("truthmark-scorecard/v0"); + expect(payload.data?.scorecard?.schemaVersion).toBe( + "truthmark-scorecard/v0", + ); }); }); diff --git a/tests/cli/index-impact-context.test.ts b/tests/cli/index-impact-context.test.ts index 9ac0bfa..c265f6f 100644 --- a/tests/cli/index-impact-context.test.ts +++ b/tests/cli/index-impact-context.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { runInit } from "../../src/init/init.js"; @@ -42,15 +43,24 @@ describe("repository intelligence CLI commands", () => { await repo.runGit(["commit", "-m", "initial"]); await repo.writeFile("src/index.ts", "export const value = 2;\n"); - const result = await runCli(["impact", "--base", "main", "--json"], { cwd: repo.rootDir }); + const result = await runCli(["impact", "--base", "main", "--json"], { + cwd: repo.rootDir, + }); const output = JSON.parse(result.stdout) as { command: string; - data: { impactSet: { schemaVersion: string; changedFiles: Array<{ path: string }> } }; + data: { + impactSet: { + schemaVersion: string; + changedFiles: Array<{ path: string }>; + }; + }; }; expect(output.command).toBe("impact"); expect(output.data.impactSet.schemaVersion).toBe("impact-set/v0"); - expect(output.data.impactSet.changedFiles.map((file) => file.path)).toContain("src/index.ts"); + expect( + output.data.impactSet.changedFiles.map((file) => file.path), + ).toContain("src/index.ts"); } finally { await repo.cleanup(); } @@ -61,13 +71,20 @@ describe("repository intelligence CLI commands", () => { try { await repo.writeFile( "package.json", - JSON.stringify({ name: "sample", scripts: { test: "vitest" } }, null, 2), + JSON.stringify( + { name: "sample", scripts: { test: "vitest" } }, + null, + 2, + ), ); await repo.writeFile( "src/math.ts", "export function add(left: number, right: number) { return left + right; }\n", ); - await repo.writeFile("tests/math.test.ts", "import { add } from '../src/math.js';\nvoid add;\n"); + await repo.writeFile( + "tests/math.test.ts", + "import { add } from '../src/math.js';\nvoid add;\n", + ); await runConfig(repo.rootDir, { force: false, stdout: false }); await runInit(repo.rootDir); await repo.runGit(["add", "."]); @@ -78,7 +95,15 @@ describe("repository intelligence CLI commands", () => { ); const result = await runCli( - ["workflow", "status", "--workflow", "truthmark-sync", "--base", "main", "--json"], + [ + "workflow", + "status", + "--workflow", + "truthmark-sync", + "--base", + "main", + "--json", + ], { cwd: repo.rootDir }, ); const output = JSON.parse(result.stdout) as { @@ -100,7 +125,11 @@ describe("repository intelligence CLI commands", () => { likelyRouteOwners: string[]; suggestedTruthDocs: string[]; openQuestions: string[]; - skippedHelperStatus: Array<{ helper: string; status: string; reason: string }>; + skippedHelperStatus: Array<{ + helper: string; + status: string; + reason: string; + }>; }; checks: { reviewChecklist: string[]; @@ -132,9 +161,13 @@ describe("repository intelligence CLI commands", () => { expect(state.actionContext.primaryTruthDocs).toEqual( state.targetTruthDocs, ); - expect(state.actionContext.allowedWritePaths).toContain("docs/truthmark/routes/areas.md"); + expect(state.actionContext.allowedWritePaths).toContain( + "docs/truthmark/routes/areas.md", + ); expect(state.targetTruthDocs.length).toBeGreaterThan(0); - expect(state.changedFiles.map((file) => file.path)).toContain("src/math.ts"); + expect(state.changedFiles.map((file) => file.path)).toContain( + "src/math.ts", + ); expect(state.affectedRoutes.length).toBeGreaterThan(0); expect(state.actionContext.evidencePrompts.length).toBeGreaterThan(0); expect(state.actionContext.requiredEvidence).toBeUndefined(); @@ -144,11 +177,16 @@ describe("repository intelligence CLI commands", () => { expect(state.checks.helpers.length).toBeGreaterThan(0); expect(state.workflowCard.affectedFiles).toContain("src/math.ts"); expect(state.workflowCard.likelyRouteOwners.length).toBeGreaterThan(0); - expect(state.workflowCard.suggestedTruthDocs).toEqual(state.targetTruthDocs); + expect(state.workflowCard.suggestedTruthDocs).toEqual( + state.targetTruthDocs, + ); expect(state.workflowCard.openQuestions).toEqual([]); expect(state.workflowCard.skippedHelperStatus).toEqual( expect.arrayContaining([ - expect.objectContaining({ helper: "validate-sync-report", status: "skipped" }), + expect.objectContaining({ + helper: "validate-sync-report", + status: "skipped", + }), ]), ); expect(Array.isArray(state.nextSteps)).toBe(true); @@ -170,7 +208,15 @@ describe("repository intelligence CLI commands", () => { await repo.writeFile("src/index.ts", "export const value = 2;\n"); const statusResult = await runCli( - ["workflow", "status", "--workflow", "truthmark-sync", "--base", "main", "--json"], + [ + "workflow", + "status", + "--workflow", + "truthmark-sync", + "--base", + "main", + "--json", + ], { cwd: repo.rootDir }, ); const statusOutput = JSON.parse(statusResult.stdout) as { @@ -191,20 +237,36 @@ describe("repository intelligence CLI commands", () => { expect(statusResult.exitCode).toBe(0); expect(statusOutput.command).toBe("workflow status"); - expect(statusOutput.data.workflowState.schemaVersion).toBe("truthmark-workflow/v0"); + expect(statusOutput.data.workflowState.schemaVersion).toBe( + "truthmark-workflow/v0", + ); expect(statusOutput.data.workflowState.workflow).toBe("truthmark-sync"); expect(statusOutput.data.workflowState.contextPack).toBeUndefined(); - expect(statusOutput.data.workflowState.actionContext.helperValidationCommands.length).toBeGreaterThan(0); - expect(statusOutput.data.workflowState.checks.helpers.length).toBeGreaterThan(0); + expect( + statusOutput.data.workflowState.actionContext.helperValidationCommands + .length, + ).toBeGreaterThan(0); + expect( + statusOutput.data.workflowState.checks.helpers.length, + ).toBeGreaterThan(0); expect(statusJson).not.toContain('"contextPack"'); expect(statusJson).not.toContain('"sourceFiles"'); expect(statusJson).not.toContain('"truthDocs":[{'); expect(statusJson).not.toContain('"routeMap"'); expect(statusJson).not.toContain('"content":'); - const impactResult = await runCli(["impact", "--base", "main", "--json"], { cwd: repo.rootDir }); + const impactResult = await runCli( + ["impact", "--base", "main", "--json"], + { cwd: repo.rootDir }, + ); const impactOutput = JSON.parse(impactResult.stdout) as { - data: { impactSet: { contextPack?: unknown; sourceFiles?: unknown; truthDocs?: unknown } }; + data: { + impactSet: { + contextPack?: unknown; + sourceFiles?: unknown; + truthDocs?: unknown; + }; + }; }; const impactJson = JSON.stringify(impactOutput.data.impactSet); @@ -228,9 +290,12 @@ describe("repository intelligence CLI commands", () => { await runConfig(repo.rootDir, { force: false, stdout: false }); await runInit(repo.rootDir); - const result = await runCli(["context", "--workflow", "truth-sync", "--base", "main", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["context", "--workflow", "truth-sync", "--base", "main", "--json"], + { + cwd: repo.rootDir, + }, + ); expect(result.exitCode).toBe(1); expect(`${result.stdout}\n${result.stderr}`).toContain("unknown command"); diff --git a/tests/cli/program.test.ts b/tests/cli/program.test.ts index 7d9644e..5d219e8 100644 --- a/tests/cli/program.test.ts +++ b/tests/cli/program.test.ts @@ -1,11 +1,14 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { buildProgram } from "../../src/cli/program.js"; describe("CLI program", () => { it("describes index as workflow routing metadata instead of semantic code indexing", () => { const program = buildProgram(); - const indexCommand = program.commands.find((command) => command.name() === "index"); + const indexCommand = program.commands.find( + (command) => command.name() === "index", + ); expect(indexCommand?.description()).toBe( "Inspect derived Truthmark workflow routing metadata for the current checkout.", diff --git a/tests/cli/validate.test.ts b/tests/cli/validate.test.ts index 3061c43..04d06f5 100644 --- a/tests/cli/validate.test.ts +++ b/tests/cli/validate.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runCli } from "../helpers/run-cli.js"; import { createTempRepo } from "../helpers/temp-repo.js"; @@ -75,9 +76,12 @@ describe("truthmark validate CLI helpers", () => { try { await repo.writeFile("report.md", validSyncReport); - const result = await runCli(["validate", "sync-report", "report.md", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["validate", "sync-report", "report.md", "--json"], + { + cwd: repo.rootDir, + }, + ); const output = JSON.parse(result.stdout) as { command: string; summary: string; @@ -111,9 +115,12 @@ describe("truthmark validate CLI helpers", () => { try { await repo.writeFile("report.md", validSyncReportWithIntent); - const result = await runCli(["validate", "sync-report", "report.md", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["validate", "sync-report", "report.md", "--json"], + { + cwd: repo.rootDir, + }, + ); const output = JSON.parse(result.stdout) as { data?: { validation?: { ok: boolean; checks?: string[] } }; }; @@ -142,9 +149,12 @@ describe("truthmark validate CLI helpers", () => { ), ); - const result = await runCli(["validate", "sync-report", "report.md", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["validate", "sync-report", "report.md", "--json"], + { + cwd: repo.rootDir, + }, + ); const output = JSON.parse(result.stdout) as { data?: { validation?: { ok: boolean; errors?: string[] } }; }; @@ -164,9 +174,12 @@ describe("truthmark validate CLI helpers", () => { try { await repo.writeFile("report.md", validDocumentReport); - const result = await runCli(["validate", "document-report", "report.md", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["validate", "document-report", "report.md", "--json"], + { + cwd: repo.rootDir, + }, + ); const output = JSON.parse(result.stdout) as { data?: { validation?: { ok: boolean; helper: string } }; }; @@ -190,12 +203,18 @@ describe("truthmark validate CLI helpers", () => { try { await repo.writeFile( "report.md", - validSyncReport.replace("validate-sync-report: ran, passed", "validate-sync-report: ran, failed"), + validSyncReport.replace( + "validate-sync-report: ran, passed", + "validate-sync-report: ran, failed", + ), ); - const result = await runCli(["validate", "sync-report", "report.md", "--json"], { - cwd: repo.rootDir, - }); + const result = await runCli( + ["validate", "sync-report", "report.md", "--json"], + { + cwd: repo.rootDir, + }, + ); const output = JSON.parse(result.stdout) as { data?: { validation?: { ok: boolean; errors?: string[] } }; }; @@ -207,7 +226,9 @@ describe("truthmark validate CLI helpers", () => { diagnostics: [], data: { validation: { ok: false } }, }); - expect(output.data?.validation?.errors?.join("\n")).toContain("ran, failed"); + expect(output.data?.validation?.errors?.join("\n")).toContain( + "ran, failed", + ); } finally { await repo.cleanup(); } @@ -216,8 +237,14 @@ describe("truthmark validate CLI helpers", () => { it("rejects write-lease path traversal and Windows absolute changed paths", async () => { const repo = await createTempRepo(); try { - await repo.writeFile("lease.yml", "allowedWrites:\n - docs/truthmark/truth/**\nforbiddenWrites:\n - src/**\n"); - await repo.writeFile("changed-files.txt", "C:/repo/docs/truthmark/truth/secret.md\n"); + await repo.writeFile( + "lease.yml", + "allowedWrites:\n - docs/truthmark/truth/**\nforbiddenWrites:\n - src/**\n", + ); + await repo.writeFile( + "changed-files.txt", + "C:/repo/docs/truthmark/truth/secret.md\n", + ); const result = await runCli( ["validate", "write-lease", "lease.yml", "changed-files.txt", "--json"], diff --git a/tests/config/config-command.test.ts b/tests/config/config-command.test.ts index 354c8bf..a941a48 100644 --- a/tests/config/config-command.test.ts +++ b/tests/config/config-command.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parse } from "yaml"; import { runConfig } from "../../src/config/command.js"; @@ -34,14 +35,16 @@ describe("runConfig", () => { expect(JSON.stringify(config)).not.toContain("docs/standards"); expect(JSON.stringify(config)).not.toContain("docs/architecture"); expect(JSON.stringify(config)).not.toContain("docs/ai"); - expect(JSON.stringify(config)).not.toContain("docs/truth\""); + expect(JSON.stringify(config)).not.toContain('docs/truth"'); expect(JSON.stringify(config)).not.toContain("product_root"); expect(JSON.stringify(config)).not.toContain("engineering_root"); expect(JSON.stringify(config)).not.toContain("docs/templates"); expect(configText).not.toContain("routes:"); expect(configText).not.toContain("templates:"); await expect(fs.stat(`${repo.rootDir}/AGENTS.md`)).rejects.toThrow(); - await expect(fs.stat(`${repo.rootDir}/docs/truthmark/routes/areas.md`)).rejects.toThrow(); + await expect( + fs.stat(`${repo.rootDir}/docs/truthmark/routes/areas.md`), + ).rejects.toThrow(); expect(result.diagnostics).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -60,11 +63,16 @@ describe("runConfig", () => { const repo = await createTempRepo(); try { - await repo.writeFile(".truthmark/config.yml", "version: 1\ncustom: true\n"); + await repo.writeFile( + ".truthmark/config.yml", + "version: 1\ncustom: true\n", + ); const result = await runConfig(repo.rootDir, {}); - expect(await repo.readFile(".truthmark/config.yml")).toBe("version: 1\ncustom: true\n"); + expect(await repo.readFile(".truthmark/config.yml")).toBe( + "version: 1\ncustom: true\n", + ); expect(result.summary).toContain("already exists"); } finally { await repo.cleanup(); @@ -75,12 +83,19 @@ describe("runConfig", () => { const repo = await createTempRepo(); try { - await repo.writeFile(".truthmark/config.yml", "version: 1\ncustom: true\n"); + await repo.writeFile( + ".truthmark/config.yml", + "version: 1\ncustom: true\n", + ); const result = await runConfig(repo.rootDir, { force: true }); - expect(await repo.readFile(".truthmark/config.yml")).toContain("workspace: docs/truthmark"); - expect(await repo.readFile(".truthmark/config.yml")).not.toContain("custom: true"); + expect(await repo.readFile(".truthmark/config.yml")).toContain( + "workspace: docs/truthmark", + ); + expect(await repo.readFile(".truthmark/config.yml")).not.toContain( + "custom: true", + ); expect(result.summary).toContain("Wrote"); } finally { await repo.cleanup(); @@ -96,8 +111,12 @@ describe("runConfig", () => { expect(result.data).toMatchObject({ path: ".truthmark/config.yml", }); - expect(String(result.data?.content)).toContain("workspace: docs/truthmark"); - await expect(fs.stat(`${repo.rootDir}/.truthmark/config.yml`)).rejects.toThrow(); + expect(String(result.data?.content)).toContain( + "workspace: docs/truthmark", + ); + await expect( + fs.stat(`${repo.rootDir}/.truthmark/config.yml`), + ).rejects.toThrow(); } finally { await repo.cleanup(); } diff --git a/tests/config/load.test.ts b/tests/config/load.test.ts index 8b3b53b..fb54b84 100644 --- a/tests/config/load.test.ts +++ b/tests/config/load.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { loadConfig } from "../../src/config/load.js"; import { createTempRepo } from "../helpers/temp-repo.js"; @@ -33,9 +34,9 @@ describe("loadConfig", () => { expect(result.status).toBe("invalid"); expect(result.config).toBeNull(); - expect(result.diagnostics.map((diagnostic) => diagnostic.message).join("\n")).toContain( - "Unsupported Truthmark config shape", - ); + expect( + result.diagnostics.map((diagnostic) => diagnostic.message).join("\n"), + ).toContain("Unsupported Truthmark config shape"); } finally { await repo.cleanup(); } @@ -59,7 +60,9 @@ describe("loadConfig", () => { expect(result.config).toBeNull(); expect(result.diagnostics).toContainEqual( expect.objectContaining({ - message: expect.stringContaining("additional property truth is not allowed"), + message: expect.stringContaining( + "additional property truth is not allowed", + ), }), ); } finally { @@ -86,10 +89,14 @@ describe("loadConfig", () => { expect(result.diagnostics).toEqual( expect.arrayContaining([ expect.objectContaining({ - message: expect.stringContaining("additional property routes is not allowed"), + message: expect.stringContaining( + "additional property routes is not allowed", + ), }), expect.objectContaining({ - message: expect.stringContaining("additional property templates is not allowed"), + message: expect.stringContaining( + "additional property templates is not allowed", + ), }), ]), ); @@ -104,7 +111,10 @@ describe("loadConfig", () => { try { await repo.writeFile( ".truthmark/config.yml", - validConfig().replace("workspace: docs/truthmark", "workspace: ../truthmark"), + validConfig().replace( + "workspace: docs/truthmark", + "workspace: ../truthmark", + ), ); const result = await loadConfig(repo.rootDir); @@ -113,7 +123,9 @@ describe("loadConfig", () => { expect(result.config).toBeNull(); expect(result.diagnostics).toContainEqual( expect.objectContaining({ - message: expect.stringContaining("truthmark.workspace must be a non-empty repo-relative directory"), + message: expect.stringContaining( + "truthmark.workspace must be a non-empty repo-relative directory", + ), }), ); } finally { @@ -127,7 +139,9 @@ describe("loadConfig", () => { try { await repo.writeFile( ".truthmark/config.yml", - validConfig(" output: docs/custom-portal\n template: docs/custom-template.md\n"), + validConfig( + " output: docs/custom-portal\n template: docs/custom-template.md\n", + ), ); const result = await loadConfig(repo.rootDir); @@ -137,10 +151,14 @@ describe("loadConfig", () => { expect(result.diagnostics).toEqual( expect.arrayContaining([ expect.objectContaining({ - message: expect.stringContaining("additional property output is not allowed"), + message: expect.stringContaining( + "additional property output is not allowed", + ), }), expect.objectContaining({ - message: expect.stringContaining("additional property template is not allowed"), + message: expect.stringContaining( + "additional property template is not allowed", + ), }), ]), ); @@ -158,15 +176,20 @@ describe("loadConfig", () => { const result = await loadConfig(repo.rootDir); expect(result.status).toBe("loaded"); - expect(result.config?.truthmark.generated.portal).toEqual({ enabled: true }); - expect(result.config?.truthmark.paths.portalOutput).toBe("docs/truthmark/generated/portal"); - expect(result.config?.truthmark.paths.portalTemplate).toBe("docs/truthmark/templates/portal.html"); + expect(result.config?.truthmark.generated.portal).toEqual({ + enabled: true, + }); + expect(result.config?.truthmark.paths.portalOutput).toBe( + "docs/truthmark/generated/portal", + ); + expect(result.config?.truthmark.paths.portalTemplate).toBe( + "docs/truthmark/templates/portal.html", + ); } finally { await repo.cleanup(); } }); - it("accepts active platform support values", async () => { const repo = await createTempRepo(); @@ -219,7 +242,9 @@ describe("loadConfig", () => { expect(result.config).toBeNull(); expect(result.diagnostics).toContainEqual( expect.objectContaining({ - message: expect.stringContaining("must be equal to one of the allowed values"), + message: expect.stringContaining( + "must be equal to one of the allowed values", + ), }), ); } finally { @@ -251,7 +276,10 @@ describe("loadConfig", () => { try { await repo.writeFile( ".truthmark/config.yml", - validConfig().replace("workspace: docs/truthmark", "workspace: docs/custom-truthmark"), + validConfig().replace( + "workspace: docs/truthmark", + "workspace: docs/custom-truthmark", + ), ); const result = await loadConfig(repo.rootDir); @@ -264,7 +292,9 @@ describe("loadConfig", () => { maxDelegationDepth: 1, }); expect(result.config?.truthmark.truth.productRoot).toBe("product"); - expect(result.config?.truthmark.truth.engineeringRoot).toBe("engineering"); + expect(result.config?.truthmark.truth.engineeringRoot).toBe( + "engineering", + ); expect(result.config?.truthmark.templates.root).toBe("templates"); expect(result.config?.truthmark.paths.routesIndex).toBe( "docs/custom-truthmark/routes/areas.md", @@ -272,7 +302,9 @@ describe("loadConfig", () => { expect(result.config?.truthmark.paths.routeAreasRoot).toBe( "docs/custom-truthmark/routes/areas", ); - expect(result.config?.truthmark.paths.productTruthRoot).toBe("docs/custom-truthmark/product"); + expect(result.config?.truthmark.paths.productTruthRoot).toBe( + "docs/custom-truthmark/product", + ); expect(result.config?.truthmark.paths.engineeringTruthRoot).toBe( "docs/custom-truthmark/engineering", ); diff --git a/tests/evidence/validate.test.ts b/tests/evidence/validate.test.ts index 1b5d744..181bab1 100644 --- a/tests/evidence/validate.test.ts +++ b/tests/evidence/validate.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { validateEvidenceReferences } from "../../src/evidence/validate.js"; import { createTempRepo, type TempRepo } from "../helpers/temp-repo.js"; diff --git a/tests/freshness/check.test.ts b/tests/freshness/check.test.ts index cd975f2..a719e27 100644 --- a/tests/freshness/check.test.ts +++ b/tests/freshness/check.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { runCheck } from "../../src/checks/check.js"; import { runConfig } from "../../src/config/command.js"; @@ -51,7 +52,9 @@ describe("freshness diagnostics", () => { category: "freshness", severity: "review", file: "src/index.ts", - message: expect.stringContaining("affected truth docs but none were changed"), + message: expect.stringContaining( + "affected truth docs but none were changed", + ), }), ); }); diff --git a/tests/fs/paths.test.ts b/tests/fs/paths.test.ts index ffc964f..4e00424 100644 --- a/tests/fs/paths.test.ts +++ b/tests/fs/paths.test.ts @@ -1,7 +1,8 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { ensureRepoFile, writeRepoFile } from "../../src/fs/paths.js"; import { createTempRepo } from "../helpers/temp-repo.js"; diff --git a/tests/generation/registry.test.ts b/tests/generation/registry.test.ts index 88c1c98..2974e9c 100644 --- a/tests/generation/registry.test.ts +++ b/tests/generation/registry.test.ts @@ -1,6 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; -import { getContentPrompt, listContentPrompts } from "../../src/generation/registry.js"; +import { + getContentPrompt, + listContentPrompts, +} from "../../src/generation/registry.js"; import { CONTENT_PROMPT_IDS } from "../../src/generation/types.js"; describe("generation prompt ids", () => { @@ -11,10 +15,12 @@ describe("generation prompt ids", () => { describe("content prompt registry", () => { it("lists registered prompts", () => { - expect(listContentPrompts().map((prompt) => prompt.id)).toEqual(["truth-doc-update"]); + expect(listContentPrompts().map((prompt) => prompt.id)).toEqual([ + "truth-doc-update", + ]); }); it("returns a prompt by id", () => { expect(getContentPrompt("truth-doc-update").title).toBe("Truth Doc Update"); }); -}); \ No newline at end of file +}); diff --git a/tests/generation/truth-doc-update.test.ts b/tests/generation/truth-doc-update.test.ts index db52fe5..3c0e626 100644 --- a/tests/generation/truth-doc-update.test.ts +++ b/tests/generation/truth-doc-update.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { truthDocUpdatePrompt } from "../../src/generation/prompts/truth-doc-update.js"; import type { ContentPromptContext } from "../../src/generation/types.js"; @@ -30,7 +31,9 @@ describe("truthDocUpdatePrompt", () => { expect(prompt).toContain('"evidenceSnippets"'); expect(prompt).toContain('"id": "E1"'); expect(prompt).toContain("SESSION_TIMEOUT_MS"); - expect(prompt).toContain("When blocked, leave targetDocs, claims, and patches empty"); + expect(prompt).toContain( + "When blocked, leave targetDocs, claims, and patches empty", + ); }); it("does not contain workflow authority language", () => { diff --git a/tests/generation/validate.test.ts b/tests/generation/validate.test.ts index 1f9efef..fa70184 100644 --- a/tests/generation/validate.test.ts +++ b/tests/generation/validate.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseTruthDocUpdateDraft } from "../../src/generation/validate.js"; import type { ContentPromptContext } from "../../src/generation/types.js"; @@ -44,7 +45,10 @@ const validDraft = { describe("parseTruthDocUpdateDraft", () => { it("accepts a valid evidence-backed draft", () => { - const draft = parseTruthDocUpdateDraft(JSON.stringify(validDraft), promptContext); + const draft = parseTruthDocUpdateDraft( + JSON.stringify(validDraft), + promptContext, + ); expect(draft.status).toBe("drafted"); expect(draft.claims[0]?.evidenceIds).toEqual(["E1"]); @@ -63,11 +67,15 @@ describe("parseTruthDocUpdateDraft", () => { ); expect(draft.status).toBe("blocked"); - expect(draft.openQuestions).toEqual(["Which bounded doc owns this behavior?"]); + expect(draft.openQuestions).toEqual([ + "Which bounded doc owns this behavior?", + ]); }); it("rejects non-json output", () => { - expect(() => parseTruthDocUpdateDraft("not json", promptContext)).toThrow("Invalid JSON"); + expect(() => parseTruthDocUpdateDraft("not json", promptContext)).toThrow( + "Invalid JSON", + ); }); it("rejects claims without evidence IDs", () => { @@ -100,7 +108,9 @@ describe("parseTruthDocUpdateDraft", () => { JSON.stringify({ ...validDraft, targetDocs: ["docs/truthmark/truth/other.md"], - patches: [{ ...validDraft.patches[0], path: "docs/truthmark/truth/other.md" }], + patches: [ + { ...validDraft.patches[0], path: "docs/truthmark/truth/other.md" }, + ], }), promptContext, ), @@ -125,7 +135,9 @@ describe("parseTruthDocUpdateDraft", () => { parseTruthDocUpdateDraft( JSON.stringify({ ...validDraft, - targetDocs: ["docs/truthmark/truth/authentication/session-timeout.md"], + targetDocs: [ + "docs/truthmark/truth/authentication/session-timeout.md", + ], patches: [], }), promptContext, @@ -197,4 +209,4 @@ describe("parseTruthDocUpdateDraft", () => { ), ).toThrow("blocked output requires at least one open question"); }); -}); \ No newline at end of file +}); diff --git a/tests/git/repository.test.ts b/tests/git/repository.test.ts index 037d33a..a516456 100644 --- a/tests/git/repository.test.ts +++ b/tests/git/repository.test.ts @@ -1,10 +1,14 @@ import fs from "node:fs"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createTempRepo } from "../helpers/temp-repo.js"; -import { getGitRepository, resolveWorktreePath } from "../../src/git/repository.js"; +import { + getGitRepository, + resolveWorktreePath, +} from "../../src/git/repository.js"; describe("getGitRepository", () => { it("returns unborn branch state without inventing a head sha", async () => { @@ -71,7 +75,9 @@ describe("resolveWorktreePath", () => { try { const repository = await getGitRepository(repo.rootDir); - expect(resolveWorktreePath(repository, "docs/truthmark/routes/areas.md")).toBe( + expect( + resolveWorktreePath(repository, "docs/truthmark/routes/areas.md"), + ).toBe( path.join(repo.rootDir, "docs", "truthmark", "routes", "areas.md"), ); expect(() => resolveWorktreePath(repository, "../outside.txt")).toThrow( @@ -86,21 +92,31 @@ describe("resolveWorktreePath", () => { const repo = await createTempRepo(); try { - const outsidePath = path.resolve(repo.rootDir, "..", "truthmark-worktree-outside.txt"); + const outsidePath = path.resolve( + repo.rootDir, + "..", + "truthmark-worktree-outside.txt", + ); await repo.writeFile("docs/placeholder.md", "# Placeholder\n"); await fs.promises.writeFile(outsidePath, "outside\n", "utf8"); - await fs.promises.symlink(outsidePath, path.join(repo.rootDir, "docs", "outside-link.txt")); + await fs.promises.symlink( + outsidePath, + path.join(repo.rootDir, "docs", "outside-link.txt"), + ); const repository = await getGitRepository(repo.rootDir); - expect(() => resolveWorktreePath(repository, "docs/outside-link.txt")).toThrow( - "must stay inside the active worktree", - ); + expect(() => + resolveWorktreePath(repository, "docs/outside-link.txt"), + ).toThrow("must stay inside the active worktree"); } finally { - await fs.promises.rm(path.resolve(repo.rootDir, "..", "truthmark-worktree-outside.txt"), { - force: true, - }); + await fs.promises.rm( + path.resolve(repo.rootDir, "..", "truthmark-worktree-outside.txt"), + { + force: true, + }, + ); await repo.cleanup(); } }); @@ -109,23 +125,33 @@ describe("resolveWorktreePath", () => { const repo = await createTempRepo(); try { - const outsideDir = path.resolve(repo.rootDir, "..", "truthmark-worktree-outside-dir"); + const outsideDir = path.resolve( + repo.rootDir, + "..", + "truthmark-worktree-outside-dir", + ); await fs.promises.mkdir(outsideDir, { recursive: true }); await repo.writeFile("docs/placeholder.md", "# Placeholder\n"); - await fs.promises.symlink(outsideDir, path.join(repo.rootDir, "docs", "outside-dir")); + await fs.promises.symlink( + outsideDir, + path.join(repo.rootDir, "docs", "outside-dir"), + ); const repository = await getGitRepository(repo.rootDir); - expect(() => resolveWorktreePath(repository, "docs/outside-dir/new.txt")).toThrow( - "must stay inside the active worktree", - ); + expect(() => + resolveWorktreePath(repository, "docs/outside-dir/new.txt"), + ).toThrow("must stay inside the active worktree"); } finally { - await fs.promises.rm(path.resolve(repo.rootDir, "..", "truthmark-worktree-outside-dir"), { - force: true, - recursive: true, - }); + await fs.promises.rm( + path.resolve(repo.rootDir, "..", "truthmark-worktree-outside-dir"), + { + force: true, + recursive: true, + }, + ); await repo.cleanup(); } }); -}); \ No newline at end of file +}); diff --git a/tests/helpers/temp-repo.test.ts b/tests/helpers/temp-repo.test.ts index 671c5d8..a797d97 100644 --- a/tests/helpers/temp-repo.test.ts +++ b/tests/helpers/temp-repo.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createTempRepo } from "./temp-repo.js"; @@ -19,4 +20,4 @@ describe("createTempRepo", () => { await expect(fs.stat(repo.rootDir)).rejects.toThrow(); }); -}); \ No newline at end of file +}); diff --git a/tests/impact/build.test.ts b/tests/impact/build.test.ts index 96520dc..84a04ab 100644 --- a/tests/impact/build.test.ts +++ b/tests/impact/build.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { buildImpactSet } from "../../src/impact/build.js"; @@ -157,7 +158,7 @@ describe("buildImpactSet", () => { ); await repo.writeFile( "tests/repo-index/build.test.ts", - "import { describe, it } from 'vitest';\ndescribe('repo index package', () => { it('builds', () => undefined); });\n", + "import { describe, it } from 'node:test';\ndescribe('repo index package', () => { it('builds', () => undefined); });\n", ); await runConfig(repo.rootDir, { force: false, stdout: false }); await runInit(repo.rootDir); diff --git a/tests/init/init-instructions.test.ts b/tests/init/init-instructions.test.ts index 30e7fd1..c46dec3 100644 --- a/tests/init/init-instructions.test.ts +++ b/tests/init/init-instructions.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { runInit } from "../../src/init/init.js"; @@ -106,7 +107,7 @@ ignore: [] ); await repo.writeFile( ".gemini/commands/truthmark/preview.toml", - "description = \"Legacy Preview Command\"\n", + 'description = "Legacy Preview Command"\n', ); await repo.writeFile("GEMINI.md", "# Legacy Gemini instructions\n"); await repo.writeFile( @@ -119,7 +120,7 @@ ignore: [] ); await repo.writeFile( ".gemini/commands/truthmark/sync.toml", - "description = \"Legacy Sync Command\"\n", + 'description = "Legacy Sync Command"\n', ); await repo.writeFile( ".agents/skills/truthmark-sync/helper-manifest.yml", @@ -152,7 +153,9 @@ ignore: [] fs.stat(`${repo.rootDir}/.gemini/commands/truthmark/sync.toml`), ).resolves.toBeDefined(); await expect( - fs.stat(`${repo.rootDir}/.agents/skills/truthmark-sync/helper-manifest.yml`), + fs.stat( + `${repo.rootDir}/.agents/skills/truthmark-sync/helper-manifest.yml`, + ), ).rejects.toThrow(); await expect( fs.stat( diff --git a/tests/init/truth-doc-templates.test.ts b/tests/init/truth-doc-templates.test.ts index 26b02fb..46efb54 100644 --- a/tests/init/truth-doc-templates.test.ts +++ b/tests/init/truth-doc-templates.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { renderBehaviorDocTemplateFile } from "../../src/templates/init-files.js"; @@ -32,11 +33,11 @@ describe("truth doc templates", () => { for (const path of behaviorTruthDocs) { const doc = await fs.readFile(path, "utf8"); - expect(doc, path).toContain("truth_kind: engineering-behavior"); - expect(doc, path).toContain("## Behavior Scenarios"); - expect(doc, path).toContain("- **GIVEN**"); - expect(doc, path).toContain("- **WHEN**"); - expect(doc, path).toContain("- **THEN**"); + expect(doc).toContain("truth_kind: engineering-behavior"); + expect(doc).toContain("## Behavior Scenarios"); + expect(doc).toContain("- **GIVEN**"); + expect(doc).toContain("- **WHEN**"); + expect(doc).toContain("- **THEN**"); } }); }); diff --git a/tests/integration/agent-workflow-contract.test.ts b/tests/integration/agent-workflow-contract.test.ts index 11ad78f..0140171 100644 --- a/tests/integration/agent-workflow-contract.test.ts +++ b/tests/integration/agent-workflow-contract.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createTempRepo } from "../helpers/temp-repo.js"; import { runCli } from "../helpers/run-cli.js"; @@ -160,9 +161,15 @@ describe("installed workflow contract", () => { repo.readFile(".gemini/commands/truthmark/preview.toml"), ).rejects.toThrow(); await expect(repo.readFile("GEMINI.md")).rejects.toThrow(); - await expect(repo.readFile(".gemini/skills/truthmark-sync/SKILL.md")).rejects.toThrow(); - await expect(repo.readFile(".gemini/commands/truthmark/sync.toml")).rejects.toThrow(); - expect(antigravitySyncRule).toContain("This rule is the Antigravity entrypoint"); + await expect( + repo.readFile(".gemini/skills/truthmark-sync/SKILL.md"), + ).rejects.toThrow(); + await expect( + repo.readFile(".gemini/commands/truthmark/sync.toml"), + ).rejects.toThrow(); + expect(antigravitySyncRule).toContain( + "This rule is the Antigravity entrypoint", + ); expect(cursorSyncSkill).toContain("Use as a Cursor Agent Skill."); expect(cursorSyncSkill).toContain("support/procedure.md"); expect(checkSkill).toContain("name: truthmark-check"); diff --git a/tests/integration/branch-scope.test.ts b/tests/integration/branch-scope.test.ts index 1f96b69..459f970 100644 --- a/tests/integration/branch-scope.test.ts +++ b/tests/integration/branch-scope.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { runCheck } from "../../src/checks/check.js"; import { runConfig } from "../../src/config/command.js"; @@ -99,16 +100,26 @@ describe("branch-scoped truth integration", () => { const primaryResult = await runCheck(repo.rootDir); const secondaryResult = await runCheck(secondary.rootDir); - const primaryHash = (primaryResult.data?.branchScope as { relevantFileHashes: Record }) - .relevantFileHashes["docs/truthmark/routes/areas.md"]; - const secondaryHash = (secondaryResult.data?.branchScope as { relevantFileHashes: Record }) - .relevantFileHashes["docs/truthmark/routes/areas.md"]; + const primaryHash = ( + primaryResult.data?.branchScope as { + relevantFileHashes: Record; + } + ).relevantFileHashes["docs/truthmark/routes/areas.md"]; + const secondaryHash = ( + secondaryResult.data?.branchScope as { + relevantFileHashes: Record; + } + ).relevantFileHashes["docs/truthmark/routes/areas.md"]; expect(primaryHash).toBeTruthy(); expect(secondaryHash).toBeTruthy(); expect(primaryHash).not.toBe(secondaryHash); - expect(await repo.readFile("docs/truthmark/routes/areas.md")).not.toContain("Feature Branch Notes"); - expect(await secondary.readFile("docs/truthmark/routes/areas.md")).toContain("Feature Branch Notes"); + expect( + await repo.readFile("docs/truthmark/routes/areas.md"), + ).not.toContain("Feature Branch Notes"); + expect( + await secondary.readFile("docs/truthmark/routes/areas.md"), + ).toContain("Feature Branch Notes"); } finally { await repo.cleanup(); } diff --git a/tests/integration/init-check-workflow.test.ts b/tests/integration/init-check-workflow.test.ts index b8a992a..4be2782 100644 --- a/tests/integration/init-check-workflow.test.ts +++ b/tests/integration/init-check-workflow.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createTempRepo } from "../helpers/temp-repo.js"; import { runCli } from "../helpers/run-cli.js"; diff --git a/tests/markdown/hash.test.ts b/tests/markdown/hash.test.ts index 780acda..5c10911 100644 --- a/tests/markdown/hash.test.ts +++ b/tests/markdown/hash.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { hashJsonLike, hashText } from "../../src/markdown/hash.js"; @@ -13,4 +14,4 @@ describe("hash helpers", () => { hashJsonLike({ nested: { x: 1, y: 2 }, a: 1, b: 2 }), ); }); -}); \ No newline at end of file +}); diff --git a/tests/markdown/parse.test.ts b/tests/markdown/parse.test.ts index 5ec1c77..2c2d9fc 100644 --- a/tests/markdown/parse.test.ts +++ b/tests/markdown/parse.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createTempRepo } from "../helpers/temp-repo.js"; import { parseMarkdownDocument } from "../../src/markdown/parse.js"; @@ -26,7 +27,10 @@ Ignore [external](https://example.com). { depth: 1, text: "Authentication" }, { depth: 2, text: "Timeouts" }, ]); - expect(document.internalLinks).toEqual(["docs/api/authentication.md", "#timeouts"]); + expect(document.internalLinks).toEqual([ + "docs/api/authentication.md", + "#timeouts", + ]); }); }); @@ -35,7 +39,10 @@ describe("resolveAuthorityPaths", () => { const repo = await createTempRepo(); try { - await repo.writeFile("docs/truthmark/routes/areas.md", "# Truthmark Areas\n"); + await repo.writeFile( + "docs/truthmark/routes/areas.md", + "# Truthmark Areas\n", + ); await repo.writeFile("docs/guides/beta.md", "# Beta\n"); await repo.writeFile("docs/guides/alpha.md", "# Alpha\n"); await repo.writeFile("docs/api/authentication.md", "# Auth API\n"); diff --git a/tests/output/render.test.ts b/tests/output/render.test.ts index f365fc8..fab26e3 100644 --- a/tests/output/render.test.ts +++ b/tests/output/render.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { DIAGNOSTIC_CATEGORIES, diff --git a/tests/package-files.test.ts b/tests/package-files.test.ts index 85c45ae..4adc6da 100644 --- a/tests/package-files.test.ts +++ b/tests/package-files.test.ts @@ -1,8 +1,9 @@ import path from "node:path"; +import { describe, it } from "node:test"; import { fileURLToPath } from "node:url"; +import { expect } from "expect"; import { execa } from "execa"; -import { describe, expect, it } from "vitest"; const repoRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), diff --git a/tests/product-boundary.test.ts b/tests/product-boundary.test.ts index 72de0f9..94457bf 100644 --- a/tests/product-boundary.test.ts +++ b/tests/product-boundary.test.ts @@ -1,7 +1,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; describe("Truthmark product boundary", () => { it("keeps spec/proposal lifecycle and workflow-engine drift out of scope", () => { diff --git a/tests/realize/report.test.ts b/tests/realize/report.test.ts index 650983d..926547a 100644 --- a/tests/realize/report.test.ts +++ b/tests/realize/report.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { renderTruthRealizeCompletedReport } from "../../src/realize/report.js"; @@ -6,7 +7,9 @@ describe("renderTruthRealizeCompletedReport", () => { it("matches the README handoff-note shape", () => { expect( renderTruthRealizeCompletedReport({ - truthDocsUsed: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUsed: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], codeUpdated: ["src/auth/session.ts"], verification: ["npm test -- auth"], }), @@ -21,4 +24,4 @@ Code updated: Verification: - npm test -- auth`); }); -}); \ No newline at end of file +}); diff --git a/tests/repo-index/build.test.ts b/tests/repo-index/build.test.ts index e174dfd..c0c8ddc 100644 --- a/tests/repo-index/build.test.ts +++ b/tests/repo-index/build.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { runInit } from "../../src/init/init.js"; diff --git a/tests/repo-index/route-map.test.ts b/tests/repo-index/route-map.test.ts index d3d9cf2..33b670c 100644 --- a/tests/repo-index/route-map.test.ts +++ b/tests/repo-index/route-map.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { runConfig } from "../../src/config/command.js"; import { runInit } from "../../src/init/init.js"; diff --git a/tests/routing/area-resolver.test.ts b/tests/routing/area-resolver.test.ts index 50d4f38..d41a4ef 100644 --- a/tests/routing/area-resolver.test.ts +++ b/tests/routing/area-resolver.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { resolveAreaRouting } from "../../src/routing/area-resolver.js"; import { createTempRepo } from "../helpers/temp-repo.js"; @@ -56,7 +57,9 @@ Update truth when: expect(result.areas).toEqual([ expect.objectContaining({ name: "Checkout", - truthDocuments: ["docs/truthmark/engineering/behaviors/payments-checkout.md"], + truthDocuments: [ + "docs/truthmark/engineering/behaviors/payments-checkout.md", + ], sourcePath: "docs/truthmark/routes/areas/payments.md", parentName: "Payments", }), @@ -96,7 +99,9 @@ Update truth when: category: "area-index", severity: "error", file: "docs/payments.md", - message: expect.stringContaining("must live under docs/truthmark/routes/areas"), + message: expect.stringContaining( + "must live under docs/truthmark/routes/areas", + ), }), ]), ); @@ -151,7 +156,9 @@ Update truth when: category: "area-index", severity: "error", file: "docs/truthmark/routes/areas/payments.md", - message: expect.stringContaining("Child area files must contain leaf areas only"), + message: expect.stringContaining( + "Child area files must contain leaf areas only", + ), }), ]), ); @@ -269,7 +276,9 @@ Update truth when: category: "area-index", severity: "review", file: "docs/truthmark/routes/areas/payments.md", - message: expect.stringContaining("outside parent area Payments code surface"), + message: expect.stringContaining( + "outside parent area Payments code surface", + ), }), ]), ); @@ -323,7 +332,9 @@ Update truth when: category: "area-index", severity: "review", file: "docs/truthmark/routes/areas/apps.md", - message: expect.stringContaining("outside parent area Apps code surface"), + message: expect.stringContaining( + "outside parent area Apps code surface", + ), }), ]), ); diff --git a/tests/routing/areas-markdown-globs.test.ts b/tests/routing/areas-markdown-globs.test.ts index f9d08ce..729fcec 100644 --- a/tests/routing/areas-markdown-globs.test.ts +++ b/tests/routing/areas-markdown-globs.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseAreasMarkdown } from "../../src/routing/areas.js"; diff --git a/tests/routing/areas.test.ts b/tests/routing/areas.test.ts index c90a02f..7c8730a 100644 --- a/tests/routing/areas.test.ts +++ b/tests/routing/areas.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseAreasMarkdown } from "../../src/routing/areas.js"; @@ -51,7 +52,10 @@ Update truth when: }), ], codeSurface: ["src/auth/**", "src/session/**"], - updateTruthWhen: ["authentication behavior changes", "permissions change"], + updateTruthWhen: [ + "authentication behavior changes", + "permissions change", + ], }), ]); }); @@ -153,7 +157,9 @@ Update truth when: expect.objectContaining({ category: "area-index", severity: "error", - message: expect.stringContaining("exactly one of Truth documents or Area files"), + message: expect.stringContaining( + "exactly one of Truth documents or Area files", + ), }), ]), ); diff --git a/tests/sync/policy.test.ts b/tests/sync/policy.test.ts index 544514e..b6483cd 100644 --- a/tests/sync/policy.test.ts +++ b/tests/sync/policy.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { TRUTH_SYNC_BOUNDARIES, diff --git a/tests/sync/report.test.ts b/tests/sync/report.test.ts index d440cde..90cc136 100644 --- a/tests/sync/report.test.ts +++ b/tests/sync/report.test.ts @@ -1,7 +1,8 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { parseTruthSyncReport, @@ -15,10 +16,13 @@ describe("Truth Sync reporting", () => { const report = renderTruthSyncCompletedReport({ changedCode: ["src/auth/session.ts"], ownershipReviewed: ["docs/truthmark/routes/areas/repository.md"], - truthDocsUpdated: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUpdated: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], evidenceChecked: [ { - claim: "Session timeout behavior is documented in the authentication truth doc.", + claim: + "Session timeout behavior is documented in the authentication truth doc.", evidence: [ "src/auth/session.ts:12", "docs/truthmark/routes/areas/repository.md:18", @@ -54,11 +58,14 @@ Notes: status: "completed", changedCode: ["src/auth/session.ts"], ownershipReviewed: ["docs/truthmark/routes/areas/repository.md"], - truthDocsUpdated: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUpdated: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], decisionRationaleCaptured: ["none provided in task conversation"], evidenceChecked: [ { - claim: "Session timeout behavior is documented in the authentication truth doc.", + claim: + "Session timeout behavior is documented in the authentication truth doc.", evidence: [ "src/auth/session.ts:12", "docs/truthmark/routes/areas/repository.md:18", @@ -74,7 +81,9 @@ Notes: const syncIntent = { changedCodeReviewed: ["src/auth/session.ts"], affectedRouteOrTruthOwner: ["docs/truthmark/routes/areas/repository.md"], - targetTruthDocs: ["docs/truthmark/engineering/behaviors/authentication.md"], + targetTruthDocs: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], intendedUpdate: ["Update documented session timeout behavior."], evidenceToVerify: ["src/auth/session.ts:12"], userProvidedDecisionRationale: [ @@ -88,10 +97,13 @@ Notes: changedCode: ["src/auth/session.ts"], syncIntent, ownershipReviewed: ["docs/truthmark/routes/areas/repository.md"], - truthDocsUpdated: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUpdated: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], evidenceChecked: [ { - claim: "Session timeout behavior is documented in the authentication truth doc.", + claim: + "Session timeout behavior is documented in the authentication truth doc.", evidence: ["src/auth/session.ts:12"], result: "supported", }, @@ -119,7 +131,9 @@ Notes: "Placed user rationale in docs/truthmark/engineering/behaviors/authentication.md under Engineering Decisions and Rationale.", ], changedCode: ["src/auth/session.ts"], - truthDocsUpdated: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUpdated: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], }); }); @@ -155,10 +169,13 @@ Notes: const report = renderTruthSyncCompletedReport({ changedCode: ["src/auth/session.ts"], ownershipReviewed: ["docs/truthmark/routes/areas/repository.md"], - truthDocsUpdated: ["docs/truthmark/engineering/behaviors/authentication.md"], + truthDocsUpdated: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], evidenceChecked: [ { - claim: "Session timeout behavior is documented in the authentication truth doc.", + claim: + "Session timeout behavior is documented in the authentication truth doc.", evidence: ["src/auth/session.ts:12"], result: "supported", }, @@ -191,7 +208,9 @@ Reason: expect( renderTruthSyncBlockedReport({ reason: "relevant tests failed before sync", - manualReviewFiles: ["docs/truthmark/engineering/behaviors/authentication.md"], + manualReviewFiles: [ + "docs/truthmark/engineering/behaviors/authentication.md", + ], nextAction: "fix the failing tests, then rerun Truth Sync", }), ).toBe(`Truth Sync: blocked diff --git a/tests/sync/surfaces.test.ts b/tests/sync/surfaces.test.ts index 052416d..71b2106 100644 --- a/tests/sync/surfaces.test.ts +++ b/tests/sync/surfaces.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { getUncommittedChanges } from "../../src/git/changes.js"; import { buildChangedSurfaces } from "../../src/sync/surfaces.js"; @@ -17,19 +18,28 @@ describe("buildChangedSurfaces", () => { const repo = await createTempRepo(); try { - await repo.writeFile("src/auth/session.ts", `${buildNumberedSource(12)}\n`); + await repo.writeFile( + "src/auth/session.ts", + `${buildNumberedSource(12)}\n`, + ); await repo.runGit(["add", "src/auth/session.ts"]); await repo.runGit(["commit", "-m", "test: baseline tracked source"]); await repo.writeFile( "src/auth/session.ts", - `${buildNumberedSource(5)}\nexport const line6 = 'updated';\n${buildNumberedSource(12) + `${buildNumberedSource(5)}\nexport const line6 = 'updated';\n${buildNumberedSource( + 12, + ) .split("\n") .slice(6) .join("\n")}\n`, ); - const surfaces = await buildChangedSurfaces(repo.rootDir, await getUncommittedChanges(repo.rootDir), []); + const surfaces = await buildChangedSurfaces( + repo.rootDir, + await getUncommittedChanges(repo.rootDir), + [], + ); expect(surfaces).toHaveLength(1); expect(surfaces[0]).toMatchObject({ @@ -42,8 +52,12 @@ describe("buildChangedSurfaces", () => { }, ], }); - expect(surfaces[0]?.segments[0]?.content.includes("line6 = 'updated'"))?.toBe(true); - expect(surfaces[0]?.segments[0]?.content.includes("line1 = 1"))?.toBe(false); + expect( + surfaces[0]?.segments[0]?.content.includes("line6 = 'updated'"), + )?.toBe(true); + expect(surfaces[0]?.segments[0]?.content.includes("line1 = 1"))?.toBe( + false, + ); } finally { await repo.cleanup(); } @@ -55,7 +69,11 @@ describe("buildChangedSurfaces", () => { try { await repo.writeFile("src/new-file.ts", `${buildNumberedSource(80)}\n`); - const surfaces = await buildChangedSurfaces(repo.rootDir, await getUncommittedChanges(repo.rootDir), []); + const surfaces = await buildChangedSurfaces( + repo.rootDir, + await getUncommittedChanges(repo.rootDir), + [], + ); expect(surfaces).toHaveLength(1); expect(surfaces[0]).toMatchObject({ @@ -68,8 +86,12 @@ describe("buildChangedSurfaces", () => { }, ], }); - expect(surfaces[0]?.segments[0]?.content.includes("line40 = 40"))?.toBe(true); - expect(surfaces[0]?.segments[0]?.content.includes("line41 = 41"))?.toBe(false); + expect(surfaces[0]?.segments[0]?.content.includes("line40 = 40"))?.toBe( + true, + ); + expect(surfaces[0]?.segments[0]?.content.includes("line41 = 41"))?.toBe( + false, + ); } finally { await repo.cleanup(); } @@ -79,13 +101,18 @@ describe("buildChangedSurfaces", () => { const repo = await createTempRepo(); try { - await repo.writeFile("src/auth/session.ts", `${buildNumberedSource(12)}\n`); + await repo.writeFile( + "src/auth/session.ts", + `${buildNumberedSource(12)}\n`, + ); await repo.runGit(["add", "src/auth/session.ts"]); await repo.runGit(["commit", "-m", "test: baseline tracked source"]); await repo.writeFile( "src/auth/session.ts", - `${buildNumberedSource(5)}\nexport const line6 = 'expanded';\n${buildNumberedSource(12) + `${buildNumberedSource(5)}\nexport const line6 = 'expanded';\n${buildNumberedSource( + 12, + ) .split("\n") .slice(6) .join("\n")}\n`, @@ -102,8 +129,12 @@ describe("buildChangedSurfaces", () => { startLine: 2, endLine: 10, }); - expect(surfaces[0]?.segments[0]?.content.includes("line2 = 2"))?.toBe(true); - expect(surfaces[0]?.segments[0]?.content.includes("line11 = 11"))?.toBe(false); + expect(surfaces[0]?.segments[0]?.content.includes("line2 = 2"))?.toBe( + true, + ); + expect(surfaces[0]?.segments[0]?.content.includes("line11 = 11"))?.toBe( + false, + ); } finally { await repo.cleanup(); } @@ -113,10 +144,19 @@ describe("buildChangedSurfaces", () => { const repo = await createTempRepo(); try { - await repo.writeFile("src/auth/session.ts", "export const session = true;\n"); - await repo.writeFile("docs/guides/authentication.md", "# Authentication\n"); + await repo.writeFile( + "src/auth/session.ts", + "export const session = true;\n", + ); + await repo.writeFile( + "docs/guides/authentication.md", + "# Authentication\n", + ); await repo.writeFile("package.json", '{"name":"truthmark"}\n'); - await repo.writeFile("dist/generated.js", "export const generated = true;\n"); + await repo.writeFile( + "dist/generated.js", + "export const generated = true;\n", + ); await repo.writeFile(".truthmark/cache/state.json", '{"cached":true}\n'); const surfaces = await buildChangedSurfaces( @@ -125,7 +165,9 @@ describe("buildChangedSurfaces", () => { ["dist/**"], ); - expect(surfaces.map((surface) => surface.path)).toEqual(["src/auth/session.ts"]); + expect(surfaces.map((surface) => surface.path)).toEqual([ + "src/auth/session.ts", + ]); } finally { await repo.cleanup(); } @@ -135,7 +177,10 @@ describe("buildChangedSurfaces", () => { const repo = await createTempRepo(); try { - await repo.writeFile("src/auth/session.ts", "export const session = true;\n"); + await repo.writeFile( + "src/auth/session.ts", + "export const session = true;\n", + ); await repo.runGit(["add", "src/auth/session.ts"]); await repo.runGit(["commit", "-m", "test: baseline tracked source"]); @@ -165,7 +210,10 @@ describe("buildChangedSurfaces", () => { const repo = await createTempRepo(); try { - await repo.writeFile("docs/guides/authentication.md", "# Authentication\n"); + await repo.writeFile( + "docs/guides/authentication.md", + "# Authentication\n", + ); await repo.runGit(["add", "docs/guides/authentication.md"]); await repo.runGit(["commit", "-m", "test: baseline tracked guide"]); @@ -182,4 +230,4 @@ describe("buildChangedSurfaces", () => { await repo.cleanup(); } }); -}); \ No newline at end of file +}); diff --git a/tests/templates/generated-surfaces.test.ts b/tests/templates/generated-surfaces.test.ts index d1405aa..211d4d9 100644 --- a/tests/templates/generated-surfaces.test.ts +++ b/tests/templates/generated-surfaces.test.ts @@ -1,7 +1,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createDefaultConfig } from "../../src/config/defaults.js"; import { renderTruthSyncSkillBody } from "../../src/agents/truth-sync.js"; @@ -50,10 +51,8 @@ describe("Truthmark Portal generated surfaces", () => { for (const procedurePath of readOnlyProcedurePaths) { const content = readFileSync(join(process.cwd(), procedurePath), "utf8"); - expect(content, procedurePath).not.toContain( - staleWriteAuthorizingLaneText, - ); - expect(content, procedurePath).toContain( + expect(content).not.toContain(staleWriteAuthorizingLaneText); + expect(content).toContain( "classify the request or changed surface as product-lane, engineering-lane, both-lane, or ambiguous for reporting only", ); } @@ -63,23 +62,19 @@ describe("Truthmark Portal generated surfaces", () => { for (const procedurePath of syncProcedurePaths) { const content = readFileSync(join(process.cwd(), procedurePath), "utf8"); - expect(content, procedurePath).toContain("Product truth decision"); - expect(content, procedurePath).toContain( + expect(content).toContain("Product truth decision"); + expect(content).toContain( "ask whether a user-visible promise, capability boundary, API contract, acceptance criterion, or explicit user/product evidence changed", ); - expect(content, procedurePath).toContain( + expect(content).toContain( "if no, default to engineering truth under docs/truthmark/engineering for internal implementation changes", ); - expect(content, procedurePath).toContain( + expect(content).toContain( "Product truth is opt-in for externally visible promises, product boundaries, APIs, acceptance criteria, or explicit user/product evidence.", ); - expect(content, procedurePath).toContain( - "User-provided decisions/rationale", - ); - expect(content, procedurePath).not.toContain( - staleWriteAuthorizingLaneText, - ); - expect(content, procedurePath).not.toContain( + expect(content).toContain("User-provided decisions/rationale"); + expect(content).not.toContain(staleWriteAuthorizingLaneText); + expect(content).not.toContain( "classify lane impact as product-lane, engineering-lane, both-lane, or ambiguous before writing", ); } @@ -112,7 +107,7 @@ describe("Truthmark Portal generated surfaces", () => { expect(publicWorkflowSurfaces.length).toBeGreaterThan(0); for (const surface of publicWorkflowSurfaces) { for (const text of forbiddenText) { - expect(surface.content, surface.path).not.toContain(text); + expect(surface.content).not.toContain(text); } } }); @@ -142,7 +137,7 @@ describe("Truthmark Portal generated surfaces", () => { for (const surface of runtimeSurfaces) { for (const text of crossHostInvocationText) { - expect(surface.content, surface.path).not.toContain(text); + expect(surface.content).not.toContain(text); } } @@ -150,8 +145,8 @@ describe("Truthmark Portal generated surfaces", () => { surface.path.endsWith("/SKILL.md"), ); for (const surface of skillEntrypoints) { - expect(surface.content, surface.path).toContain("Progressive disclosure:"); - expect(surface.content, surface.path).not.toContain("Read support/"); + expect(surface.content).toContain("Progressive disclosure:"); + expect(surface.content).not.toContain("Read support/"); } const flatRules = runtimeSurfaces.filter( @@ -161,16 +156,12 @@ describe("Truthmark Portal generated surfaces", () => { ); expect(flatRules.length).toBeGreaterThan(0); for (const surface of flatRules) { - expect(surface.content, surface.path).not.toContain("Quick procedure:"); - expect(surface.content, surface.path).not.toContain("support/procedure.md"); - expect(surface.content, surface.path).not.toContain( - "support/report-template.md", - ); - expect(surface.content, surface.path).not.toContain( - "support/subagents-and-leases.md", - ); - expect(surface.content, surface.path).toContain("## Procedure"); - expect(surface.content, surface.path).toContain("## Report Template"); + expect(surface.content).not.toContain("Quick procedure:"); + expect(surface.content).not.toContain("support/procedure.md"); + expect(surface.content).not.toContain("support/report-template.md"); + expect(surface.content).not.toContain("support/subagents-and-leases.md"); + expect(surface.content).toContain("## Procedure"); + expect(surface.content).toContain("## Report Template"); } }); @@ -189,7 +180,9 @@ describe("Truthmark Portal generated surfaces", () => { const syncProcedure = byPath.get(".agents/skills/truthmark-sync/support/procedure.md") ?? ""; - expect(syncSkill).toContain("Use this skill automatically before finishing"); + expect(syncSkill).toContain( + "Use this skill automatically before finishing", + ); expect(syncSkill).not.toContain("Parent workflow:"); expect(syncSkill).toContain("support/procedure.md"); expect(syncSkill).toContain("support/report-template.md"); @@ -200,32 +193,50 @@ describe("Truthmark Portal generated surfaces", () => { expect(syncProcedure).toContain( "truthmark validate sync-report --json", ); - expect(byPath.has(".agents/skills/truthmark-sync/helper-manifest.yml")).toBe(false); - expect(byPath.has(".agents/skills/truthmark-sync/support/helper-policy.md")).toBe(false); + expect( + byPath.has(".agents/skills/truthmark-sync/helper-manifest.yml"), + ).toBe(false); + expect( + byPath.has(".agents/skills/truthmark-sync/support/helper-policy.md"), + ).toBe(false); expect(byPath.has(".agents/skills/truthmark-preview/SKILL.md")).toBe(false); - expect(byPath.has(".github/prompts/truthmark-preview.prompt.md")).toBe(false); + expect(byPath.has(".github/prompts/truthmark-preview.prompt.md")).toBe( + false, + ); expect(byPath.has(".gemini/commands/truthmark/preview.toml")).toBe(false); expect(byPath.has("GEMINI.md")).toBe(false); expect(byPath.has(".gemini/skills/truthmark-sync/SKILL.md")).toBe(false); expect(byPath.has(".gemini/commands/truthmark/sync.toml")).toBe(false); expect(byPath.has(".antigravity/rules/truthmark-sync.md")).toBe(true); expect(byPath.has(".cursor/skills/truthmark-sync/SKILL.md")).toBe(true); - expect(byPath.has(".cursor/skills/truthmark-sync/support/procedure.md")).toBe(true); - expect(byPath.has(".cursor/skills/truthmark-sync/support/report-template.md")).toBe(true); - expect(byPath.has(".agents/skills/truthmark-preview/agents/openai.yaml")).toBe( - false, - ); + expect( + byPath.has(".cursor/skills/truthmark-sync/support/procedure.md"), + ).toBe(true); + expect( + byPath.has(".cursor/skills/truthmark-sync/support/report-template.md"), + ).toBe(true); + expect( + byPath.has(".agents/skills/truthmark-preview/agents/openai.yaml"), + ).toBe(false); }); it("does not render unused repo-local agent package copies", () => { const config = createDefaultConfig(); config.platforms = [...allPlatforms]; - const paths = renderGeneratedSurfaces(config).map((surface) => surface.path); + const paths = renderGeneratedSurfaces(config).map( + (surface) => surface.path, + ); - expect(paths.some((path) => path.startsWith(".truthmark/agent/"))).toBe(false); + expect(paths.some((path) => path.startsWith(".truthmark/agent/"))).toBe( + false, + ); expect(paths).toContain(".agents/skills/truthmark-sync/SKILL.md"); - expect(paths).toContain(".agents/skills/truthmark-sync/support/procedure.md"); - expect(paths).toContain(".opencode/skills/truthmark-sync/support/procedure.md"); + expect(paths).toContain( + ".agents/skills/truthmark-sync/support/procedure.md", + ); + expect(paths).toContain( + ".opencode/skills/truthmark-sync/support/procedure.md", + ); }); it("renders host skill packages with colocated native resources", () => { @@ -242,14 +253,17 @@ describe("Truthmark Portal generated surfaces", () => { const codexProcedure = byPath.get(".agents/skills/truthmark-sync/support/procedure.md") ?? ""; const opencodeReport = - byPath.get(".opencode/skills/truthmark-sync/support/report-template.md") ?? - ""; + byPath.get( + ".opencode/skills/truthmark-sync/support/report-template.md", + ) ?? ""; expect(claudeProcedure).toContain("Parent workflow:"); expect(codexProcedure).toContain("Parent workflow:"); expect(opencodeReport).toContain("Changed code reviewed:"); expect(opencodeReport).toContain("Decision/rationale captured:"); - expect(claudeProcedure).not.toContain("truthmark:adapter-mode=expanded-adapter"); + expect(claudeProcedure).not.toContain( + "truthmark:adapter-mode=expanded-adapter", + ); }); it("does not teach Sync to update bootstrap routing as behavior truth", () => { @@ -260,7 +274,8 @@ describe("Truthmark Portal generated surfaces", () => { (surface) => surface.path.endsWith("truthmark-sync/support/report-template.md") || surface.path === ".antigravity/rules/truthmark-sync.md" || - surface.path === ".cursor/skills/truthmark-sync/support/report-template.md", + surface.path === + ".cursor/skills/truthmark-sync/support/report-template.md", ); const checkedInSyncReportPaths = [ ".agents/skills/truthmark-sync/support/report-template.md", @@ -271,7 +286,10 @@ describe("Truthmark Portal generated surfaces", () => { ".cursor/skills/truthmark-sync/support/report-template.md", ]; const surfacesToCheck = [ - { path: "renderTruthSyncSkillBody", content: renderTruthSyncSkillBody(config) }, + { + path: "renderTruthSyncSkillBody", + content: renderTruthSyncSkillBody(config), + }, ...generatedSyncReportSurfaces, ...checkedInSyncReportPaths.map((path) => ({ path, @@ -281,28 +299,28 @@ describe("Truthmark Portal generated surfaces", () => { expect(generatedSyncReportSurfaces.length).toBeGreaterThan(0); for (const surface of surfacesToCheck) { - expect(surface.content, surface.path).toContain( + expect(surface.content).toContain( "docs/truthmark/routes/areas/authentication.md", ); - expect(surface.content, surface.path).toContain( + expect(surface.content).toContain( "docs/truthmark/engineering/behaviors/session-timeout.md", ); - expect(surface.content, surface.path).toContain( + expect(surface.content).toContain( "Changed code maps only to the provisional bootstrap route.", ); - expect(surface.content, surface.path).toContain( + expect(surface.content).toContain( "Run Truth Structure for src/auth/** before updating behavior truth.", ); - expect(surface.content, surface.path).not.toContain( + expect(surface.content).not.toContain( "- Target truth docs: docs/truthmark/engineering/repository/bootstrap-routing.md", ); - expect(surface.content, surface.path).not.toContain( + expect(surface.content).not.toContain( [ "Truth docs updated:", "- docs/truthmark/engineering/repository/bootstrap-routing.md", ].join("\n"), ); - expect(surface.content, surface.path).not.toContain( + expect(surface.content).not.toContain( "Session timeout behavior is documented in the mapped repository truth doc.", ); } @@ -345,7 +363,8 @@ describe("Truthmark Portal generated surfaces", () => { byPath.get(".github/prompts/truthmark-portal.prompt.md") ?? ""; const antigravityRule = byPath.get(".antigravity/rules/truthmark-portal.md") ?? ""; - const cursorSkill = byPath.get(".cursor/skills/truthmark-portal/SKILL.md") ?? ""; + const cursorSkill = + byPath.get(".cursor/skills/truthmark-portal/SKILL.md") ?? ""; const cursorProcedure = byPath.get(".cursor/skills/truthmark-portal/support/procedure.md") ?? ""; const agentsBlock = renderAgentsBlock(config); diff --git a/tests/templates/github-action.test.ts b/tests/templates/github-action.test.ts index e274447..e48706a 100644 --- a/tests/templates/github-action.test.ts +++ b/tests/templates/github-action.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { renderGitHubActionExample } from "../../src/templates/github-action.js"; diff --git a/tests/truth/docs.test.ts b/tests/truth/docs.test.ts index dbe0c3f..3a2ec15 100644 --- a/tests/truth/docs.test.ts +++ b/tests/truth/docs.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { createDefaultConfig } from "../../src/config/defaults.js"; import { resolveTruthDocsRoot } from "../../src/truth/docs.js"; diff --git a/tests/version.test.ts b/tests/version.test.ts index bc08593..383556c 100644 --- a/tests/version.test.ts +++ b/tests/version.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; -import { describe, expect, it } from "vitest"; +import { describe, it } from "node:test"; +import { expect } from "expect"; import { TRUTHMARK_VERSION } from "../src/version.js"; diff --git a/tests/workflow-state/build.test.ts b/tests/workflow-state/build.test.ts index 774114b..e19bdfa 100644 --- a/tests/workflow-state/build.test.ts +++ b/tests/workflow-state/build.test.ts @@ -1,7 +1,8 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, it } from "node:test"; +import { expect } from "expect"; import { TRUTHMARK_WORKFLOW_MANIFEST } from "../../src/agents/workflow-manifest.js"; import { runConfig } from "../../src/config/command.js"; @@ -11,7 +12,10 @@ import { buildWorkflowState } from "../../src/workflow-state/build.js"; import type { WorkflowState } from "../../src/workflow-state/types.js"; import { createTempRepo, type TempRepo } from "../helpers/temp-repo.js"; -const readTree = async (rootDir: string, relativeRoot: string): Promise> => { +const readTree = async ( + rootDir: string, + relativeRoot: string, +): Promise> => { const absoluteRoot = path.join(rootDir, relativeRoot); const snapshot: Record = {}; @@ -28,7 +32,9 @@ const readTree = async (rootDir: string, relativeRoot: string): Promise { it("restricts sync and document to routed truth and route writes", () => { for (const workflow of ["truthmark-sync", "truthmark-document"] as const) { - const context = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST[workflow], { - routeIndexPath: "docs/truthmark/routes/areas.md", - routeFiles: ["docs/truthmark/routes/areas/repository.md"], - truthDocs: ["docs/truthmark/engineering/repository/overview.md"], - }); + const context = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST[workflow], + { + routeIndexPath: "docs/truthmark/routes/areas.md", + routeFiles: ["docs/truthmark/routes/areas/repository.md"], + truthDocs: ["docs/truthmark/engineering/repository/overview.md"], + }, + ); expect(context.mode).toBe("truth-doc-write"); expect(context.allowedWritePaths).toEqual([ @@ -158,11 +170,14 @@ describe("action context", () => { }); it("restricts structure to route files and starter truth docs", () => { - const context = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-structure"], { - routeIndexPath: "docs/truthmark/routes/areas.md", - routeFiles: ["docs/truthmark/routes/areas/new-area.md"], - starterTruthDocs: ["docs/truthmark/engineering/new-area.md"], - }); + const context = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-structure"], + { + routeIndexPath: "docs/truthmark/routes/areas.md", + routeFiles: ["docs/truthmark/routes/areas/new-area.md"], + starterTruthDocs: ["docs/truthmark/engineering/new-area.md"], + }, + ); expect(context.mode).toBe("route-write"); expect(context.allowedWritePaths).toEqual([ @@ -174,13 +189,16 @@ describe("action context", () => { }); it("forbids truth documentation writes for realize", () => { - const context = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-realize"], { - routeIndexPath: "docs/truthmark/routes/areas.md", - routeFiles: ["docs/truthmark/routes/areas/repository.md"], - truthRoot: "docs/truthmark/engineering", - truthDocs: ["docs/truthmark/engineering/repository/overview.md"], - codeWritePaths: ["src/**/*.ts"], - }); + const context = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-realize"], + { + routeIndexPath: "docs/truthmark/routes/areas.md", + routeFiles: ["docs/truthmark/routes/areas/repository.md"], + truthRoot: "docs/truthmark/engineering", + truthDocs: ["docs/truthmark/engineering/repository/overview.md"], + codeWritePaths: ["src/**/*.ts"], + }, + ); expect(context.mode).toBe("code-write"); expect(context.allowedWritePaths).toEqual(["src/**/*.ts"]); @@ -193,17 +211,25 @@ describe("action context", () => { }); it("restricts portal writes to configured output when enabled", () => { - const enabled = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-portal"], { - portalEnabled: true, - portalOutputPath: "docs/truthmark/generated/portal", - }); - const disabled = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-portal"], { - portalEnabled: false, - portalOutputPath: "docs/truthmark/generated/portal", - }); + const enabled = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-portal"], + { + portalEnabled: true, + portalOutputPath: "docs/truthmark/generated/portal", + }, + ); + const disabled = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-portal"], + { + portalEnabled: false, + portalOutputPath: "docs/truthmark/generated/portal", + }, + ); expect(enabled.mode).toBe("portal-write"); - expect(enabled.allowedWritePaths).toEqual(["docs/truthmark/generated/portal/**"]); + expect(enabled.allowedWritePaths).toEqual([ + "docs/truthmark/generated/portal/**", + ]); expect(disabled.allowedWritePaths).toEqual([]); }); @@ -214,7 +240,9 @@ describe("action context", () => { "truthmark-structure", "truthmark-portal", ] as const) { - const context = buildWorkflowActionContext(TRUTHMARK_WORKFLOW_MANIFEST[workflow]); + const context = buildWorkflowActionContext( + TRUTHMARK_WORKFLOW_MANIFEST[workflow], + ); expect(context.allowedWritePaths).toEqual([]); expect(context.allowedWritePaths).not.toContain("*"); @@ -243,7 +271,9 @@ describe("buildWorkflowState", () => { expect(state.workflow).toBe("truthmark-sync"); expect("base" in state).toBe(false); expect(state.workflow).not.toBe("truth-sync"); - expect(state.changedFiles).toContainEqual(expect.objectContaining({ path: "src/math.ts" })); + expect(state.changedFiles).toContainEqual( + expect.objectContaining({ path: "src/math.ts" }), + ); expect(state.affectedRoutes.length).toBeGreaterThan(0); expect(state.targetTruthDocs.length).toBeGreaterThan(0); expect(state.actionContext.mode).toBe("truth-doc-write"); @@ -251,25 +281,40 @@ describe("buildWorkflowState", () => { expect.arrayContaining(state.targetTruthDocs), ); expect(state.checks.reviewChecklist).toEqual( - expect.arrayContaining(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-sync"].reviewQuestions), + expect.arrayContaining( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-sync"].reviewQuestions, + ), + ); + expect(state.actionContext.evidencePrompts.join("\n")).toContain( + "Evidence checklist", ); - expect(state.actionContext.evidencePrompts.join("\n")).toContain("Evidence checklist"); expect(state.workflowCard.affectedFiles).toContain("src/math.ts"); expect(state.workflowCard.likelyRouteOwners.length).toBeGreaterThan(0); - expect(state.workflowCard.suggestedTruthDocs).toEqual(state.targetTruthDocs); + expect(state.workflowCard.suggestedTruthDocs).toEqual( + state.targetTruthDocs, + ); expect(state.workflowCard.skippedHelperStatus).toEqual( expect.arrayContaining([ - expect.objectContaining({ helper: "validate-sync-report", status: "skipped" }), + expect.objectContaining({ + helper: "validate-sync-report", + status: "skipped", + }), ]), ); expect(JSON.stringify(state)).not.toContain(["required", "Gates"].join("")); expect(JSON.stringify(state)).not.toContain("requiredEvidence"); expect(JSON.stringify(state)).not.toContain("reviewQuestions"); - expect(state.checks.helpers.map((helper) => helper.id)).toContain("validate-sync-report"); - expect(JSON.stringify((state.checks as { affectedTests?: string[] }).affectedTests ?? [])).toContain( - "tests/math.test.ts", + expect(state.checks.helpers.map((helper) => helper.id)).toContain( + "validate-sync-report", + ); + expect( + JSON.stringify( + (state.checks as { affectedTests?: string[] }).affectedTests ?? [], + ), + ).toContain("tests/math.test.ts"); + expect(state.reportSections).toEqual( + TRUTHMARK_WORKFLOW_MANIFEST["truthmark-sync"].reportSections, ); - expect(state.reportSections).toEqual(TRUTHMARK_WORKFLOW_MANIFEST["truthmark-sync"].reportSections); expect(Array.isArray(state.diagnostics)).toBe(true); expect("base" in state).toBe(false); expect("contextPack" in state).toBe(false); @@ -290,15 +335,15 @@ describe("buildWorkflowState", () => { expect(state.affectedRoutes.map((route) => route.sourcePath)).toEqual([ "docs/truthmark/routes/areas/repository.md", ]); - expect(state.targetTruthDocs).toEqual(["docs/truthmark/engineering/repository/bootstrap-routing.md"]); - expect(state.actionContext.allowedWritePaths).toEqual( - [ - "docs/truthmark/engineering/repository/bootstrap-routing.md", - "docs/truthmark/routes/areas.md", - "docs/truthmark/routes/areas/repository.md", - "docs/truthmark/routes/areas/unrelated.md", - ], - ); + expect(state.targetTruthDocs).toEqual([ + "docs/truthmark/engineering/repository/bootstrap-routing.md", + ]); + expect(state.actionContext.allowedWritePaths).toEqual([ + "docs/truthmark/engineering/repository/bootstrap-routing.md", + "docs/truthmark/routes/areas.md", + "docs/truthmark/routes/areas/repository.md", + "docs/truthmark/routes/areas/unrelated.md", + ]); expect(state.actionContext.candidateStaleTruthDocs).toEqual([]); }); @@ -311,7 +356,9 @@ describe("buildWorkflowState", () => { base: "main", // Legacy callers may still pass this at runtime; WorkflowState must ignore it. includeContextPack: true, - } as Parameters[1] & { includeContextPack: true }); + } as Parameters[1] & { + includeContextPack: true; + }); expect("contextPack" in state).toBe(false); expect("sourceFiles" in state).toBe(false); @@ -323,10 +370,14 @@ describe("buildWorkflowState", () => { const repo = await setupConfiguredRepo(); repos.push(repo); - const state = await buildWorkflowState(repo.rootDir, { workflow: "truthmark-sync" }); + const state = await buildWorkflowState(repo.rootDir, { + workflow: "truthmark-sync", + }); expect(state.applicability.state).not.toBe("blocked"); - expect(state.changedFiles).toContainEqual(expect.objectContaining({ path: "src/math.ts" })); + expect(state.changedFiles).toContainEqual( + expect.objectContaining({ path: "src/math.ts" }), + ); expect(state.targetTruthDocs.length).toBeGreaterThan(0); expect(state.nextSteps.join("\n")).not.toContain("--base"); }); @@ -336,10 +387,14 @@ describe("buildWorkflowState", () => { repos.push(repo); await repo.runGit(["branch", "-m", "feature/no-base"]); - const state = await buildWorkflowState(repo.rootDir, { workflow: "truthmark-sync" }); + const state = await buildWorkflowState(repo.rootDir, { + workflow: "truthmark-sync", + }); expect(state.applicability.state).toBe("needs_manual_review"); - expect(state.applicability.reasons.join("\n")).toContain("Choose a comparison base with --base "); + expect(state.applicability.reasons.join("\n")).toContain( + "Choose a comparison base with --base ", + ); expect(state.actionContext.allowedWritePaths).toEqual([]); expect(state.changedFiles).toEqual([]); expect(state.targetTruthDocs).toEqual([]); @@ -351,10 +406,16 @@ describe("buildWorkflowState", () => { repos.push(repo); await repo.writeFile("src/index.ts", "export const value = 1;\n"); - const state = await buildWorkflowState(repo.rootDir, { workflow: "truthmark-sync" }); + const state = await buildWorkflowState(repo.rootDir, { + workflow: "truthmark-sync", + }); - expect(["needs_manual_review", "not_applicable"]).toContain(state.applicability.state); - expect(state.applicability.reasons.join("\n")).toContain("Missing .truthmark/config.yml"); + expect(["needs_manual_review", "not_applicable"]).toContain( + state.applicability.state, + ); + expect(state.applicability.reasons.join("\n")).toContain( + "Missing .truthmark/config.yml", + ); expect(state.actionContext.allowedWritePaths).toEqual([]); }); @@ -368,7 +429,10 @@ describe("buildWorkflowState", () => { "docs/truthmark/routes/areas.md", "# Truthmark Areas\n\n## Docs Only\n\nTruth documents:\n- docs/truthmark/engineering/docs-only.md\n\nCode surface:\n- docs/**\n\nUpdate truth when:\n- docs change\n", ); - await repo.writeFile("docs/truthmark/engineering/docs-only.md", "# Docs Only\n"); + await repo.writeFile( + "docs/truthmark/engineering/docs-only.md", + "# Docs Only\n", + ); await repo.runGit(["add", "."]); await repo.runGit(["commit", "-m", "initial"]); await repo.writeFile("src/unmapped.ts", "export const value = 2;\n"); @@ -378,7 +442,9 @@ describe("buildWorkflowState", () => { base: "main", }); - expect(["needs_routing_review", "needs_manual_review"]).toContain(state.applicability.state); + expect(["needs_routing_review", "needs_manual_review"]).toContain( + state.applicability.state, + ); expect(state.targetTruthDocs).toEqual([]); expect(state.nextSteps.join("\n")).toMatch(/Truth Structure|route repair/u); }); @@ -392,7 +458,6 @@ describe("buildWorkflowState", () => { ).rejects.toThrow(/Unknown Truthmark workflow/u); }); - it("does not mutate truth, route, or generated files", async () => { const repo = await setupConfiguredRepo(); repos.push(repo); diff --git a/vitest.config.ts b/vitest.config.ts deleted file mode 100644 index 3ae90fb..0000000 --- a/vitest.config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - environment: "node", - fileParallelism: false, - include: ["tests/**/*.test.ts"], - globals: true, - testTimeout: 30_000, - }, -}); diff --git a/workflow-eval-framwork/README.md b/workflow-eval-framwork/README.md new file mode 100644 index 0000000..55162fd --- /dev/null +++ b/workflow-eval-framwork/README.md @@ -0,0 +1,88 @@ +# Workflow Eval Framwork + +`workflow-eval-framwork` is Truthmark's contributor-only, manual, token-expensive behavioral eval framework for installed workflow skills and prompts. + +It is not part of the published npm package and is not needed by normal Truthmark users. + +The folder name is intentionally spelled `framwork` because that is the requested canonical path. + +## One-command manual run + +Run the whole catalog with one explicit command when checking workflow quality: + +```bash +node scripts/workflow-eval-framwork/run-agent-scenario.mjs \ + --catalog workflow-eval-framwork/catalog.yaml \ + --host codex \ + --agent-command "codex exec --full-auto" \ + --agent-model "gpt-5.5" \ + --judge-command "your-llm-judge-command" \ + --judge-model "gpt-5.5" \ + --require-usage +``` + +Catalog runs create a suite folder with exactly two persistent files: + +- `final-report.md` — one comprehensive human-facing report for the full evaluation run. +- `audit.json` — compact machine-readable audit data with run metadata, scenario verdicts, deterministic failures, changed-file summaries, and judge summaries. + +Per-scenario command output, patches, reports, and judge scratch files are temporary in catalog mode and are discarded after the two suite files are written. Scenario setup patches are still applied after the fixture baseline commit so agents evaluate a real pre-existing diff rather than an empty checkout. + +Scenarios should model realistic development situations rather than one-file toy repos. Fixtures include source, tests, product truth, engineering behavior truth, API contract truth, operations truth, and route ownership so agents must choose the correct workflow lane and bounded owner. + +A catalog run without `--judge-command` is intentionally `not_evaluable`. It can smoke-test fixture materialization and deterministic gates, but it is not a meaningful workflow-quality result. When `--agent-model` or `--judge-model` is supplied, the runner records those model names in `audit.json`, `final-report.md`, and the `TRUTHMARK_EVAL_AGENT_MODEL` / `TRUTHMARK_EVAL_JUDGE_MODEL` environment variables for wrapper scripts. + +## Token usage tracking + +The runner exports usage sidecar paths for wrappers: + +- `TRUTHMARK_EVAL_AGENT_USAGE` — JSON path for the agent command to write token usage. +- `TRUTHMARK_EVAL_JUDGE_USAGE` — JSON path for the judge command to write token usage. + +Each sidecar may contain one usage object or an array of usage objects: + +```json +{ + "schemaVersion": 1, + "source": "codex-jsonl-wrapper", + "model": "gpt-5.5", + "inputTokens": 1000, + "cachedInputTokens": 200, + "outputTokens": 300, + "reasoningOutputTokens": 50, + "totalTokens": 1300 +} +``` + +Catalog `audit.json`, debug `result.json`, and `final-report.md` record measured agent tokens, judge tokens, and suite totals. The framework intentionally does not enforce token budgets or estimate dollar cost; token counts are a trend signal for comparing prompt and workflow-surface changes over time. + +Missing sidecars are recorded as `usage.status: "unavailable"`; invalid sidecars are recorded as `usage.status: "invalid"` with warnings. + +Use `--require-usage` for real model-comparison runs. In required-usage mode, missing or invalid usage data makes the run non-passing so usage regressions are not silently untracked. + +Run one scenario only when debugging a specific failure, and mark it debug-only: + +```bash +node scripts/workflow-eval-framwork/run-agent-scenario.mjs \ + --debug-scenario \ + --scenario workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml \ + --host fake \ + --agent-command "node workflow-eval-framwork/fake-agents/pass.mjs" \ + --agent-model "fake-agent" \ + --judge-command "node workflow-eval-framwork/fake-agents/judge-pass.mjs" \ + --judge-model "fake-judge" +``` + +By default, catalog artifacts are written under: + +```text +workflow-eval-framwork/runs/-all-workflows-/ +``` + +Single-scenario debug runs still write detailed per-scenario files under the selected debug output directory because they are for harness repair and failure investigation, not normal full-evaluation reporting. + +Use `failure-taxonomy.md` and `improvement-loop.md` when converting a failed run into a prompt/skill fix or a smaller regression scenario. + +Real model runners and LLM judges are manual. Normal `truthmark check`, package install, downstream repository use, and default CI must not launch real agents or judges. + +Deterministic gates catch objective failures first, but workflow quality claims require LLM judge output plus human review. Without semantic judge results, a run proves only harness mechanics and write-boundary behavior, and the runner records it as `not_evaluable`. diff --git a/workflow-eval-framwork/catalog.yaml b/workflow-eval-framwork/catalog.yaml new file mode 100644 index 0000000..0809af9 --- /dev/null +++ b/workflow-eval-framwork/catalog.yaml @@ -0,0 +1,59 @@ +version: 1 +manual_only: true +last_reviewed: 2026-06-29 +workflows: + - id: truthmark-sync + priority: critical + manual_only: true + surface_paths: + - src/agents/truth-sync.ts + - src/templates/workflow-surfaces.ts + rubric: workflow-eval-framwork/rubrics/truthmark-sync.yaml + scenarios: + - workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml + - workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml + - workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml + - id: truthmark-document + priority: high + manual_only: true + surface_paths: + - src/agents/truth-document.ts + rubric: workflow-eval-framwork/rubrics/truthmark-document.yaml + scenarios: + - workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml + - id: truthmark-realize + priority: high + manual_only: true + surface_paths: + - src/agents/prompts.ts + - src/realize/report.ts + rubric: workflow-eval-framwork/rubrics/truthmark-realize.yaml + scenarios: + - workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml + - id: truthmark-check + priority: high + manual_only: true + surface_paths: + - src/agents/truth-check.ts + rubric: workflow-eval-framwork/rubrics/truthmark-check.yaml + scenarios: + - workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml + - workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml + - id: truthmark-structure + priority: medium + manual_only: true + surface_paths: + - src/agents/truth-structure.ts + rubric: workflow-eval-framwork/rubrics/truthmark-structure.yaml + scenarios: + - workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml +runners: + harness_hosts: + - fake + manual_agent_hosts: + - codex + - claude-code +budgets: + max_agent_runs_per_scenario: 1 + max_judge_calls_per_scenario: 3 + default_timeout_seconds: 900 diff --git a/workflow-eval-framwork/examples/human-review-accepted.yaml b/workflow-eval-framwork/examples/human-review-accepted.yaml new file mode 100644 index 0000000..f69f284 --- /dev/null +++ b/workflow-eval-framwork/examples/human-review-accepted.yaml @@ -0,0 +1,7 @@ +status: accepted +reviewer: Merlin +reviewed_at: 2026-06-29 +notes: + - Deterministic gates passed. + - Semantic behavior matched the scenario expectation. +follow_up: [] diff --git a/workflow-eval-framwork/examples/human-review-rejected.yaml b/workflow-eval-framwork/examples/human-review-rejected.yaml new file mode 100644 index 0000000..acb2293 --- /dev/null +++ b/workflow-eval-framwork/examples/human-review-rejected.yaml @@ -0,0 +1,9 @@ +status: rejected +reviewer: Merlin +reviewed_at: 2026-06-29 +notes: + - Agent selected the wrong Truthmark workflow. + - Changed files violated the scenario write boundary. +follow_up: + - Tighten the workflow skill trigger description. + - Promote the failure into a smaller regression scenario. diff --git a/workflow-eval-framwork/failure-taxonomy.md b/workflow-eval-framwork/failure-taxonomy.md new file mode 100644 index 0000000..1c98718 --- /dev/null +++ b/workflow-eval-framwork/failure-taxonomy.md @@ -0,0 +1,20 @@ +# Failure Taxonomy + +Use stable labels when reviewing workflow eval failures. + +- `skill-not-triggered`: the expected workflow skill or prompt was not used. +- `wrong-workflow`: the agent selected a different Truthmark workflow. +- `over-triggered-workflow`: the agent ran a write workflow for a read-only or general task. +- `missing-route-read`: the agent did not inspect required route/config evidence. +- `wrong-route-owner`: the agent updated or cited the wrong bounded truth owner. +- `bootstrap-route-misuse`: the agent treated a bootstrap catch-all as behavior truth. +- `forbidden-code-write`: a documentation workflow changed functional code. +- `forbidden-doc-write`: a code-realization workflow changed truth docs or routes. +- `missing-evidence`: a changed truth claim lacks checkout evidence. +- `unsupported-claim`: the result preserves or adds a claim without evidence. +- `stale-truth-followed`: Realize followed stale truth instead of blocking. +- `report-invalid`: the workflow report is absent or fails validation. +- `verification-skipped-without-rationale`: required checks were neither run nor explained. +- `token-bloat`: the agent loaded or repeated unnecessary context. +- `agent-runner-failed`: the configured runner command failed before behavior could be evaluated. +- `judge-not-evaluable`: judge output was missing, malformed, or insufficient. diff --git a/workflow-eval-framwork/fake-agents/forbidden-write.mjs b/workflow-eval-framwork/fake-agents/forbidden-write.mjs new file mode 100644 index 0000000..1cbe88a --- /dev/null +++ b/workflow-eval-framwork/fake-agents/forbidden-write.mjs @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import { appendFileSync, writeFileSync } from "node:fs"; + +appendFileSync("README.md", "\nForbidden write from fake agent.\n", "utf8"); +if (process.env.TRUTHMARK_EVAL_REPORT) { + writeFileSync(process.env.TRUTHMARK_EVAL_REPORT, "# Fake Agent Report\n\nResult: completed with forbidden write\n", "utf8"); +} +console.log("fake agent made forbidden write"); diff --git a/workflow-eval-framwork/fake-agents/invalid-report.mjs b/workflow-eval-framwork/fake-agents/invalid-report.mjs new file mode 100644 index 0000000..18aa806 --- /dev/null +++ b/workflow-eval-framwork/fake-agents/invalid-report.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { writeFileSync } from "node:fs"; + +if (process.env.TRUTHMARK_EVAL_REPORT) { + writeFileSync(process.env.TRUTHMARK_EVAL_REPORT, "invalid", "utf8"); +} +console.log("fake agent wrote invalid report"); diff --git a/workflow-eval-framwork/fake-agents/judge-fail.mjs b/workflow-eval-framwork/fake-agents/judge-fail.mjs new file mode 100644 index 0000000..612f1c8 --- /dev/null +++ b/workflow-eval-framwork/fake-agents/judge-fail.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +const result = { + status: "failed", + judges: [ + { + id: "fake-llm-judge", + status: "failed", + summary: "Fake judge failure for harness status-mapping tests.", + }, + ], +}; +console.log(JSON.stringify(result)); diff --git a/workflow-eval-framwork/fake-agents/judge-pass.mjs b/workflow-eval-framwork/fake-agents/judge-pass.mjs new file mode 100644 index 0000000..afb8b7e --- /dev/null +++ b/workflow-eval-framwork/fake-agents/judge-pass.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { writeFileSync } from "node:fs"; + +const judgeInputTokens = Number(process.env.TRUTHMARK_EVAL_FAKE_JUDGE_INPUT_TOKENS ?? 500); + +if (process.env.TRUTHMARK_EVAL_JUDGE_USAGE) { + writeFileSync( + process.env.TRUTHMARK_EVAL_JUDGE_USAGE, + `${JSON.stringify({ + schemaVersion: 1, + source: "fake-judge", + model: process.env.TRUTHMARK_EVAL_JUDGE_MODEL || "fake-judge", + inputTokens: judgeInputTokens, + cachedInputTokens: 50, + outputTokens: 125, + reasoningOutputTokens: 25, + totalTokens: judgeInputTokens + 150, + }, null, 2)}\n`, + "utf8", + ); +} + +const result = { + status: "passed", + judges: [ + { + id: "fake-llm-judge", + status: "passed", + summary: "Fake judge for harness tests only; real workflow-quality claims require a maintainer-configured LLM judge.", + }, + ], +}; +console.log(JSON.stringify(result)); diff --git a/workflow-eval-framwork/fake-agents/no-usage-pass.mjs b/workflow-eval-framwork/fake-agents/no-usage-pass.mjs new file mode 100644 index 0000000..7340dd5 --- /dev/null +++ b/workflow-eval-framwork/fake-agents/no-usage-pass.mjs @@ -0,0 +1,35 @@ +#!/usr/bin/env node +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +import { parse as parseYaml } from "yaml"; + +const expectedPath = process.env.TRUTHMARK_EVAL_EXPECTED; +const expected = expectedPath ? parseYaml(readFileSync(expectedPath, "utf8")) : {}; +const mustChange = Array.isArray(expected.must_change) ? expected.must_change : []; +const mustRead = Array.isArray(expected.must_read) ? expected.must_read : []; + +for (const file of mustChange) { + if (typeof file !== "string" || file.includes("*")) { + continue; + } + const target = path.resolve(process.cwd(), file); + if (!target.startsWith(`${process.cwd()}${path.sep}`)) { + continue; + } + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, `fake-agent touched ${file}\n`, "utf8"); +} + +if (process.env.TRUTHMARK_EVAL_REPORT) { + writeFileSync( + process.env.TRUTHMARK_EVAL_REPORT, + `# Fake Agent Report\n\nResult: completed\nWorkflow: ${process.env.TRUTHMARK_EVAL_WORKFLOW ?? "unknown"}\n`, + "utf8", + ); +} + +for (const file of mustRead) { + console.log(`fake agent read ${file}`); +} +console.log("fake agent completed without usage sidecar for required-usage tests"); diff --git a/workflow-eval-framwork/fake-agents/pass.mjs b/workflow-eval-framwork/fake-agents/pass.mjs new file mode 100644 index 0000000..905af36 --- /dev/null +++ b/workflow-eval-framwork/fake-agents/pass.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +import { parse as parseYaml } from "yaml"; + +const expectedPath = process.env.TRUTHMARK_EVAL_EXPECTED; +const expected = expectedPath ? parseYaml(readFileSync(expectedPath, "utf8")) : {}; +const mustChange = Array.isArray(expected.must_change) ? expected.must_change : []; +const mustRead = Array.isArray(expected.must_read) ? expected.must_read : []; + +for (const file of mustChange) { + if (typeof file !== "string" || file.includes("*")) { + continue; + } + const target = path.resolve(process.cwd(), file); + if (!target.startsWith(`${process.cwd()}${path.sep}`)) { + continue; + } + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, `fake-agent touched ${file}\n`, "utf8"); +} + +if (process.env.TRUTHMARK_EVAL_REPORT) { + writeFileSync( + process.env.TRUTHMARK_EVAL_REPORT, + `# Fake Agent Report\n\nResult: completed\nWorkflow: ${process.env.TRUTHMARK_EVAL_WORKFLOW ?? "unknown"}\n`, + "utf8", + ); +} + +const agentInputTokens = Number(process.env.TRUTHMARK_EVAL_FAKE_AGENT_INPUT_TOKENS ?? 1000); + +if (process.env.TRUTHMARK_EVAL_AGENT_USAGE) { + writeFileSync( + process.env.TRUTHMARK_EVAL_AGENT_USAGE, + `${JSON.stringify({ + schemaVersion: 1, + source: "fake-agent", + model: process.env.TRUTHMARK_EVAL_AGENT_MODEL || "fake-agent", + inputTokens: agentInputTokens, + cachedInputTokens: 100, + outputTokens: 250, + reasoningOutputTokens: 50, + totalTokens: agentInputTokens + 300, + }, null, 2)}\n`, + "utf8", + ); +} + +for (const file of mustRead) { + console.log(`fake agent read ${file}`); +} +console.log("fake agent completed; skipped npx tsx src/cli/main.ts check --json because this is a fake-agent smoke test"); diff --git a/workflow-eval-framwork/improvement-loop.md b/workflow-eval-framwork/improvement-loop.md new file mode 100644 index 0000000..9075725 --- /dev/null +++ b/workflow-eval-framwork/improvement-loop.md @@ -0,0 +1,17 @@ +# Improvement Loop + +When a workflow eval fails: + +1. For catalog runs, read `final-report.md` first and use `audit.json` for scenario verdicts, deterministic failures, changed-file summaries, and judge summaries. For single-scenario debug runs, read the detailed `report.md`, `result.json`, deterministic results, command output, changed files, patches, and judge results. +2. Record a human review as `accepted`, `rejected`, `needs-rerun`, or `not-evaluable`. +3. Classify failures with labels from `failure-taxonomy.md`. +4. Decide the smallest fix target: + - skill or prompt text when the agent misunderstood workflow behavior; + - rubric text when the evaluation expectation was underspecified; + - scenario fixture when the setup did not isolate the intended failure; + - deterministic validator when an objective violation was missed; + - judge prompt/schema when semantic scoring was malformed. +5. Promote real failures into minimal scenarios instead of checking in large run artifacts. +6. Rerun the scenario with one manual command and compare the new report folder. + +LLM judge scores are advisory until calibrated by human review. Deterministic gates cannot be overridden by a judge score. diff --git a/workflow-eval-framwork/judges/diff-grader.md b/workflow-eval-framwork/judges/diff-grader.md new file mode 100644 index 0000000..5ad0e9f --- /dev/null +++ b/workflow-eval-framwork/judges/diff-grader.md @@ -0,0 +1,4 @@ +# diff-grader + +Return strict JSON matching `workflow-eval-framwork/schemas/judge-results.schema.json`. +Grade only the provided artifacts. Do not assume repository state that is not included. diff --git a/workflow-eval-framwork/judges/report-grader.md b/workflow-eval-framwork/judges/report-grader.md new file mode 100644 index 0000000..528eafc --- /dev/null +++ b/workflow-eval-framwork/judges/report-grader.md @@ -0,0 +1,4 @@ +# report-grader + +Return strict JSON matching `workflow-eval-framwork/schemas/judge-results.schema.json`. +Grade only the provided artifacts. Do not assume repository state that is not included. diff --git a/workflow-eval-framwork/judges/trace-grader.md b/workflow-eval-framwork/judges/trace-grader.md new file mode 100644 index 0000000..a99f052 --- /dev/null +++ b/workflow-eval-framwork/judges/trace-grader.md @@ -0,0 +1,4 @@ +# trace-grader + +Return strict JSON matching `workflow-eval-framwork/schemas/judge-results.schema.json`. +Grade only the provided artifacts. Do not assume repository state that is not included. diff --git a/workflow-eval-framwork/judges/triggering-grader.md b/workflow-eval-framwork/judges/triggering-grader.md new file mode 100644 index 0000000..96ed664 --- /dev/null +++ b/workflow-eval-framwork/judges/triggering-grader.md @@ -0,0 +1,4 @@ +# triggering-grader + +Return strict JSON matching `workflow-eval-framwork/schemas/judge-results.schema.json`. +Grade only the provided artifacts. Do not assume repository state that is not included. diff --git a/workflow-eval-framwork/rubrics/shared-write-boundaries.yaml b/workflow-eval-framwork/rubrics/shared-write-boundaries.yaml new file mode 100644 index 0000000..9471a4b --- /dev/null +++ b/workflow-eval-framwork/rubrics/shared-write-boundaries.yaml @@ -0,0 +1,18 @@ +version: 1 +manual_only: true +deterministic_gates: + - fixture-setup + - forbidden-writes + - missing-report + - invalid-report + - failed-required-validator + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/rubrics/truthmark-check.yaml b/workflow-eval-framwork/rubrics/truthmark-check.yaml new file mode 100644 index 0000000..5855b59 --- /dev/null +++ b/workflow-eval-framwork/rubrics/truthmark-check.yaml @@ -0,0 +1,17 @@ +version: 1 +manual_only: true +deterministic_gates: + - read-only-audit + - evidence-backed-findings + - correct-next-workflow + - no-silent-rewrites + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/rubrics/truthmark-document.yaml b/workflow-eval-framwork/rubrics/truthmark-document.yaml new file mode 100644 index 0000000..f4316ff --- /dev/null +++ b/workflow-eval-framwork/rubrics/truthmark-document.yaml @@ -0,0 +1,18 @@ +version: 1 +manual_only: true +deterministic_gates: + - document-existing-behavior + - no-functional-code-writes + - bounded-canonical-doc + - route-only-when-needed + - preserve-decisions-rationale + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/rubrics/truthmark-realize.yaml b/workflow-eval-framwork/rubrics/truthmark-realize.yaml new file mode 100644 index 0000000..bccff2e --- /dev/null +++ b/workflow-eval-framwork/rubrics/truthmark-realize.yaml @@ -0,0 +1,18 @@ +version: 1 +manual_only: true +deterministic_gates: + - truth-as-requirements + - code-only-writes + - stale-truth-blocking + - test-verification + - no-truth-doc-edits + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/rubrics/truthmark-structure.yaml b/workflow-eval-framwork/rubrics/truthmark-structure.yaml new file mode 100644 index 0000000..36ac22f --- /dev/null +++ b/workflow-eval-framwork/rubrics/truthmark-structure.yaml @@ -0,0 +1,17 @@ +version: 1 +manual_only: true +deterministic_gates: + - detect-broad-routes + - bounded-ownership-proposals + - lane-separation + - preserve-decisions-rationale + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/rubrics/truthmark-sync.yaml b/workflow-eval-framwork/rubrics/truthmark-sync.yaml new file mode 100644 index 0000000..8272ca5 --- /dev/null +++ b/workflow-eval-framwork/rubrics/truthmark-sync.yaml @@ -0,0 +1,19 @@ +version: 1 +manual_only: true +deterministic_gates: + - skip-non-code-changes + - bounded-route-owner + - no-functional-code-writes + - structure-handoff + - evidence-backed-claims + - valid-sync-report + +semantic_dimensions: + - triggering + - context-selection + - routing-judgment + - write-boundary + - evidence-discipline + - report-quality + - human-review-quality + - token-discipline diff --git a/workflow-eval-framwork/runs/.gitkeep b/workflow-eval-framwork/runs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/expected.yaml new file mode 100644 index 0000000..6ac15cf --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/expected.yaml @@ -0,0 +1,26 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - docs/truthmark/product/capabilities/widget-cache.md + +must_change: + [] + +must_not_change: + - README.md + - src/** + - tests/** + - docs/truthmark/** + +must_run_or_explain: + [] + +report: + required: false + validator: none + +assertions: + + - id: read-only-question + kind: diff + pass: general questions do not trigger write workflows diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/prompt.md b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/prompt.md new file mode 100644 index 0000000..100cd3f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/prompt.md @@ -0,0 +1,10 @@ +Use the installed Truthmark Check workflow behavior for this scenario. + +Development situation: +- A user asks a general question about widget cache behavior in a repository with realistic code, tests, routes, product truth, engineering truth, and operations truth. +- The question is informational and should not trigger Document, Sync, Realize, or Structure writes. + +Acceptance signal: +- Answer or report read-only findings. +- Do not edit any checkout files. +- Distinguish read-only Check behavior from write-capable workflows. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml new file mode 100644 index 0000000..128bc42 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml @@ -0,0 +1,21 @@ +id: general-question-no-write +workflow: truthmark-check +priority: medium +manual_only: true +summary: General question should not trigger write workflows. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/expected.yaml new file mode 100644 index 0000000..12d916f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/expected.yaml @@ -0,0 +1,27 @@ +must_read: + - .truthmark/config.yml + - src/cache.ts + - tests/widget-cache.test.ts + - docs/truthmark/engineering/behaviors/widget-cache.md + +must_change: + [] + +must_not_change: + - README.md + - src/** + - tests/** + - docs/truthmark/** + +must_run_or_explain: + [] + +report: + required: false + validator: none + +assertions: + + - id: read-only-audit + kind: diff + pass: Truth Check does not edit files while auditing claim support diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/prompt.md b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/prompt.md new file mode 100644 index 0000000..df07ede --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/prompt.md @@ -0,0 +1,11 @@ +Use the installed Truthmark Check workflow behavior for this scenario. + +Development situation: +- The engineering behavior truth claims a cache TTL, product truth claims stable response shape, and code/tests provide the implementation evidence. +- The task is an audit request, not a request to repair anything. +- Multiple truth lanes are present, so the checker must cite mismatches without writing files. + +Acceptance signal: +- Produce a read-only audit of unsupported or supported claims. +- Do not edit README, source, tests, routes, or truth docs. +- Keep the report scoped to evidence from the checkout. diff --git a/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml new file mode 100644 index 0000000..2545000 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml @@ -0,0 +1,21 @@ +id: unsupported-claim-audit +workflow: truthmark-check +priority: high +manual_only: true +summary: Truth Check should audit unsupported claims read-only. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/expected.yaml new file mode 100644 index 0000000..b0289e4 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/expected.yaml @@ -0,0 +1,28 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - src/cache.ts + - docs/truthmark/engineering/contracts/widget-api.md + +must_change: + - docs/truthmark/engineering/behaviors/widget-cache.md + +must_not_change: + - src/** + - tests/** + - docs/truthmark/product/** + - docs/truthmark/engineering/contracts/** + - docs/truthmark/engineering/operations/** + +must_run_or_explain: + - npx tsx src/cli/main.ts check --json + +report: + required: true + validator: none + +assertions: + + - id: document-current-behavior + kind: outcome + pass: the missing behavior truth is created from source evidence without code edits diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/prompt.md b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/prompt.md new file mode 100644 index 0000000..ef1e2ed --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/prompt.md @@ -0,0 +1,12 @@ +Use the installed Truthmark Document workflow behavior for this scenario. + +Development situation: +- The TypeScript widget cache behavior exists in code and tests. +- Product, API contract, and operations truth are present, but the bounded engineering behavior truth file is missing. +- The correct work is documentation of implemented behavior, not code edits. + +Acceptance signal: +- Create the missing engineering behavior truth doc for widget cache behavior. +- Preserve source files and existing product/contract/operations docs. +- Include source-backed current-state claims, not future requirements. +- Run `npx tsx src/cli/main.ts check --json`, or explicitly explain why that exact repository verification command was skipped for this docs-only fixture. diff --git a/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml new file mode 100644 index 0000000..eabe950 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml @@ -0,0 +1,21 @@ +id: implemented-behavior-missing-truth +workflow: truthmark-document +priority: high +manual_only: true +summary: Implemented behavior missing canonical truth should be documented. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/expected.yaml new file mode 100644 index 0000000..633d257 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/expected.yaml @@ -0,0 +1,25 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/engineering/behaviors/widget-cache.md + - docs/truthmark/product/capabilities/widget-cache.md + - src/cache.ts + +must_change: + - src/cache.ts + - tests/widget-cache.test.ts + +must_not_change: + - docs/truthmark/** + +must_run_or_explain: + [] + +report: + required: true + validator: none + +assertions: + + - id: code-only-realize + kind: diff + pass: Realize implements code while preserving truth docs diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..8e73f3e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 120 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 75 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/prompt.md b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/prompt.md new file mode 100644 index 0000000..0588c48 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/prompt.md @@ -0,0 +1,12 @@ +Use the installed Truthmark Realize workflow behavior for this scenario. + +Development situation: +- Product truth says the public response shape must stay stable. +- Engineering behavior truth asks for a 120 second cache freshness window. +- API contract and tests are present, so the implementation should be a bounded cache-owner code/test change, not a truth-doc rewrite. + +Acceptance signal: +- Implement the behavior in `src/cache.ts` and update the cache behavior test in `tests/widget-cache.test.ts`. +- Preserve the public response shape and avoid artificial changes to unrelated source surfaces. +- Do not edit any `docs/truthmark/**` truth or route files. +- Report the implementation evidence and any checks run or skipped. diff --git a/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml new file mode 100644 index 0000000..9f4dd4a --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml @@ -0,0 +1,21 @@ +id: implement-truth-without-doc-edits +workflow: truthmark-realize +priority: high +manual_only: true +summary: Truth Realize should change code without editing truth docs. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/expected.yaml new file mode 100644 index 0000000..e8629c9 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/expected.yaml @@ -0,0 +1,33 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - docs/truthmark/routes/areas/fixture.md + - docs/truthmark/engineering/contracts/widget-api.md + +must_change: + - docs/truthmark/routes/areas/widget-cache.md + +must_not_change: + - src/** + - tests/** + - docs/truthmark/engineering/behaviors/widget-cache.md + - docs/truthmark/product/** + - docs/truthmark/engineering/contracts/** + - docs/truthmark/engineering/operations/** + +must_run_or_explain: + - npx tsx src/cli/main.ts check --json + +report: + required: true + validator: none + +assertions: + + - id: bounded-route-owner + kind: outcome + pass: a broad catch-all route is narrowed to bounded widget-cache ownership + + - id: no-functional-write + kind: diff + pass: Truth Structure does not change functional code or truth docs diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..006d3e7 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,16 @@ +# Fixture broad route + +## Catch-all repository ownership + +Truth documents: +- docs/truthmark/product/capabilities/widget-cache.md +- docs/truthmark/engineering/behaviors/widget-cache.md +- docs/truthmark/engineering/contracts/widget-api.md +- docs/truthmark/engineering/operations/widget-observability.md + +Code surface: +- src/** +- tests/** + +Update truth when: +- Any source, test, product, contract, or operations file changes. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/prompt.md b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/prompt.md new file mode 100644 index 0000000..a0fa99c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/prompt.md @@ -0,0 +1,11 @@ +The fixture has a broad catch-all route covering a widget cache service with code, tests, product truth, behavior truth, API contract truth, and operations truth. + +Development situation: +- Route ownership is too broad for safe future Sync or Document writes. +- The behavior owner should be narrowed without changing functional code or rewriting behavior truth. +- Nearby product, contract, and operations docs should remain available but not be used as a catch-all owner. + +Acceptance signal: +- Run Truth Structure to create or propose bounded ownership for widget cache behavior. +- Keep source files and existing truth docs unchanged. +- Run or specifically explain `npx tsx src/cli/main.ts check --json`. diff --git a/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml new file mode 100644 index 0000000..2dbfe96 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml @@ -0,0 +1,21 @@ +id: broad-route-repair +workflow: truthmark-structure +priority: high +manual_only: true +summary: Truth Structure should split a broad catch-all route into bounded ownership before other workflows write truth. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/expected.yaml new file mode 100644 index 0000000..cd232a1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/expected.yaml @@ -0,0 +1,29 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - docs/truthmark/routes/areas/fixture.md + +must_change: + [] + +must_not_change: + - docs/truthmark/engineering/bootstrap-routing.md + - src/** + - tests/** + +must_run_or_explain: + - npx tsx src/cli/main.ts check --json + +report: + required: true + validator: none + +assertions: + + - id: bootstrap-handoff + kind: outcome + pass: agent blocks write-capable Sync and recommends bounded Truth Structure repair + + - id: no-functional-write + kind: diff + pass: agent does not change source or tests diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/bootstrap-routing.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/bootstrap-routing.md new file mode 100644 index 0000000..babe53c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/bootstrap-routing.md @@ -0,0 +1,11 @@ +--- +status: active +truth_kind: engineering-workflow +last_reviewed: 2026-06-29 +--- + +# Bootstrap Routing + +## Current Implementation Behavior + +- This provisional document exists only to block unsafe write-capable truth updates until Structure narrows ownership. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..6c66671 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,13 @@ +# Bootstrap Fixture Area + +This area is intentionally provisional and too broad for write-capable Sync. + +Truth documents: +- docs/truthmark/engineering/bootstrap-routing.md + +Code surface: +- src/** +- tests/** + +Open topology question: +- Split widget cache behavior, API contract, product capability, and operations ownership before Sync writes truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/prompt.md b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/prompt.md new file mode 100644 index 0000000..51de33d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/prompt.md @@ -0,0 +1,12 @@ +Use the installed Truthmark Sync workflow behavior for this scenario. + +Development situation: +- A widget cache source change is present, but route ownership is still too broad/bootstrap-like for safe truth writes. +- Product, behavior, contract, operations, and test files are all nearby and plausible. +- The right outcome is to stop and hand off to Truth Structure instead of guessing a truth owner. + +Acceptance signal: +- Report the routing/topology blocker. +- Do not edit source files or behavior truth. +- Recommend bounded Structure repair before Sync writes truth. +- Run `npx tsx src/cli/main.ts check --json`, or explicitly explain why that exact repository check command was skipped after the blocked decision. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml new file mode 100644 index 0000000..b8ee6d1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml @@ -0,0 +1,21 @@ +id: bootstrap-route-blocked +workflow: truthmark-sync +priority: critical +manual_only: true +summary: Bootstrap route ownership should block and recommend Structure. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/expected.yaml new file mode 100644 index 0000000..9345bbe --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/expected.yaml @@ -0,0 +1,24 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - README.md + +must_change: + [] + +must_not_change: + - src/** + - docs/truthmark/engineering/** + +must_run_or_explain: + - npx tsx src/cli/main.ts check --json + +report: + required: false + validator: none + +assertions: + + - id: write-boundary + kind: diff + pass: changed files stay inside the workflow boundary diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/AGENTS.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/AGENTS.md new file mode 100644 index 0000000..c5082fd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/AGENTS.md @@ -0,0 +1,13 @@ +# Fixture Agent Instructions + +Use the installed Truthmark workflow behavior for this eval fixture. + +## Truthmark Workflow + +Hierarchy hints: config `.truthmark/config.yml` when present; routes `docs/truthmark/routes/areas.md` and `docs/truthmark/routes/areas/**/*.md` when present; Truth docs under `docs/truthmark/**` when present. + +### Truth Sync + +After functional code changes, run relevant tests, then use the Truth Sync review before finishing. Skip Sync for docs-only/no-code changes, formatting-only changes, behavior-preserving renames with no truth impact, or missing config. When skipping, report the reason and the verification command you ran or explicitly explain why it was skipped. + +Truth Sync may write truth docs and truth routing files when functional behavior changed, and must not rewrite functional code. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/prompt.md b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/prompt.md new file mode 100644 index 0000000..f3c3987 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/prompt.md @@ -0,0 +1,12 @@ +Use the installed Truthmark Sync workflow behavior for this scenario. + +Development situation: +- A maintainer only changed README release-note prose. +- The checkout also contains a realistic widget cache service, product truth, engineering behavior truth, API contract truth, operations truth, and tests. +- There is no functional source diff for the agent to reconcile. + +Acceptance signal: +- Identify this as docs-only/no-code work. +- Do not edit source or canonical truth docs. +- If reporting, explain why no write workflow should launch. +- Run `npx tsx src/cli/main.ts check --json`, or explicitly explain why that exact repository check command was skipped for docs-only/no-code work. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml new file mode 100644 index 0000000..c76e0eb --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml @@ -0,0 +1,23 @@ +id: docs-only-skip +workflow: truthmark-sync +priority: critical +manual_only: true +summary: Docs-only change should not launch a write workflow. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +setup: + patch: setup.patch +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/setup.patch b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/setup.patch new file mode 100644 index 0000000..7360280 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/setup.patch @@ -0,0 +1,5 @@ +--- a/README.md ++++ b/README.md +@@ -5,0 +6,2 @@ ++ ++Docs-only release note: clarify that cache tuning work shipped without a public API change. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/expected.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/expected.yaml new file mode 100644 index 0000000..74243f4 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/expected.yaml @@ -0,0 +1,29 @@ +must_read: + - .truthmark/config.yml + - docs/truthmark/routes/areas.md + - src/cache.ts + - docs/truthmark/engineering/behaviors/widget-cache.md + +must_change: + - docs/truthmark/engineering/behaviors/widget-cache.md + +must_not_change: + - src/** + - tests/** + - README.md + - docs/truthmark/product/** + - docs/truthmark/engineering/contracts/** + - docs/truthmark/engineering/operations/** + +must_run_or_explain: + - npx tsx src/cli/main.ts check --json + +report: + required: true + validator: none + +assertions: + + - id: bounded-truth-update + kind: diff + pass: only the routed engineering behavior truth is changed by the agent diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/.truthmark/config.yml b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/.truthmark/config.yml new file mode 100644 index 0000000..c6811bd --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/.truthmark/config.yml @@ -0,0 +1,5 @@ +version: 2 +platforms: + - codex +truthmark: + workspace: docs/truthmark diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/README.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/README.md new file mode 100644 index 0000000..7f9b869 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/README.md @@ -0,0 +1,5 @@ +# Widget Cache Fixture + +This fixture models a small TypeScript service with product truth, engineering behavior truth, API contract truth, operations truth, route ownership, and tests. + +The scenario deliberately includes nearby but non-target documents so workflow agents must choose the right lane and owner instead of editing every plausible file. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/behaviors/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/behaviors/widget-cache.md new file mode 100644 index 0000000..5a2090c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/behaviors/widget-cache.md @@ -0,0 +1,19 @@ +--- +status: active +truth_kind: engineering-behavior +last_reviewed: 2026-06-29 +--- + +# Widget Cache + +## Current Implementation Behavior + +- Widget cache entries use a 60 second freshness window. +- Widget cache entries use a 30 second stale-while-revalidate window. +- Tenant widget reads preserve the `:` API response shape. + +## Behavior Scenarios + +- GIVEN a default cache entry + WHEN its age is 45 seconds + THEN the entry is still treated as fresh. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md new file mode 100644 index 0000000..df4086e --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/contracts/widget-api.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: engineering-contract +last_reviewed: 2026-06-29 +--- + +# Widget API Contract + +## Contract Surface + +- `readWidgetForTenant(tenant)` returns `:`. +- `defaultCacheEntry(key, value)` owns the default freshness and stale-revalidation windows. +- Metrics names remain stable for dashboards and alert rules. + +## Compatibility Notes + +- Internal cache tuning may change `maxAgeSeconds`. +- The API response shape must not change during cache tuning. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md new file mode 100644 index 0000000..2d728a3 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/engineering/operations/widget-observability.md @@ -0,0 +1,16 @@ +--- +status: active +truth_kind: engineering-operations +last_reviewed: 2026-06-29 +--- + +# Widget Observability + +## Operational Signals + +- Cache hit counters use the `widget.cache.hit` metric name. +- Tenant tags identify cache behavior by tenant without logging raw widget values. + +## Review Notes + +- Workflow agents should distinguish operational truth from API contract truth. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md new file mode 100644 index 0000000..25cbb7c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/product/capabilities/widget-cache.md @@ -0,0 +1,18 @@ +--- +status: active +truth_kind: product-capability +last_reviewed: 2026-06-29 +--- + +# Widget Cache Capability + +## Current Product Behavior + +- Widget reads return cached values per tenant. +- Users should not see cache implementation details in response text. +- Operators expect cache behavior changes to remain invisible unless a documented user-facing freshness promise changes. + +## Acceptance Signals + +- Tenant-specific widget reads continue to return the same value shape. +- Cache tuning work should preserve the public response contract. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas.md new file mode 100644 index 0000000..a57c75d --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas.md @@ -0,0 +1,20 @@ +--- +status: active +doc_type: routing +--- + +# Areas + +## Widget Cache Area + +Area files: + +- docs/truthmark/routes/areas/fixture.md + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas/fixture.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas/fixture.md new file mode 100644 index 0000000..d0d7ce1 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/docs/truthmark/routes/areas/fixture.md @@ -0,0 +1,31 @@ +--- +status: active +doc_type: routing +--- + +# Widget Cache Area + +Truth documents: +```yaml +truth_documents: + - path: docs/truthmark/product/capabilities/widget-cache.md + kind: product-capability + lane: product + - path: docs/truthmark/engineering/behaviors/widget-cache.md + kind: engineering-behavior + lane: engineering + - path: docs/truthmark/engineering/contracts/widget-api.md + kind: engineering-contract + lane: engineering + - path: docs/truthmark/engineering/operations/widget-observability.md + kind: engineering-operations + lane: engineering +``` + +Code surface: + +- src/widget.ts +- src/cache.ts +- src/api.ts +- src/metrics.ts +- tests/widget-cache.test.ts diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/package.json b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/package.json new file mode 100644 index 0000000..afa2f0c --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "truthmark-eval-fixture", + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test tests/widget-cache.test.ts", + "check": "node --check src/widget.ts" + }, + "dependencies": {} +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/api.ts b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/api.ts new file mode 100644 index 0000000..719a024 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/api.ts @@ -0,0 +1,8 @@ +import { defaultCacheEntry } from "./cache.ts"; +import { cacheHitMetric } from "./metrics.ts"; + +export function readWidgetForTenant(tenant: string): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + const metric = cacheHitMetric(tenant); + return `${entry.value}:${metric.tags.tenant}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/cache.ts b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/cache.ts new file mode 100644 index 0000000..dd50bc6 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/cache.ts @@ -0,0 +1,19 @@ +export type CacheEntry = { + key: string; + value: string; + maxAgeSeconds: number; + staleWhileRevalidateSeconds: number; +}; + +export function defaultCacheEntry(key: string, value: string): CacheEntry { + return { + key, + value, + maxAgeSeconds: 60, + staleWhileRevalidateSeconds: 30, + }; +} + +export function isFresh(entry: CacheEntry, ageSeconds: number): boolean { + return ageSeconds <= entry.maxAgeSeconds; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/metrics.ts b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/metrics.ts new file mode 100644 index 0000000..1743f51 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/metrics.ts @@ -0,0 +1,13 @@ +export type WidgetMetric = { + name: string; + value: number; + tags: Record; +}; + +export function cacheHitMetric(tenant: string): WidgetMetric { + return { + name: "widget.cache.hit", + value: 1, + tags: { tenant }, + }; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/widget.ts b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/widget.ts new file mode 100644 index 0000000..0b2432f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/src/widget.ts @@ -0,0 +1,10 @@ +import { defaultCacheEntry } from "./cache.ts"; + +export function widgetValue(tenant = "default"): string { + const entry = defaultCacheEntry(`widget:${tenant}`, "old"); + return entry.value; +} + +export function widgetSummary(tenant: string): string { + return `${tenant}:${widgetValue(tenant)}`; +} diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/tests/widget-cache.test.ts b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/tests/widget-cache.test.ts new file mode 100644 index 0000000..177c237 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/fixture/tests/widget-cache.test.ts @@ -0,0 +1,15 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; + +import { defaultCacheEntry, isFresh } from "../src/cache.ts"; +import { readWidgetForTenant } from "../src/api.ts"; + +test("cache entries use the documented default freshness window", () => { + const entry = defaultCacheEntry("widget:acme", "old"); + assert.equal(entry.maxAgeSeconds, 60); + assert.equal(isFresh(entry, 45), true); +}); + +test("tenant reads include the tenant marker", () => { + assert.equal(readWidgetForTenant("acme"), "old:acme"); +}); diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/prompt.md b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/prompt.md new file mode 100644 index 0000000..1206d16 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/prompt.md @@ -0,0 +1,11 @@ +Use the installed Truthmark Sync workflow behavior for this scenario. + +Development situation: +- The fixture starts with a functional cache-tuning diff in `src/cache.ts`. +- The product capability and API contract remain stable, while engineering behavior truth is stale. +- Operations truth is adjacent but not the owner of the cache freshness behavior. + +Acceptance signal: +- Update only the bounded engineering behavior truth for widget cache freshness. +- Do not change source, product capability, API contract, operations truth, or README files. +- Run or specifically explain `npx tsx src/cli/main.ts check --json`. diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml new file mode 100644 index 0000000..6b2002f --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml @@ -0,0 +1,23 @@ +id: stale-engineering-truth +workflow: truthmark-sync +priority: critical +manual_only: true +summary: Functional code change should update bounded engineering truth. +fixture: fixture +prompt: prompt.md +expected: expected.yaml +setup: + patch: setup.patch +requires: + generated_surfaces: false + truthmark_config: true + git_repo: true +grading: + deterministic: + - changed-files + - write-boundary + - report-presence + llm_judges: + - workflow-eval-framwork/judges/trace-grader.md + - workflow-eval-framwork/judges/diff-grader.md + - workflow-eval-framwork/judges/report-grader.md diff --git a/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/setup.patch b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/setup.patch new file mode 100644 index 0000000..5af1d80 --- /dev/null +++ b/workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/setup.patch @@ -0,0 +1,12 @@ +--- a/src/cache.ts ++++ b/src/cache.ts +@@ -9,8 +9,8 @@ + return { + key, + value, +- maxAgeSeconds: 60, +- staleWhileRevalidateSeconds: 30, ++ maxAgeSeconds: 300, ++ staleWhileRevalidateSeconds: 120, + }; + } diff --git a/workflow-eval-framwork/schemas/catalog.schema.json b/workflow-eval-framwork/schemas/catalog.schema.json new file mode 100644 index 0000000..d41e92b --- /dev/null +++ b/workflow-eval-framwork/schemas/catalog.schema.json @@ -0,0 +1,33 @@ +{ + "type": "object", + "required": ["version", "manual_only", "workflows"], + "additionalProperties": true, + "properties": { + "version": { "type": "integer", "minimum": 1 }, + "manual_only": { "const": true }, + "last_reviewed": { "type": ["string", "null"] }, + "workflows": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "priority", "rubric", "scenarios"], + "additionalProperties": true, + "properties": { + "id": { "type": "string", "pattern": "^truthmark-[a-z-]+$" }, + "priority": { "enum": ["critical", "high", "medium", "low"] }, + "manual_only": { "const": true }, + "surface_paths": { "type": "array", "items": { "type": "string" } }, + "rubric": { "type": "string", "pattern": "^workflow-eval-framwork/rubrics/.+\\.yaml$" }, + "scenarios": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "^workflow-eval-framwork/scenarios/.+/scenario\\.yaml$" } + } + } + } + }, + "runners": { "type": "object" }, + "budgets": { "type": "object" } + } +} diff --git a/workflow-eval-framwork/schemas/expected.schema.json b/workflow-eval-framwork/schemas/expected.schema.json new file mode 100644 index 0000000..14d6caa --- /dev/null +++ b/workflow-eval-framwork/schemas/expected.schema.json @@ -0,0 +1,32 @@ +{ + "type": "object", + "additionalProperties": true, + "properties": { + "must_read": { "type": "array", "items": { "type": "string" } }, + "must_change": { "type": "array", "items": { "type": "string" } }, + "must_not_change": { "type": "array", "items": { "type": "string" } }, + "must_run_or_explain": { "type": "array", "items": { "type": "string" } }, + "report": { + "type": "object", + "additionalProperties": true, + "properties": { + "required": { "type": "boolean" }, + "validator": { "type": "string" }, + "required_sections": { "type": "array", "items": { "type": "string" } } + } + }, + "assertions": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "pass"], + "additionalProperties": true, + "properties": { + "id": { "type": "string" }, + "kind": { "type": "string" }, + "pass": { "type": "string" } + } + } + } + } +} diff --git a/workflow-eval-framwork/schemas/human-review.schema.json b/workflow-eval-framwork/schemas/human-review.schema.json new file mode 100644 index 0000000..2f10a19 --- /dev/null +++ b/workflow-eval-framwork/schemas/human-review.schema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "required": ["status", "reviewer", "notes"], + "properties": { + "status": { "enum": ["accepted", "rejected", "needs-rerun", "not-evaluable"] }, + "reviewer": { "type": "string" }, + "reviewed_at": { "type": "string" }, + "notes": { "type": "array", "items": { "type": "string" } }, + "follow_up": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false +} diff --git a/workflow-eval-framwork/schemas/judge-results.schema.json b/workflow-eval-framwork/schemas/judge-results.schema.json new file mode 100644 index 0000000..80a1641 --- /dev/null +++ b/workflow-eval-framwork/schemas/judge-results.schema.json @@ -0,0 +1,22 @@ +{ + "type": "object", + "required": ["status", "judges"], + "properties": { + "status": { "enum": ["passed", "failed", "not_evaluable", "skipped"] }, + "judges": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "status"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "status": { "enum": ["passed", "failed", "not_evaluable", "skipped"] }, + "summary": { "type": "string" } + }, + "additionalProperties": true + } + }, + "note": { "type": "string" } + }, + "additionalProperties": true +} diff --git a/workflow-eval-framwork/schemas/run.schema.json b/workflow-eval-framwork/schemas/run.schema.json new file mode 100644 index 0000000..4ca4e70 --- /dev/null +++ b/workflow-eval-framwork/schemas/run.schema.json @@ -0,0 +1,27 @@ +{ + "type": "object", + "required": ["status", "runDir", "scenario", "host", "deterministic", "judge", "usage"], + "properties": { + "status": { "enum": ["passed", "failed", "blocked", "not_evaluable"] }, + "runDir": { "type": "string" }, + "scenario": { "type": "string" }, + "host": { "type": "string" }, + "agentModel": { "type": "string" }, + "judgeModel": { "type": "string" }, + "deterministic": { "type": "object" }, + "judge": { "type": "object" }, + "usage": { + "type": "object", + "required": ["status", "agent", "judge", "total", "requirement"], + "additionalProperties": true, + "properties": { + "status": { "enum": ["measured", "unavailable", "invalid"] }, + "agent": { "type": "object" }, + "judge": { "type": "object" }, + "total": { "type": "object" }, + "requirement": { "type": "object" } + } + } + }, + "additionalProperties": true +} diff --git a/workflow-eval-framwork/schemas/scenario.schema.json b/workflow-eval-framwork/schemas/scenario.schema.json new file mode 100644 index 0000000..c80f314 --- /dev/null +++ b/workflow-eval-framwork/schemas/scenario.schema.json @@ -0,0 +1,32 @@ +{ + "type": "object", + "required": ["id", "workflow", "priority", "manual_only", "summary", "fixture", "prompt", "expected", "grading"], + "additionalProperties": true, + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "workflow": { "type": "string", "pattern": "^truthmark-[a-z-]+$" }, + "priority": { "enum": ["critical", "high", "medium", "low"] }, + "manual_only": { "const": true }, + "summary": { "type": "string", "minLength": 1 }, + "fixture": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string", "minLength": 1 }, + "expected": { "type": "string", "minLength": 1 }, + "setup": { + "type": "object", + "additionalProperties": false, + "properties": { + "patch": { "type": "string", "pattern": "^.+\\.patch$" } + } + }, + "requires": { "type": "object" }, + "grading": { + "type": "object", + "required": ["deterministic", "llm_judges"], + "additionalProperties": true, + "properties": { + "deterministic": { "type": "array", "minItems": 1, "items": { "type": "string" } }, + "llm_judges": { "type": "array", "minItems": 1, "items": { "type": "string" } } + } + } + } +}