mirror of
https://github.com/merlinhu1/truthmark.git
synced 2026-08-25 07:53:25 +02:00
Merlin's CatGitHubCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>MerlinHCopilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>* feat: add workflow eval framework Add workflow evaluation scenarios, rubrics, schemas, and runner scripts for installed Truthmark workflows. Move research notes under docs/research and migrate tests from Vitest to node:test. * Potential fix for pull request finding 'CodeQL / Replacement of a substring with itself' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: MerlinH <merlinh221@gmail.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
60 lines
2.1 KiB
YAML
60 lines
2.1 KiB
YAML
version: 1
|
|
manual_only: true
|
|
last_reviewed: 2026-06-29
|
|
workflows:
|
|
- id: truthmark-sync
|
|
priority: critical
|
|
manual_only: true
|
|
surface_paths:
|
|
- src/agents/truth-sync.ts
|
|
- src/templates/workflow-surfaces.ts
|
|
rubric: workflow-eval-framwork/rubrics/truthmark-sync.yaml
|
|
scenarios:
|
|
- workflow-eval-framwork/scenarios/truthmark-sync/docs-only-skip/scenario.yaml
|
|
- workflow-eval-framwork/scenarios/truthmark-sync/bootstrap-route-blocked/scenario.yaml
|
|
- workflow-eval-framwork/scenarios/truthmark-sync/stale-engineering-truth/scenario.yaml
|
|
- id: truthmark-document
|
|
priority: high
|
|
manual_only: true
|
|
surface_paths:
|
|
- src/agents/truth-document.ts
|
|
rubric: workflow-eval-framwork/rubrics/truthmark-document.yaml
|
|
scenarios:
|
|
- workflow-eval-framwork/scenarios/truthmark-document/implemented-behavior-missing-truth/scenario.yaml
|
|
- id: truthmark-realize
|
|
priority: high
|
|
manual_only: true
|
|
surface_paths:
|
|
- src/agents/prompts.ts
|
|
- src/realize/report.ts
|
|
rubric: workflow-eval-framwork/rubrics/truthmark-realize.yaml
|
|
scenarios:
|
|
- workflow-eval-framwork/scenarios/truthmark-realize/implement-truth-without-doc-edits/scenario.yaml
|
|
- id: truthmark-check
|
|
priority: high
|
|
manual_only: true
|
|
surface_paths:
|
|
- src/agents/truth-check.ts
|
|
rubric: workflow-eval-framwork/rubrics/truthmark-check.yaml
|
|
scenarios:
|
|
- workflow-eval-framwork/scenarios/truthmark-check/unsupported-claim-audit/scenario.yaml
|
|
- workflow-eval-framwork/scenarios/truthmark-check/general-question-no-write/scenario.yaml
|
|
- id: truthmark-structure
|
|
priority: medium
|
|
manual_only: true
|
|
surface_paths:
|
|
- src/agents/truth-structure.ts
|
|
rubric: workflow-eval-framwork/rubrics/truthmark-structure.yaml
|
|
scenarios:
|
|
- workflow-eval-framwork/scenarios/truthmark-structure/broad-route-repair/scenario.yaml
|
|
runners:
|
|
harness_hosts:
|
|
- fake
|
|
manual_agent_hosts:
|
|
- codex
|
|
- claude-code
|
|
budgets:
|
|
max_agent_runs_per_scenario: 1
|
|
max_judge_calls_per_scenario: 3
|
|
default_timeout_seconds: 900
|