fix: fail required-read on explicit denial

This commit is contained in:
MerlinH
2026-07-02 09:27:58 +10:00
parent 2e7ed2566f
commit b2dc45e36b
2 changed files with 28 additions and 2 deletions
+17 -2
View File
@@ -178,8 +178,21 @@ function asArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
}
function hasExplicitReadDenial(evidence: string, required: string): boolean {
const escaped = escapeRegExp(required);
const denialBeforePath = new RegExp(
`(?:did\\s+not|didn't|never|not|no|failed\\s+to|unable\\s+to|could\\s+not|couldn't|skipped|without)\\s+(?:explicitly\\s+)?(?:read|reading|inspect|inspecting|open|opening|load|loading)[\\s\\S]{0,120}${escaped}`,
"iu"
);
const pathBeforeDenial = new RegExp(
`${escaped}[\\s\\S]{0,120}(?:was|were)?\\s*(?:not|never)\\s+(?:read|inspected|opened|loaded)`,
"iu"
);
return denialBeforePath.test(evidence) || pathBeforeDenial.test(evidence);
}
function includesRequiredEvidence(evidence: string, required: string): boolean {
return evidence.includes(required);
return evidence.includes(required) && !hasExplicitReadDenial(evidence, required);
}
function commandWasRunOrExplained(evidence: string, command: string): boolean {
@@ -200,7 +213,9 @@ export function gradePerformanceEvaluationScenario(
const evidence = `${observation.traceEvidence}\n${observation.finalReport ?? ""}`;
for (const file of scenario.expected.mustRead) {
if (!includesRequiredEvidence(evidence, file)) {
if (hasExplicitReadDenial(evidence, file)) {
failures.push({ id: "required-read-explicitly-denied", message: `${file} was explicitly reported as not read` });
} else if (!includesRequiredEvidence(evidence, file)) {
failures.push({ id: "required-read-not-recorded", message: `${file} was not recorded in trace evidence` });
}
}
@@ -59,6 +59,17 @@ describe("performance evaluation framework", () => {
expect(failResult.failures.map((failure) => failure.id)).not.toContain("skill-missing");
});
test("fails required-read gate when evidence explicitly says the file was not read", () => {
const result = gradePerformanceEvaluationScenario(scenario, {
traceEvidence: "Did not read .agents/skills/cgs-skill-test/SKILL.md; read eval-framework/rubrics/skill-behavior.json; ran npm run validate",
changedFiles: ["production/session-state/eval-report.md"],
finalReport: "## Eval Report\nPASS"
});
expect(result.status).toBe("fail");
expect(result.failures.map((failure) => failure.id)).toContain("required-read-explicitly-denied");
});
test("loads first-pass coverage across workflow prompts, skills, and role prompts", () => {
const framework = loadPerformanceEvaluationFramework(process.cwd());
const summary = summarizePerformanceEvaluationCoverage(framework);