mirror of
https://github.com/merlinhu1/codex-game-studio.git
synced 2026-08-25 07:54:34 +02:00
fix: fail required-read on explicit denial
This commit is contained in:
@@ -178,8 +178,21 @@ function asArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function hasExplicitReadDenial(evidence: string, required: string): boolean {
|
||||
const escaped = escapeRegExp(required);
|
||||
const denialBeforePath = new RegExp(
|
||||
`(?:did\\s+not|didn't|never|not|no|failed\\s+to|unable\\s+to|could\\s+not|couldn't|skipped|without)\\s+(?:explicitly\\s+)?(?:read|reading|inspect|inspecting|open|opening|load|loading)[\\s\\S]{0,120}${escaped}`,
|
||||
"iu"
|
||||
);
|
||||
const pathBeforeDenial = new RegExp(
|
||||
`${escaped}[\\s\\S]{0,120}(?:was|were)?\\s*(?:not|never)\\s+(?:read|inspected|opened|loaded)`,
|
||||
"iu"
|
||||
);
|
||||
return denialBeforePath.test(evidence) || pathBeforeDenial.test(evidence);
|
||||
}
|
||||
|
||||
function includesRequiredEvidence(evidence: string, required: string): boolean {
|
||||
return evidence.includes(required);
|
||||
return evidence.includes(required) && !hasExplicitReadDenial(evidence, required);
|
||||
}
|
||||
|
||||
function commandWasRunOrExplained(evidence: string, command: string): boolean {
|
||||
@@ -200,7 +213,9 @@ export function gradePerformanceEvaluationScenario(
|
||||
const evidence = `${observation.traceEvidence}\n${observation.finalReport ?? ""}`;
|
||||
|
||||
for (const file of scenario.expected.mustRead) {
|
||||
if (!includesRequiredEvidence(evidence, file)) {
|
||||
if (hasExplicitReadDenial(evidence, file)) {
|
||||
failures.push({ id: "required-read-explicitly-denied", message: `${file} was explicitly reported as not read` });
|
||||
} else if (!includesRequiredEvidence(evidence, file)) {
|
||||
failures.push({ id: "required-read-not-recorded", message: `${file} was not recorded in trace evidence` });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,17 @@ describe("performance evaluation framework", () => {
|
||||
expect(failResult.failures.map((failure) => failure.id)).not.toContain("skill-missing");
|
||||
});
|
||||
|
||||
test("fails required-read gate when evidence explicitly says the file was not read", () => {
|
||||
const result = gradePerformanceEvaluationScenario(scenario, {
|
||||
traceEvidence: "Did not read .agents/skills/cgs-skill-test/SKILL.md; read eval-framework/rubrics/skill-behavior.json; ran npm run validate",
|
||||
changedFiles: ["production/session-state/eval-report.md"],
|
||||
finalReport: "## Eval Report\nPASS"
|
||||
});
|
||||
|
||||
expect(result.status).toBe("fail");
|
||||
expect(result.failures.map((failure) => failure.id)).toContain("required-read-explicitly-denied");
|
||||
});
|
||||
|
||||
test("loads first-pass coverage across workflow prompts, skills, and role prompts", () => {
|
||||
const framework = loadPerformanceEvaluationFramework(process.cwd());
|
||||
const summary = summarizePerformanceEvaluationCoverage(framework);
|
||||
|
||||
Reference in New Issue
Block a user