fix(guard): block rejections that never ran their cheapest discriminating test

This commit is contained in:
Violin
2026-08-11 17:57:02 +01:00
parent f04116377e
commit 9a40ec645c
2 changed files with 94 additions and 0 deletions
@@ -101,6 +101,36 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None:
unresolved = [
f"H-{item.id}" for item in board if item.canonical_status() in {"Candidate", "Likely"}
]
# A Rejected hypothesis disposed as not_implemented must still carry a
# real executed test and evidence. "N/A - placeholder" with no evidence
# means the cheapest discriminating test never ran — the agent disposed
# it from the conversation (the 2026-08-11 161722 run rejected H-001
# (admin/admin login check) this way and weak-admin-creds was never
# tested). Surface-mapping hypotheses (e.g. "API surface enumeration")
# pass when they cite real bundle/probe evidence.
if not unresolved:
untested_disposals = []
for item in board:
if item.canonical_status() != "Rejected":
continue
if item.verification_status.strip().lower() != "not_implemented":
continue
cmd = item.test_command.strip().lower()
evidence_cited = bool(
(item.runtime_evidence or item.evidence or "").strip()
)
if cmd in {"", "n/a", "na", "none", "-"} or cmd.startswith("n/a") or not evidence_cited:
untested_disposals.append(
f"H-{item.id} (cheapest test: {(item.cheapest_test or '?').strip()!r})"
)
if untested_disposals:
raise ValueError(
"VULN_RESEARCH cannot close with rejections that never ran their "
"cheapest discriminating test: "
+ ", ".join(untested_disposals)
+ ". Execute the test and record Test Command/Test Response/Runtime "
"Evidence (or keep the hypothesis active) before closing."
)
if unresolved:
raise ValueError(
"VULN_RESEARCH cannot close with unresolved hypotheses: " + ", ".join(unresolved)