fix(guard): block rejections that never ran their cheapest discriminating test

This commit is contained in:
Violin
2026-08-11 17:57:02 +01:00
parent f04116377e
commit 9a40ec645c
2 changed files with 94 additions and 0 deletions
@@ -101,6 +101,36 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None:
unresolved = [
f"H-{item.id}" for item in board if item.canonical_status() in {"Candidate", "Likely"}
]
# A Rejected hypothesis disposed as not_implemented must still carry a
# real executed test and evidence. "N/A - placeholder" with no evidence
# means the cheapest discriminating test never ran — the agent disposed
# it from the conversation (the 2026-08-11 161722 run rejected H-001
# (admin/admin login check) this way and weak-admin-creds was never
# tested). Surface-mapping hypotheses (e.g. "API surface enumeration")
# pass when they cite real bundle/probe evidence.
if not unresolved:
untested_disposals = []
for item in board:
if item.canonical_status() != "Rejected":
continue
if item.verification_status.strip().lower() != "not_implemented":
continue
cmd = item.test_command.strip().lower()
evidence_cited = bool(
(item.runtime_evidence or item.evidence or "").strip()
)
if cmd in {"", "n/a", "na", "none", "-"} or cmd.startswith("n/a") or not evidence_cited:
untested_disposals.append(
f"H-{item.id} (cheapest test: {(item.cheapest_test or '?').strip()!r})"
)
if untested_disposals:
raise ValueError(
"VULN_RESEARCH cannot close with rejections that never ran their "
"cheapest discriminating test: "
+ ", ".join(untested_disposals)
+ ". Execute the test and record Test Command/Test Response/Runtime "
"Evidence (or keep the hypothesis active) before closing."
)
if unresolved:
raise ValueError(
"VULN_RESEARCH cannot close with unresolved hypotheses: " + ", ".join(unresolved)
@@ -291,6 +291,70 @@ def test_vuln_research_exit_accepts_evidence_backed_not_applicable(tmp_path: Pat
_validate_phase_exit(engagement, "PT-030", "[x]") # no exception
def test_vuln_research_exit_blocks_not_implemented_rejection_without_evidence(
tmp_path: Path,
) -> None:
engagement = tmp_path / "engagement"
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0
scope = engagement / "scope" / "scope.yaml"
scope.write_text(
scope.read_text(encoding="utf-8") + "\nbenchmark:\n mode: true\n",
encoding="utf-8",
)
matrix = engagement / "state" / "coverage-matrix.yaml"
matrix.write_text(
"coverage:\n routes:\n status: tested\n evidence_or_reason: 'evidence/recon/probe.txt'\n",
encoding="utf-8",
)
hypotheses.update_hypothesis(
engagement / "hypotheses.md",
id="001",
title="Admin login check",
status="Rejected",
verification_status="not_implemented",
test_command="N/A - placeholder hypothesis",
test_response="never executed",
rejection_reason="placeholder superseded",
cheapest_test="Login as admin (admin/admin)",
)
with pytest.raises(
ValueError, match="rejections that never ran their cheapest discriminating test"
):
_validate_phase_exit(engagement, "PT-030", "[x]")
def test_vuln_research_exit_accepts_surface_mapping_rejection_with_evidence(
tmp_path: Path,
) -> None:
"""Known-good pattern: a recon surface-mapping hypothesis rejected as
not_implemented is fine when it cites real bundle/probe evidence."""
engagement = tmp_path / "engagement"
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0
scope = engagement / "scope" / "scope.yaml"
scope.write_text(
scope.read_text(encoding="utf-8") + "\nbenchmark:\n mode: true\n",
encoding="utf-8",
)
matrix = engagement / "state" / "coverage-matrix.yaml"
matrix.write_text(
"coverage:\n routes:\n status: tested\n evidence_or_reason: 'evidence/recon/probe.txt'\n",
encoding="utf-8",
)
hypotheses.update_hypothesis(
engagement / "hypotheses.md",
id="001",
title="API surface enumeration from JS bundle",
status="Rejected",
verification_status="not_implemented",
test_command="GET /api/v1/products/, /testimonials/",
test_response="surface mapped, see evidence",
rejection_reason="not a vulnerability claim",
evidence="evidence/recon/recon_bundle.js",
cheapest_test="Probe each derived endpoint",
)
_validate_phase_exit(engagement, "PT-030", "[x]") # no exception
def test_validated_hypothesis_rejects_escaping_or_empty_evidence(tmp_path: Path) -> None:
engagement = tmp_path / "engagement"
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0