mirror of
https://github.com/Strategic-Automation/violin.git
synced 2026-08-14 12:33:37 +02:00
fix(guard): block rejections that never ran their cheapest discriminating test
This commit is contained in:
@@ -101,6 +101,36 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None:
|
||||
unresolved = [
|
||||
f"H-{item.id}" for item in board if item.canonical_status() in {"Candidate", "Likely"}
|
||||
]
|
||||
# A Rejected hypothesis disposed as not_implemented must still carry a
|
||||
# real executed test and evidence. "N/A - placeholder" with no evidence
|
||||
# means the cheapest discriminating test never ran — the agent disposed
|
||||
# it from the conversation (the 2026-08-11 161722 run rejected H-001
|
||||
# (admin/admin login check) this way and weak-admin-creds was never
|
||||
# tested). Surface-mapping hypotheses (e.g. "API surface enumeration")
|
||||
# pass when they cite real bundle/probe evidence.
|
||||
if not unresolved:
|
||||
untested_disposals = []
|
||||
for item in board:
|
||||
if item.canonical_status() != "Rejected":
|
||||
continue
|
||||
if item.verification_status.strip().lower() != "not_implemented":
|
||||
continue
|
||||
cmd = item.test_command.strip().lower()
|
||||
evidence_cited = bool(
|
||||
(item.runtime_evidence or item.evidence or "").strip()
|
||||
)
|
||||
if cmd in {"", "n/a", "na", "none", "-"} or cmd.startswith("n/a") or not evidence_cited:
|
||||
untested_disposals.append(
|
||||
f"H-{item.id} (cheapest test: {(item.cheapest_test or '?').strip()!r})"
|
||||
)
|
||||
if untested_disposals:
|
||||
raise ValueError(
|
||||
"VULN_RESEARCH cannot close with rejections that never ran their "
|
||||
"cheapest discriminating test: "
|
||||
+ ", ".join(untested_disposals)
|
||||
+ ". Execute the test and record Test Command/Test Response/Runtime "
|
||||
"Evidence (or keep the hypothesis active) before closing."
|
||||
)
|
||||
if unresolved:
|
||||
raise ValueError(
|
||||
"VULN_RESEARCH cannot close with unresolved hypotheses: " + ", ".join(unresolved)
|
||||
|
||||
@@ -291,6 +291,70 @@ def test_vuln_research_exit_accepts_evidence_backed_not_applicable(tmp_path: Pat
|
||||
_validate_phase_exit(engagement, "PT-030", "[x]") # no exception
|
||||
|
||||
|
||||
def test_vuln_research_exit_blocks_not_implemented_rejection_without_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
engagement = tmp_path / "engagement"
|
||||
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0
|
||||
scope = engagement / "scope" / "scope.yaml"
|
||||
scope.write_text(
|
||||
scope.read_text(encoding="utf-8") + "\nbenchmark:\n mode: true\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
matrix = engagement / "state" / "coverage-matrix.yaml"
|
||||
matrix.write_text(
|
||||
"coverage:\n routes:\n status: tested\n evidence_or_reason: 'evidence/recon/probe.txt'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
hypotheses.update_hypothesis(
|
||||
engagement / "hypotheses.md",
|
||||
id="001",
|
||||
title="Admin login check",
|
||||
status="Rejected",
|
||||
verification_status="not_implemented",
|
||||
test_command="N/A - placeholder hypothesis",
|
||||
test_response="never executed",
|
||||
rejection_reason="placeholder superseded",
|
||||
cheapest_test="Login as admin (admin/admin)",
|
||||
)
|
||||
with pytest.raises(
|
||||
ValueError, match="rejections that never ran their cheapest discriminating test"
|
||||
):
|
||||
_validate_phase_exit(engagement, "PT-030", "[x]")
|
||||
|
||||
|
||||
def test_vuln_research_exit_accepts_surface_mapping_rejection_with_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Known-good pattern: a recon surface-mapping hypothesis rejected as
|
||||
not_implemented is fine when it cites real bundle/probe evidence."""
|
||||
engagement = tmp_path / "engagement"
|
||||
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0
|
||||
scope = engagement / "scope" / "scope.yaml"
|
||||
scope.write_text(
|
||||
scope.read_text(encoding="utf-8") + "\nbenchmark:\n mode: true\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
matrix = engagement / "state" / "coverage-matrix.yaml"
|
||||
matrix.write_text(
|
||||
"coverage:\n routes:\n status: tested\n evidence_or_reason: 'evidence/recon/probe.txt'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
hypotheses.update_hypothesis(
|
||||
engagement / "hypotheses.md",
|
||||
id="001",
|
||||
title="API surface enumeration from JS bundle",
|
||||
status="Rejected",
|
||||
verification_status="not_implemented",
|
||||
test_command="GET /api/v1/products/, /testimonials/",
|
||||
test_response="surface mapped, see evidence",
|
||||
rejection_reason="not a vulnerability claim",
|
||||
evidence="evidence/recon/recon_bundle.js",
|
||||
cheapest_test="Probe each derived endpoint",
|
||||
)
|
||||
_validate_phase_exit(engagement, "PT-030", "[x]") # no exception
|
||||
|
||||
|
||||
def test_validated_hypothesis_rejects_escaping_or_empty_evidence(tmp_path: Path) -> None:
|
||||
engagement = tmp_path / "engagement"
|
||||
assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0
|
||||
|
||||
Reference in New Issue
Block a user