fix(guard): coverage completeness — every scored challenge needs an evidence-backed matrix cell

This commit is contained in:
Violin
2026-08-11 20:20:07 +01:00
parent b84cd852f2
commit 61e3ed8b61
3 changed files with 96 additions and 0 deletions
@@ -73,7 +73,19 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None:
entries = matrix.get("coverage") if isinstance(matrix, dict) else None
if not isinstance(entries, dict) or not entries:
raise ValueError("coverage matrix must contain a non-empty coverage mapping")
challenge_ids = (scope_data.get("benchmark") or {}).get("challenge_ids") or []
covered_ids = (
{str(name).lower() for name in entries}
| {str(entry.get("challenge_id") or "").lower() for entry in entries.values() if isinstance(entry, dict)}
)
unresolved_coverage = []
# Scored challenges must map to a matrix cell — the 2026-08-11
# 184338 run self-declared ssrf/redirects/rate_limits "tested" or
# N/A while ssrf-*, no-rate-limiting, open-redirect, sqli-color-
# filter and user-enumeration were never probed at all.
for cid in challenge_ids:
if str(cid).strip().lower() not in covered_ids:
unresolved_coverage.append(f"{cid} (no coverage-matrix cell)")
for name, entry in entries.items():
if not isinstance(entry, dict):
unresolved_coverage.append(name)
@@ -93,6 +105,15 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None:
unresolved_coverage.append(f"{name} (not_applicable without evidence file)")
elif status == "blocked" and "guard" not in reason.lower():
unresolved_coverage.append(f"{name} (blocked without guard reference)")
# tested must cite a canonical artifact (evidence path, FIND,
# or hypothesis id) — a bare narrative ("no open redirect
# parameter found") is aspirational coverage, not proof.
elif status == "tested" and not re.search(
r"evidence/|FIND-\d+|H-\d+", reason, re.IGNORECASE
):
unresolved_coverage.append(
f"{name} (tested without evidence/FIND/hypothesis reference)"
)
if unresolved_coverage:
raise ValueError(
"VULN_RESEARCH cannot close with undispositioned coverage: "