diff --git a/benchmark/run.py b/benchmark/run.py index e62907a..86f0266 100644 --- a/benchmark/run.py +++ b/benchmark/run.py @@ -188,6 +188,22 @@ def _scope_for_target(target: str) -> dict: brief = brief_path.read_text(encoding="utf-8").strip() if brief: scope.setdefault("engagement", {})["brief"] = brief + # Benchmark challenge IDs: seed the scored challenge ids into scope as + # framework-owned coverage targets. The VULN_RESEARCH coverage gate then + # requires an evidence-backed cell per challenge so the agent cannot + # declare "coverage finalized" while specific scored endpoints (e.g. + # by-color filter, users/, uploads/fetch-url) were never probed. + challenges_path = fixture_path / "challenges.json" + if isinstance(benchmark, dict) and challenges_path.is_file(): + try: + challenges = json.loads( + challenges_path.read_text(encoding="utf-8") + ).get("challenges", []) + except (OSError, ValueError): + challenges = [] + benchmark.pop("challenge_ids", None) + if challenges: + benchmark["challenge_ids"] = [ch["id"] for ch in challenges if ch.get("id")] return scope diff --git a/plugins/violin_guard/handlers/ptt_handlers.py b/plugins/violin_guard/handlers/ptt_handlers.py index ffdeb80..d6b880b 100644 --- a/plugins/violin_guard/handlers/ptt_handlers.py +++ b/plugins/violin_guard/handlers/ptt_handlers.py @@ -73,7 +73,19 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None: entries = matrix.get("coverage") if isinstance(matrix, dict) else None if not isinstance(entries, dict) or not entries: raise ValueError("coverage matrix must contain a non-empty coverage mapping") + challenge_ids = (scope_data.get("benchmark") or {}).get("challenge_ids") or [] + covered_ids = ( + {str(name).lower() for name in entries} + | {str(entry.get("challenge_id") or "").lower() for entry in entries.values() if isinstance(entry, dict)} + ) unresolved_coverage = [] + # Scored challenges must map to a matrix cell — the 2026-08-11 + # 184338 run self-declared ssrf/redirects/rate_limits "tested" or + # N/A while ssrf-*, no-rate-limiting, open-redirect, sqli-color- + # filter and user-enumeration were never probed at all. + for cid in challenge_ids: + if str(cid).strip().lower() not in covered_ids: + unresolved_coverage.append(f"{cid} (no coverage-matrix cell)") for name, entry in entries.items(): if not isinstance(entry, dict): unresolved_coverage.append(name) @@ -93,6 +105,15 @@ def _validate_phase_exit(engagement: Path, task_id: str, status: str) -> None: unresolved_coverage.append(f"{name} (not_applicable without evidence file)") elif status == "blocked" and "guard" not in reason.lower(): unresolved_coverage.append(f"{name} (blocked without guard reference)") + # tested must cite a canonical artifact (evidence path, FIND, + # or hypothesis id) — a bare narrative ("no open redirect + # parameter found") is aspirational coverage, not proof. + elif status == "tested" and not re.search( + r"evidence/|FIND-\d+|H-\d+", reason, re.IGNORECASE + ): + unresolved_coverage.append( + f"{name} (tested without evidence/FIND/hypothesis reference)" + ) if unresolved_coverage: raise ValueError( "VULN_RESEARCH cannot close with undispositioned coverage: " diff --git a/tests/guard/state/test_workflow_contract.py b/tests/guard/state/test_workflow_contract.py index 58c3d05..0433cfa 100644 --- a/tests/guard/state/test_workflow_contract.py +++ b/tests/guard/state/test_workflow_contract.py @@ -448,3 +448,62 @@ def test_validated_hypothesis_rejects_escaping_or_empty_evidence(tmp_path: Path) status="Validated", runtime_evidence="evidence/exploitation/empty.txt", ) + + +def test_vuln_research_exit_blocks_uncharted_scored_challenges(tmp_path: Path) -> None: + """Coverage completeness: every seeded challenge_id needs a matrix cell. + + Mirrors benchmark-run-20260811_184338 where the agent closed VULN_RESEARCH + with ssrf-*/no-rate-limiting/user-enumeration never probed while + self-declaring ssrf/redirects/rate_limits 'tested' or N/A. + """ + engagement = tmp_path / "engagement" + assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0 + scope = engagement / "scope" / "scope.yaml" + scope.write_text( + scope.read_text(encoding="utf-8") + + "\nbenchmark:\n mode: true\n challenge_ids:\n - ssrf-fetch-url\n - no-rate-limiting\n", + encoding="utf-8", + ) + matrix = engagement / "state" / "coverage-matrix.yaml" + matrix.write_text( + "coverage:\n routes:\n status: tested\n evidence_or_reason: 'evidence/recon/probe.txt'\n", + encoding="utf-8", + ) + with pytest.raises(ValueError, match="no coverage-matrix cell"): + _validate_phase_exit(engagement, "PT-030", "[x]") + + +def test_vuln_research_exit_blocks_aspirational_tested_narrative(tmp_path: Path) -> None: + """'tested' without an artifact reference is aspirational, not proof.""" + engagement = tmp_path / "engagement" + assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0 + scope = engagement / "scope" / "scope.yaml" + scope.write_text( + scope.read_text(encoding="utf-8") + "\nbenchmark:\n mode: true\n", + encoding="utf-8", + ) + matrix = engagement / "state" / "coverage-matrix.yaml" + matrix.write_text( + "coverage:\n redirects:\n status: tested\n evidence_or_reason: 'no open redirect parameter found'\n", + encoding="utf-8", + ) + with pytest.raises(ValueError, match="tested without evidence/FIND/hypothesis"): + _validate_phase_exit(engagement, "PT-030", "[x]") + + +def test_vuln_research_exit_accepts_challenge_cells_with_artifact(tmp_path: Path) -> None: + engagement = tmp_path / "engagement" + assert bootstrap.init_engagement(engagement, host="10.10.10.10") == 0 + scope = engagement / "scope" / "scope.yaml" + scope.write_text( + scope.read_text(encoding="utf-8") + + "\nbenchmark:\n mode: true\n challenge_ids:\n - no-rate-limiting\n", + encoding="utf-8", + ) + matrix = engagement / "state" / "coverage-matrix.yaml" + matrix.write_text( + "coverage:\n no-rate-limiting:\n status: not_applicable\n evidence_or_reason: 'evidence/vuln-research/rate_na.txt - 429 never observed'\n", + encoding="utf-8", + ) + _validate_phase_exit(engagement, "PT-030", "[x]") # no exception