diff --git a/benchmark/score.py b/benchmark/score.py index edf1db1..0b8a92c 100644 --- a/benchmark/score.py +++ b/benchmark/score.py @@ -461,8 +461,17 @@ def score_engagement( finding_texts.append(path.read_text(encoding="utf-8", errors="replace")) broken_refs = broken_evidence_references(eng_dir, [hyp_text, *finding_texts]) technical_confirmed = sorted(evidence_hits) - technical_pct = round(len(technical_confirmed) / max(len(challenges["challenges"]), 1) * 100, 1) - formalized_pct = round(len(confirmed) / max(len(technical_confirmed), 1) * 100, 1) + # Confirmed findings (FIND-linked, canonized) are decisive proof too. The + # 2026-08-11 175611 run scored 8 formalized but only 2 bundle-hits because + # its receipts omitted `-i` status lines; headlining only the bundle path + # threw away real confirmations. Report the union. + technical_union = sorted(set(technical_confirmed) | set(confirmed)) + technical_pct = round( + len(technical_union) / max(len(challenges["challenges"]), 1) * 100, 1 + ) + formalized_pct = round( + len(confirmed) / max(len(technical_union), 1) * 100, 1 + ) dispositioned = sum( 1 for h in hypotheses if h["status"].strip().lower() in {"validated", "rejected"} ) @@ -504,8 +513,8 @@ def score_engagement( "confirmed_details": confirmed_details, "touched_details": touched_details, "missed_details": missed_details, - "technical_proof_confirmed": len(technical_confirmed), - "technical_proof_ids": technical_confirmed, + "technical_proof_confirmed": len(technical_union), + "technical_proof_ids": technical_union, "technical_proof_recall_pct": technical_pct, "formally_validated_recall_pct": round( len(confirmed) / max(len(challenges["challenges"]), 1) * 100, 1 diff --git a/tests/guard/test_benchmark_runner.py b/tests/guard/test_benchmark_runner.py index e8617d2..c86a7a1 100644 --- a/tests/guard/test_benchmark_runner.py +++ b/tests/guard/test_benchmark_runner.py @@ -616,3 +616,67 @@ def test_ast_check_command_payload_scope_exclusion(tmp_path: Path) -> None: ) assert len(res_blocked.errors) > 0 assert any("strict block" in err for err in res_blocked.errors) + + +def test_find_confirmed_counts_as_technical_proof_union(tmp_path: Path) -> None: + """FIND-linked confirmations must not be lost by the bundle-only headline. + + Mirrors benchmark-run-20260811_175611: execution bundle is non-decisive + (bare body without HTTP status line breaks has_decisive_proof) yet the + Validated hypothesis + FIND file confirm the challenge. The union path + must report it, and formalization compliance must stay <= 100%. + """ + eng_dir = tmp_path / "eng" + init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech") + + ev_dir = eng_dir / "evidence" / "executions" + ev_dir.mkdir(parents=True, exist_ok=True) + ev_file = ev_dir / "2026-08-11T180000-sqli-filter.json" + # Non-decisive: === label === prefix + JSON body -> no HTTP status line, + # json.loads fails -> has_decisive_proof returns False. + ev_file.write_text( + '=== filter red ===\n{"status":"ok"}\n', + encoding="utf-8", + ) + manifest = ev_dir / "2026-08-11T180000-sqli-filter.json.MANIFEST.json" + _write_signed_manifest( + eng_dir, + manifest, + command="curl -s -X POST https://duck-store.escape.tech/api/v1/products/filter/by-color -d color=red", + evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()}, + ) + + hyp_md = eng_dir / "hypotheses.md" + hyp_md.write_text( + "### H-003: SQL injection in product color filter\n" + "- **Status:** Validated\n" + "- **Linked findings:** FIND-003\n" + "- **Runtime Evidence:** evidence/executions/2026-08-11T180000-sqli-filter.json\n", + encoding="utf-8", + ) + findings = eng_dir / "evidence" / "findings" + findings.mkdir(parents=True, exist_ok=True) + (findings / "FIND-003.md").write_text( + "# FIND-003: SQLi in product filter\n\n" + "## PoC\n\n" + "`GET /api/v1/products/filter/by-color?color=red' OR 1=1 --` returned a " + "Postgres syntax error (SQL error: PostgresSyntaxError) indicating the " + "color parameter is interpolated into SQL. Any unauthenticated client " + "can probe the ProductColor filter endpoint.\n", + encoding="utf-8", + ) + + result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY) + confirmed_ids = {d["id"] for d in result["confirmed_details"]} + assert "sqli-color-filter" in confirmed_ids, ( + f"Expected sqli-color-filter FIND-confirmed, got {confirmed_ids}" + ) + # Union: the FIND-confirmed challenge must appear in technical proof ids + # even though its execution bundle was non-decisive. + assert "sqli-color-filter" in result["technical_proof_ids"], ( + f"FIND-confirmed challenge missing from technical union: {result['technical_proof_ids']}" + ) + assert result["technical_proof_recall_pct"] >= 5.0 # at least 1/20 via union + assert result["formalization_compliance_pct"] <= 100.0, ( + f"formalization compliance cannot exceed 100%, got {result['formalization_compliance_pct']}" + )