From a82c1dfd864b5bed0cf9ff5a0ec3c91c37dee64f Mon Sep 17 00:00:00 2001 From: Violin Date: Wed, 12 Aug 2026 21:32:56 +0100 Subject: [PATCH] fix(score): confirm findings via reverse hypothesis links Run 20260812_184313 scored 0/20 formalized despite 7 FIND files and 8 Validated hypotheses: the agent canonized findings carrying 'Linked Hypothesis: H-00N' but never wrote the forward 'Linked findings' board field (it closed PT-102 by editing state/ptt.md directly, bypassing the record_ptt handler gates). The scorer now: - parse_findings extracts reverse 'Linked Hypothesis' references - validated_challenge_ids falls back to reverse links / shared evidence when a Validated hypothesis has no forward links - paper trail is unchanged: Validated hypothesis + substantive finding over real evidence, in either direction Re-scoring 184313: formalized 0% -> 30% (6 confirmed). Regression test test_validated_hypothesis_confirms_via_find_reverse_link added. --- benchmark/score.py | 30 +++++++++++++++- tests/guard/test_benchmark_runner.py | 51 ++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/benchmark/score.py b/benchmark/score.py index 0b8a92c..0dea196 100644 --- a/benchmark/score.py +++ b/benchmark/score.py @@ -124,11 +124,26 @@ def parse_findings(eng_dir: Path) -> list[dict]: continue fid = fpath.stem evidence_files: set[str] = set() + linked_hypotheses: set[str] = set() for line in txt.splitlines(): if "evidence/" in line: for part in re.findall(r"evidence/[^\s,)`\]]+", line): evidence_files.add(Path(part).name) - results.append({"id": fid, "evidence_files": evidence_files, "text": txt}) + hyp_link = re.search( + r"linked\s+hypothes[ie]s?\s*:?\s*[*:\-]*\s*(H-\d+)", + line, + re.IGNORECASE, + ) + if hyp_link: + linked_hypotheses.add(hyp_link.group(1).upper()) + results.append( + { + "id": fid, + "evidence_files": evidence_files, + "linked_hypotheses": linked_hypotheses, + "text": txt, + } + ) return results @@ -140,6 +155,15 @@ def validated_challenge_ids( """Return proof-linked IDs with a Validated hypothesis and canonical finding.""" ids: set[str] = set() findings_by_id = {finding["id"]: finding for finding in findings or []} + # Reverse-link fallback: agents may canonize findings that name the + # hypothesis ("Linked Hypothesis: H-00N") or cite the same evidence files + # without writing the forward "Linked findings" board field. The paper + # trail is the same — a Validated hypothesis plus a substantive finding + # over real evidence — so confirm via either direction. + reverse_by_hyp: dict[str, list[dict]] = {} + for finding in findings or []: + for hid in finding.get("linked_hypotheses") or set(): + reverse_by_hyp.setdefault(hid, []).append(finding) for h in hypotheses: if h["status"].strip().lower() != "validated": continue @@ -148,11 +172,15 @@ def validated_challenge_ids( for finding_id in h.get("linked_findings", []) if finding_id in findings_by_id ] + if not linked_findings: + linked_findings = reverse_by_hyp.get(h["id"].upper(), []) if not linked_findings: continue cited_files = set(h.get("evidence_files", set())) for finding in linked_findings: cited_files.update(finding.get("evidence_files", set())) + if not cited_files and finding.get("linked_hypotheses"): + cited_files.update(h.get("evidence_files", set())) ids.update(h["linked"]) for challenge_id, proof_files in (evidence_hits or {}).items(): if {path.name for path in proof_files}.intersection(cited_files): diff --git a/tests/guard/test_benchmark_runner.py b/tests/guard/test_benchmark_runner.py index b72e6ec..d6fabb5 100644 --- a/tests/guard/test_benchmark_runner.py +++ b/tests/guard/test_benchmark_runner.py @@ -704,3 +704,54 @@ def test_find_confirmed_counts_as_technical_proof_union(tmp_path: Path) -> None: assert result["formalization_compliance_pct"] <= 100.0, ( f"formalization compliance cannot exceed 100%, got {result['formalization_compliance_pct']}" ) + + +def test_validated_hypothesis_confirms_via_find_reverse_link(tmp_path: Path) -> None: + """A FIND naming 'Linked Hypothesis: H-00N' confirms without board links. + + Mirrors benchmark-run-20260812_184313: the agent canonized 7 FIND files + that each carry 'Linked Hypothesis: H-00N' but never wrote the forward + 'Linked findings' field on the hypothesis board. The scorer must recover + the confirmation from the finding's reverse link + shared evidence. + """ + eng_dir = tmp_path / "eng" + init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech") + + ev_dir = eng_dir / "evidence" / "executions" + ev_dir.mkdir(parents=True, exist_ok=True) + ev_file = ev_dir / "2026-08-12T185112-idor-profiles.json" + ev_file.write_text( + 'HTTP/1.1 200 OK\n{"email":"admin@duck.store","role":"admin"}\n', + encoding="utf-8", + ) + manifest = ev_dir / "2026-08-12T185112-idor-profiles.json.MANIFEST.json" + _write_signed_manifest( + eng_dir, + manifest, + command="curl -si https://duck-store.escape.tech/api/v1/users/11111111-1111-1111-1111-111111111111", + evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()}, + ) + + # Validated hypothesis WITHOUT the forward 'Linked findings' field. + hyp_md = eng_dir / "hypotheses.md" + hyp_md.write_text( + "### H-002: IDOR on user profiles\n" + "- **Status:** Validated\n" + "- **Runtime Evidence:** evidence/executions/2026-08-12T185112-idor-profiles.json\n", + encoding="utf-8", + ) + findings = eng_dir / "evidence" / "findings" + findings.mkdir(parents=True, exist_ok=True) + (findings / "FIND-005.md").write_text( + "# FIND-005: IDOR on user profiles\n\n" + "- **Linked Hypothesis:** H-002\n\n" + "## PoC\n\n" + "`GET /api/v1/users/{uuid}` unauthenticated returns any profile.\n", + encoding="utf-8", + ) + + result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY) + confirmed_ids = {d["id"] for d in result["confirmed_details"]} + assert "idor-user-profiles" in confirmed_ids, ( + f"reverse-link FIND confirmation failed, got {confirmed_ids}" + )