fix(score): confirm findings via reverse hypothesis links

Run 20260812_184313 scored 0/20 formalized despite 7 FIND files and 8
Validated hypotheses: the agent canonized findings carrying 'Linked
Hypothesis: H-00N' but never wrote the forward 'Linked findings' board
field (it closed PT-102 by editing state/ptt.md directly, bypassing the
record_ptt handler gates). The scorer now:

- parse_findings extracts reverse 'Linked Hypothesis' references
- validated_challenge_ids falls back to reverse links / shared evidence
  when a Validated hypothesis has no forward links
- paper trail is unchanged: Validated hypothesis + substantive finding
  over real evidence, in either direction

Re-scoring 184313: formalized 0% -> 30% (6 confirmed). Regression test
test_validated_hypothesis_confirms_via_find_reverse_link added.
This commit is contained in:
Violin
2026-08-12 21:32:56 +01:00
parent 723e349291
commit a82c1dfd86
2 changed files with 80 additions and 1 deletions
+29 -1
View File
@@ -124,11 +124,26 @@ def parse_findings(eng_dir: Path) -> list[dict]:
continue
fid = fpath.stem
evidence_files: set[str] = set()
linked_hypotheses: set[str] = set()
for line in txt.splitlines():
if "evidence/" in line:
for part in re.findall(r"evidence/[^\s,)`\]]+", line):
evidence_files.add(Path(part).name)
results.append({"id": fid, "evidence_files": evidence_files, "text": txt})
hyp_link = re.search(
r"linked\s+hypothes[ie]s?\s*:?\s*[*:\-]*\s*(H-\d+)",
line,
re.IGNORECASE,
)
if hyp_link:
linked_hypotheses.add(hyp_link.group(1).upper())
results.append(
{
"id": fid,
"evidence_files": evidence_files,
"linked_hypotheses": linked_hypotheses,
"text": txt,
}
)
return results
@@ -140,6 +155,15 @@ def validated_challenge_ids(
"""Return proof-linked IDs with a Validated hypothesis and canonical finding."""
ids: set[str] = set()
findings_by_id = {finding["id"]: finding for finding in findings or []}
# Reverse-link fallback: agents may canonize findings that name the
# hypothesis ("Linked Hypothesis: H-00N") or cite the same evidence files
# without writing the forward "Linked findings" board field. The paper
# trail is the same — a Validated hypothesis plus a substantive finding
# over real evidence — so confirm via either direction.
reverse_by_hyp: dict[str, list[dict]] = {}
for finding in findings or []:
for hid in finding.get("linked_hypotheses") or set():
reverse_by_hyp.setdefault(hid, []).append(finding)
for h in hypotheses:
if h["status"].strip().lower() != "validated":
continue
@@ -148,11 +172,15 @@ def validated_challenge_ids(
for finding_id in h.get("linked_findings", [])
if finding_id in findings_by_id
]
if not linked_findings:
linked_findings = reverse_by_hyp.get(h["id"].upper(), [])
if not linked_findings:
continue
cited_files = set(h.get("evidence_files", set()))
for finding in linked_findings:
cited_files.update(finding.get("evidence_files", set()))
if not cited_files and finding.get("linked_hypotheses"):
cited_files.update(h.get("evidence_files", set()))
ids.update(h["linked"])
for challenge_id, proof_files in (evidence_hits or {}).items():
if {path.name for path in proof_files}.intersection(cited_files):
+51
View File
@@ -704,3 +704,54 @@ def test_find_confirmed_counts_as_technical_proof_union(tmp_path: Path) -> None:
assert result["formalization_compliance_pct"] <= 100.0, (
f"formalization compliance cannot exceed 100%, got {result['formalization_compliance_pct']}"
)
def test_validated_hypothesis_confirms_via_find_reverse_link(tmp_path: Path) -> None:
"""A FIND naming 'Linked Hypothesis: H-00N' confirms without board links.
Mirrors benchmark-run-20260812_184313: the agent canonized 7 FIND files
that each carry 'Linked Hypothesis: H-00N' but never wrote the forward
'Linked findings' field on the hypothesis board. The scorer must recover
the confirmation from the finding's reverse link + shared evidence.
"""
eng_dir = tmp_path / "eng"
init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech")
ev_dir = eng_dir / "evidence" / "executions"
ev_dir.mkdir(parents=True, exist_ok=True)
ev_file = ev_dir / "2026-08-12T185112-idor-profiles.json"
ev_file.write_text(
'HTTP/1.1 200 OK\n{"email":"admin@duck.store","role":"admin"}\n',
encoding="utf-8",
)
manifest = ev_dir / "2026-08-12T185112-idor-profiles.json.MANIFEST.json"
_write_signed_manifest(
eng_dir,
manifest,
command="curl -si https://duck-store.escape.tech/api/v1/users/11111111-1111-1111-1111-111111111111",
evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()},
)
# Validated hypothesis WITHOUT the forward 'Linked findings' field.
hyp_md = eng_dir / "hypotheses.md"
hyp_md.write_text(
"### H-002: IDOR on user profiles\n"
"- **Status:** Validated\n"
"- **Runtime Evidence:** evidence/executions/2026-08-12T185112-idor-profiles.json\n",
encoding="utf-8",
)
findings = eng_dir / "evidence" / "findings"
findings.mkdir(parents=True, exist_ok=True)
(findings / "FIND-005.md").write_text(
"# FIND-005: IDOR on user profiles\n\n"
"- **Linked Hypothesis:** H-002\n\n"
"## PoC\n\n"
"`GET /api/v1/users/{uuid}` unauthenticated returns any profile.\n",
encoding="utf-8",
)
result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
assert "idor-user-profiles" in confirmed_ids, (
f"reverse-link FIND confirmation failed, got {confirmed_ids}"
)