mirror of
https://github.com/Strategic-Automation/violin.git
synced 2026-08-14 12:33:37 +02:00
fix(score): confirm findings via reverse hypothesis links
Run 20260812_184313 scored 0/20 formalized despite 7 FIND files and 8 Validated hypotheses: the agent canonized findings carrying 'Linked Hypothesis: H-00N' but never wrote the forward 'Linked findings' board field (it closed PT-102 by editing state/ptt.md directly, bypassing the record_ptt handler gates). The scorer now: - parse_findings extracts reverse 'Linked Hypothesis' references - validated_challenge_ids falls back to reverse links / shared evidence when a Validated hypothesis has no forward links - paper trail is unchanged: Validated hypothesis + substantive finding over real evidence, in either direction Re-scoring 184313: formalized 0% -> 30% (6 confirmed). Regression test test_validated_hypothesis_confirms_via_find_reverse_link added.
This commit is contained in:
+29
-1
@@ -124,11 +124,26 @@ def parse_findings(eng_dir: Path) -> list[dict]:
|
||||
continue
|
||||
fid = fpath.stem
|
||||
evidence_files: set[str] = set()
|
||||
linked_hypotheses: set[str] = set()
|
||||
for line in txt.splitlines():
|
||||
if "evidence/" in line:
|
||||
for part in re.findall(r"evidence/[^\s,)`\]]+", line):
|
||||
evidence_files.add(Path(part).name)
|
||||
results.append({"id": fid, "evidence_files": evidence_files, "text": txt})
|
||||
hyp_link = re.search(
|
||||
r"linked\s+hypothes[ie]s?\s*:?\s*[*:\-]*\s*(H-\d+)",
|
||||
line,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if hyp_link:
|
||||
linked_hypotheses.add(hyp_link.group(1).upper())
|
||||
results.append(
|
||||
{
|
||||
"id": fid,
|
||||
"evidence_files": evidence_files,
|
||||
"linked_hypotheses": linked_hypotheses,
|
||||
"text": txt,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
@@ -140,6 +155,15 @@ def validated_challenge_ids(
|
||||
"""Return proof-linked IDs with a Validated hypothesis and canonical finding."""
|
||||
ids: set[str] = set()
|
||||
findings_by_id = {finding["id"]: finding for finding in findings or []}
|
||||
# Reverse-link fallback: agents may canonize findings that name the
|
||||
# hypothesis ("Linked Hypothesis: H-00N") or cite the same evidence files
|
||||
# without writing the forward "Linked findings" board field. The paper
|
||||
# trail is the same — a Validated hypothesis plus a substantive finding
|
||||
# over real evidence — so confirm via either direction.
|
||||
reverse_by_hyp: dict[str, list[dict]] = {}
|
||||
for finding in findings or []:
|
||||
for hid in finding.get("linked_hypotheses") or set():
|
||||
reverse_by_hyp.setdefault(hid, []).append(finding)
|
||||
for h in hypotheses:
|
||||
if h["status"].strip().lower() != "validated":
|
||||
continue
|
||||
@@ -148,11 +172,15 @@ def validated_challenge_ids(
|
||||
for finding_id in h.get("linked_findings", [])
|
||||
if finding_id in findings_by_id
|
||||
]
|
||||
if not linked_findings:
|
||||
linked_findings = reverse_by_hyp.get(h["id"].upper(), [])
|
||||
if not linked_findings:
|
||||
continue
|
||||
cited_files = set(h.get("evidence_files", set()))
|
||||
for finding in linked_findings:
|
||||
cited_files.update(finding.get("evidence_files", set()))
|
||||
if not cited_files and finding.get("linked_hypotheses"):
|
||||
cited_files.update(h.get("evidence_files", set()))
|
||||
ids.update(h["linked"])
|
||||
for challenge_id, proof_files in (evidence_hits or {}).items():
|
||||
if {path.name for path in proof_files}.intersection(cited_files):
|
||||
|
||||
@@ -704,3 +704,54 @@ def test_find_confirmed_counts_as_technical_proof_union(tmp_path: Path) -> None:
|
||||
assert result["formalization_compliance_pct"] <= 100.0, (
|
||||
f"formalization compliance cannot exceed 100%, got {result['formalization_compliance_pct']}"
|
||||
)
|
||||
|
||||
|
||||
def test_validated_hypothesis_confirms_via_find_reverse_link(tmp_path: Path) -> None:
|
||||
"""A FIND naming 'Linked Hypothesis: H-00N' confirms without board links.
|
||||
|
||||
Mirrors benchmark-run-20260812_184313: the agent canonized 7 FIND files
|
||||
that each carry 'Linked Hypothesis: H-00N' but never wrote the forward
|
||||
'Linked findings' field on the hypothesis board. The scorer must recover
|
||||
the confirmation from the finding's reverse link + shared evidence.
|
||||
"""
|
||||
eng_dir = tmp_path / "eng"
|
||||
init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech")
|
||||
|
||||
ev_dir = eng_dir / "evidence" / "executions"
|
||||
ev_dir.mkdir(parents=True, exist_ok=True)
|
||||
ev_file = ev_dir / "2026-08-12T185112-idor-profiles.json"
|
||||
ev_file.write_text(
|
||||
'HTTP/1.1 200 OK\n{"email":"admin@duck.store","role":"admin"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = ev_dir / "2026-08-12T185112-idor-profiles.json.MANIFEST.json"
|
||||
_write_signed_manifest(
|
||||
eng_dir,
|
||||
manifest,
|
||||
command="curl -si https://duck-store.escape.tech/api/v1/users/11111111-1111-1111-1111-111111111111",
|
||||
evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()},
|
||||
)
|
||||
|
||||
# Validated hypothesis WITHOUT the forward 'Linked findings' field.
|
||||
hyp_md = eng_dir / "hypotheses.md"
|
||||
hyp_md.write_text(
|
||||
"### H-002: IDOR on user profiles\n"
|
||||
"- **Status:** Validated\n"
|
||||
"- **Runtime Evidence:** evidence/executions/2026-08-12T185112-idor-profiles.json\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
findings = eng_dir / "evidence" / "findings"
|
||||
findings.mkdir(parents=True, exist_ok=True)
|
||||
(findings / "FIND-005.md").write_text(
|
||||
"# FIND-005: IDOR on user profiles\n\n"
|
||||
"- **Linked Hypothesis:** H-002\n\n"
|
||||
"## PoC\n\n"
|
||||
"`GET /api/v1/users/{uuid}` unauthenticated returns any profile.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
|
||||
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
|
||||
assert "idor-user-profiles" in confirmed_ids, (
|
||||
f"reverse-link FIND confirmation failed, got {confirmed_ids}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user