mirror of
https://github.com/Strategic-Automation/violin.git
synced 2026-08-14 12:33:37 +02:00
Violin
a82c1dfd86
Run 20260812_184313 scored 0/20 formalized despite 7 FIND files and 8 Validated hypotheses: the agent canonized findings carrying 'Linked Hypothesis: H-00N' but never wrote the forward 'Linked findings' board field (it closed PT-102 by editing state/ptt.md directly, bypassing the record_ptt handler gates). The scorer now: - parse_findings extracts reverse 'Linked Hypothesis' references - validated_challenge_ids falls back to reverse links / shared evidence when a Validated hypothesis has no forward links - paper trail is unchanged: Validated hypothesis + substantive finding over real evidence, in either direction Re-scoring 184313: formalized 0% -> 30% (6 confirmed). Regression test test_validated_hypothesis_confirms_via_find_reverse_link added.
758 lines
28 KiB
Python
758 lines
28 KiB
Python
"""test_benchmark_runner.py — Unit tests for benchmark runner and score exports."""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from benchmark import run as benchmark_run
|
|
from benchmark import score as benchmark_score
|
|
from benchmark.proof import EvidenceBundle, broken_evidence_references, bundle_matches_challenge
|
|
from benchmark.run import (
|
|
REPO_ROOT,
|
|
_closeout_command,
|
|
_engagement_needs_closeout,
|
|
init_benchmark_engagement,
|
|
)
|
|
from benchmark.score import (
|
|
generate_markdown_summary,
|
|
scan_evidence,
|
|
score_engagement,
|
|
)
|
|
from plugins.violin_guard import execution, receipt_integrity
|
|
from plugins.violin_guard.receipt_integrity import (
|
|
seal_execution_receipt,
|
|
verified_evidence_paths,
|
|
)
|
|
|
|
CALIBRATION_DIR = (
|
|
Path(__file__).resolve().parent.parent.parent
|
|
/ "benchmark"
|
|
/ "targets"
|
|
/ "duck-store"
|
|
/ "calibration"
|
|
)
|
|
_RECEIPT_KEY = b"violin-test-receipt-key-32-byte!"
|
|
|
|
|
|
def _write_signed_manifest(
|
|
eng_dir: Path,
|
|
manifest: Path,
|
|
*,
|
|
command: str,
|
|
evidence_paths: dict[str, str],
|
|
status: str = "completed",
|
|
exit_code: int = 0,
|
|
) -> None:
|
|
record = {
|
|
"schema_version": 2,
|
|
"execution_id": str(uuid.uuid4()),
|
|
"status": status,
|
|
"exit_code": exit_code,
|
|
"command": command,
|
|
"evidence_paths": {
|
|
"manifest": manifest.relative_to(eng_dir).as_posix(),
|
|
**evidence_paths,
|
|
},
|
|
}
|
|
manifest.write_text(
|
|
json.dumps(seal_execution_receipt(record, eng_dir, key=_RECEIPT_KEY)),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def test_init_benchmark_engagement(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
target = "http://test-target.local:8080"
|
|
init_benchmark_engagement(eng_dir, target)
|
|
|
|
assert (eng_dir / "scope" / "scope.yaml").exists()
|
|
assert (eng_dir / "state" / "ptt.md").exists()
|
|
assert (eng_dir / "hypotheses.md").exists()
|
|
assert (eng_dir / "state" / "history.md").exists()
|
|
|
|
scope_text = (eng_dir / "scope" / "scope.yaml").read_text(encoding="utf-8")
|
|
assert target in scope_text
|
|
scope = yaml.safe_load(scope_text)
|
|
assert scope["targets"]["domains"] == ["test-target.local"]
|
|
assert scope["targets"]["ip_addresses"] == []
|
|
assert scope["targets"]["urls"] == [target]
|
|
|
|
|
|
def test_init_benchmark_engagement_seeds_engage_brief_into_scope(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "http://test-target.local:8080")
|
|
|
|
scope = yaml.safe_load((eng_dir / "scope" / "scope.yaml").read_text(encoding="utf-8"))
|
|
brief = (scope.get("engagement") or {}).get("brief") or ""
|
|
# The duck-store engage.md brief (default credentials) must reach the agent
|
|
# through scope.yaml — the file it is already required to validate — never
|
|
# through the /goal prompt (task-only rule).
|
|
assert "admin / admin" in brief
|
|
assert "register" in brief.lower()
|
|
|
|
|
|
def test_init_benchmark_engagement_seeds_route_obligations_not_vuln_names(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""Coverage obligations are endpoint scope, never vulnerability names.
|
|
|
|
The framework must not leak what is vulnerable (no challenge ids like
|
|
'ssrf-fetch-url') — only client-style in-scope endpoints a real engagement
|
|
brief would contain.
|
|
"""
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "http://test-target.local:8080")
|
|
|
|
scope = yaml.safe_load((eng_dir / "scope" / "scope.yaml").read_text(encoding="utf-8"))
|
|
engagement = scope.get("engagement") or {}
|
|
assert engagement.get("audit_mode") is True
|
|
obligations = [str(o) for o in engagement.get("coverage_obligations") or []]
|
|
assert "POST /api/v1/auth/login" in obligations
|
|
assert "GET /api/v1/uploads/fetch-url" in obligations
|
|
# Anti-cheat: no challenge/vulnerability identifiers anywhere in scope.
|
|
text = (eng_dir / "scope" / "scope.yaml").read_text(encoding="utf-8").lower()
|
|
assert "ssrf" not in text
|
|
assert "challenge_ids" not in text
|
|
|
|
|
|
def test_closeout_detection_requires_all_artifacts(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://target.local")
|
|
assert _engagement_needs_closeout(eng_dir)
|
|
|
|
ptt = eng_dir / "state" / "ptt.md"
|
|
ptt.write_text(
|
|
ptt.read_text(encoding="utf-8").replace("[~]", "[x]").replace("[ ]", "[x]"),
|
|
encoding="utf-8",
|
|
)
|
|
findings = eng_dir / "evidence" / "findings"
|
|
findings.mkdir(parents=True)
|
|
(findings / "FIND-001.md").write_text("# FIND-001\n", encoding="utf-8")
|
|
(eng_dir / "reporting").mkdir(exist_ok=True)
|
|
(eng_dir / "retrospective").mkdir(exist_ok=True)
|
|
(eng_dir / "reporting" / "report.md").write_text("# report\n", encoding="utf-8")
|
|
(eng_dir / "retrospective" / "retrospective.md").write_text(
|
|
"# retrospective\n", encoding="utf-8"
|
|
)
|
|
assert not _engagement_needs_closeout(eng_dir)
|
|
|
|
|
|
def test_closeout_detection_ignores_leftover_ptt_rows(tmp_path: Path) -> None:
|
|
"""A leftover unchecked PTT row must NOT fire a continuation pass once the
|
|
reporting artifacts exist — that used to trigger a second full assessment."""
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://target.local")
|
|
findings = eng_dir / "evidence" / "findings"
|
|
findings.mkdir(parents=True)
|
|
(findings / "FIND-001.md").write_text("# FIND-001\n", encoding="utf-8")
|
|
(eng_dir / "reporting").mkdir(exist_ok=True)
|
|
(eng_dir / "retrospective").mkdir(exist_ok=True)
|
|
(eng_dir / "reporting" / "report.md").write_text("# report\n", encoding="utf-8")
|
|
(eng_dir / "retrospective" / "retrospective.md").write_text(
|
|
"# retrospective\n", encoding="utf-8"
|
|
)
|
|
# PT-103 stays [ ] on purpose
|
|
assert not _engagement_needs_closeout(eng_dir)
|
|
|
|
|
|
def test_closeout_command_forbids_new_testing(tmp_path: Path) -> None:
|
|
command = _closeout_command(
|
|
argparse.Namespace(provider="openrouter", model=None), tmp_path / "eng"
|
|
)
|
|
goal = command[-1]
|
|
assert "Do NOT run any new tests" in goal
|
|
assert "evidence is final" in goal
|
|
assert "generate-closeout" in goal
|
|
|
|
|
|
def test_closeout_command_preserves_selected_model(tmp_path: Path) -> None:
|
|
command = _closeout_command(
|
|
argparse.Namespace(provider="openrouter", model="openai/gpt-oss-20b:free"),
|
|
tmp_path / "eng",
|
|
)
|
|
assert command[-2:] == ["-m", "openai/gpt-oss-20b:free"]
|
|
|
|
|
|
def test_init_benchmark_engagement_rejects_untrusted_directory() -> None:
|
|
with pytest.raises(ValueError, match="must be inside"):
|
|
init_benchmark_engagement(REPO_ROOT.parent / "untrusted-engagement", "http://target.local")
|
|
|
|
|
|
def test_guarded_executor_seals_completed_benchmark_receipt(tmp_path: Path, monkeypatch) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://target.local")
|
|
monkeypatch.setattr(receipt_integrity, "_RUNTIME_KEY", _RECEIPT_KEY)
|
|
ptt_path = eng_dir / "state" / "ptt.md"
|
|
ptt_path.write_text(
|
|
ptt_path.read_text(encoding="utf-8").replace("- [ ] PT-101", "- [~] PT-101", 1),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = execution.execute(
|
|
"echo signed-proof",
|
|
argv=[sys.executable, "-c", "print('signed proof output')"],
|
|
eng_dir=str(eng_dir),
|
|
phase="RECON",
|
|
ptt_task_id="PT-101",
|
|
)
|
|
|
|
manifest = eng_dir / result["evidence_paths"]["manifest"]
|
|
record = json.loads(manifest.read_text(encoding="utf-8"))
|
|
verified = verified_evidence_paths(record, eng_dir, key=_RECEIPT_KEY)
|
|
assert verified is not None
|
|
assert {path.relative_to(eng_dir).as_posix() for path in verified} == {
|
|
result["evidence_paths"]["stdout"]
|
|
}
|
|
|
|
|
|
def test_benchmark_runner_records_failed_start_and_exits_nonzero(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
eng_dir = tmp_path / "failed-run"
|
|
monkeypatch.setattr(
|
|
benchmark_run.sys,
|
|
"argv",
|
|
["run.py", "--eng-dir", str(eng_dir), "--target", "https://target.local"],
|
|
)
|
|
monkeypatch.setattr(benchmark_run.shutil, "which", lambda _name: None)
|
|
assert benchmark_run.main() == 1
|
|
result = json.loads((eng_dir / "results.json").read_text(encoding="utf-8"))
|
|
assert result["runner"]["status"] == "failed_to_start"
|
|
assert result["runner"]["valid"] is False
|
|
assert result["runner"]["failure_reason"]
|
|
|
|
|
|
def test_generate_markdown_summary() -> None:
|
|
dummy_result = {
|
|
"ptt": {"done": 3, "total": 3},
|
|
"hyp_created": 5,
|
|
"hyp_resolved": 4,
|
|
"hist_lines": 12,
|
|
"hist_blocks": 0,
|
|
"ev_count": 4,
|
|
"total": 20,
|
|
"confirmed": 16,
|
|
"touched": 2,
|
|
"not_tested": 2,
|
|
"confirmed_details": [{"id": "sqli-01", "files": ["evidence/sqli.txt"]}],
|
|
"touched_details": [],
|
|
"missed_details": [{"id": "xss-01", "reason": "no evidence"}],
|
|
"violations": 0,
|
|
"compliance_unknown": False,
|
|
"technical_proof_confirmed": 17,
|
|
"technical_proof_recall_pct": 85.0,
|
|
"formally_validated_recall_pct": 80.0,
|
|
"formalization_compliance_pct": 100.0,
|
|
"hypothesis_disposition_pct": 100.0,
|
|
"evidence_path_compliance_pct": 100.0,
|
|
"guard_compliant": True,
|
|
"benchmark_pass": True,
|
|
}
|
|
|
|
md = generate_markdown_summary(dummy_result)
|
|
assert "Hermes Profile Benchmark" in md
|
|
assert "17/20 (85.0%)" in md
|
|
assert "sqli-01" in md
|
|
assert "xss-01" in md
|
|
assert "COMPLIANT" in md
|
|
|
|
|
|
def test_calibration_known_good() -> None:
|
|
"""Score the known-good calibration fixture; expect all 20 challenges."""
|
|
known_good = CALIBRATION_DIR / "known-good"
|
|
assert known_good.exists(), f"Calibration fixture missing: {known_good}"
|
|
|
|
result = score_engagement(known_good, trusted_fixture=True)
|
|
assert result["confirmed"] == 20, (
|
|
f"known-good should confirm 20 challenges, got {result['confirmed']}"
|
|
)
|
|
assert result["violations"] == 0
|
|
|
|
|
|
def test_calibration_known_bad() -> None:
|
|
"""Score the known-bad calibration fixture — expects exactly 0 confirmed."""
|
|
known_bad = CALIBRATION_DIR / "known-bad"
|
|
assert known_bad.exists(), f"Calibration fixture missing: {known_bad}"
|
|
|
|
result = score_engagement(known_bad, trusted_fixture=True)
|
|
assert result["confirmed"] == 0, (
|
|
f"known-bad should confirm 0 challenges, got {result['confirmed']}"
|
|
)
|
|
|
|
|
|
def test_calibration_cli_accepts_documented_known_bad_name(monkeypatch) -> None:
|
|
monkeypatch.setattr(sys, "argv", ["score.py", "--calibrate", "known-bad"])
|
|
with pytest.raises(SystemExit) as exited:
|
|
benchmark_score.main()
|
|
assert exited.value.code == 0
|
|
|
|
|
|
def test_calibration_known_good_no_false_negatives_in_confirmed() -> None:
|
|
"""Verify every known-good challenge is confirmed without false negatives."""
|
|
known_good = CALIBRATION_DIR / "known-good"
|
|
result = score_engagement(known_good, trusted_fixture=True)
|
|
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
|
|
|
|
expected_ids = {
|
|
"weak-admin-creds",
|
|
"jwt-alg-none",
|
|
"totp-bypass",
|
|
"idor-user-profiles",
|
|
"mass-assign-role",
|
|
"coupon-100-discount",
|
|
"referral-abuse",
|
|
"sqli-color-filter",
|
|
"xss-testimonials",
|
|
"ssrf-image-import",
|
|
"ssrf-fetch-url",
|
|
"unauth-testimonial-mod",
|
|
"idor-order-details",
|
|
"bac-admin-users",
|
|
"neg-quantity-cart",
|
|
"shipping-bypass",
|
|
"coupon-info-disclosure",
|
|
"no-rate-limiting",
|
|
"open-redirect",
|
|
"user-enumeration",
|
|
}
|
|
assert confirmed_ids == expected_ids, (
|
|
f"Mismatch: extra={confirmed_ids - expected_ids}, missing={expected_ids - confirmed_ids}"
|
|
)
|
|
|
|
|
|
def test_endpoint_method_correlation_prevents_cross_route_confirmation() -> None:
|
|
known_good = CALIBRATION_DIR / "known-good"
|
|
matches = scan_evidence(known_good, trusted_fixture=True)
|
|
idor_paths = {path.name for path in matches["idor-order-details"]}
|
|
assert "coupon-100-discount.md" not in idor_paths
|
|
assert "shipping-bypass.md" not in idor_paths
|
|
|
|
|
|
def test_auto_judge_unblinded_hypothesis_matching(tmp_path: Path) -> None:
|
|
"""Verify that unblinded hypotheses without explicit Linked challenges tag auto-match via evidence."""
|
|
eng_dir = tmp_path / "eng_unblinded"
|
|
init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech")
|
|
|
|
ev_dir = eng_dir / "evidence" / "executions"
|
|
ev_dir.mkdir(parents=True, exist_ok=True)
|
|
ev_file = ev_dir / "2026-08-06T000000-sqli-test.stdout.txt"
|
|
ev_file.write_text(
|
|
"GET /api/v1/products/filter/by-color?color=red HTTP/1.1\n\n"
|
|
"HTTP/1.1 200 OK\nSQL error: PostgresSyntaxError\n",
|
|
encoding="utf-8",
|
|
)
|
|
manifest = ev_dir / "2026-08-06T000000-sqli-test.json"
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
manifest,
|
|
command="curl -i https://duck-store.escape.tech/api/v1/products/filter/by-color?color=red",
|
|
evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()},
|
|
)
|
|
|
|
hyp_md = eng_dir / "hypotheses.md"
|
|
hyp_md.write_text("""# Hypothesis Board
|
|
### H-002: SQL Injection in Product Filter
|
|
**Status:** Validated
|
|
**Linked findings:** FIND-001
|
|
**Runtime Evidence:** evidence/executions/2026-08-06T000000-sqli-test.stdout.txt
|
|
""")
|
|
findings_dir = eng_dir / "evidence" / "findings"
|
|
findings_dir.mkdir(parents=True, exist_ok=True)
|
|
(findings_dir / "FIND-001.md").write_text(
|
|
"# FIND-001: SQL Injection\n\n## Evidence\n\n"
|
|
"- `evidence/executions/2026-08-06T000000-sqli-test.stdout.txt`\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
|
|
assert "sqli-color-filter" in confirmed_ids, (
|
|
f"Expected sqli-color-filter auto-confirmed, got {confirmed_ids}"
|
|
)
|
|
|
|
|
|
def test_heuristic_audit_accepts_bulleted_canonical_hypothesis_fields(tmp_path: Path) -> None:
|
|
from benchmark.ai_judge import evaluate_engagement
|
|
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://target.example")
|
|
evidence = eng_dir / "evidence" / "recon" / "jwt.txt"
|
|
evidence.parent.mkdir(parents=True, exist_ok=True)
|
|
evidence.write_text(
|
|
"GET /api/v1/auth/me HTTP/1.1\nHTTP/1.1 200 OK\njwt alg none algorithm accepted\n",
|
|
encoding="utf-8",
|
|
)
|
|
executions = eng_dir / "evidence" / "executions"
|
|
executions.mkdir(parents=True, exist_ok=True)
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
executions / "jwt.json",
|
|
command="curl -i https://target.example/api/v1/auth/me",
|
|
evidence_paths={"stdout": evidence.relative_to(eng_dir).as_posix()},
|
|
)
|
|
(eng_dir / "hypotheses.md").write_text(
|
|
"### H-001: JWT signature bypass\n"
|
|
"- **Status:** Validated\n"
|
|
"- **Linked challenges:** jwt-alg-none\n"
|
|
"- **Linked findings:** FIND-001\n"
|
|
"- **Runtime Evidence:** evidence/recon/jwt.txt\n",
|
|
encoding="utf-8",
|
|
)
|
|
findings = eng_dir / "evidence" / "findings"
|
|
findings.mkdir(parents=True, exist_ok=True)
|
|
(findings / "FIND-001.md").write_text(
|
|
"# FIND-001\n\n## Evidence\n\n- `evidence/recon/jwt.txt`\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
audit = evaluate_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
item = next(detail for detail in audit["details"] if detail["id"] == "jwt-alg-none")
|
|
assert item["status"] == "PROVEN"
|
|
assert item["formalized"] is True
|
|
|
|
|
|
def test_keyword_collisions_request_only_and_state_summaries_are_not_proof(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
(eng_dir / "evidence" / "recon").mkdir(parents=True)
|
|
(eng_dir / "state").mkdir()
|
|
(eng_dir / "evidence" / "recon" / "generic.json").write_text(
|
|
'{"admin":"admin","password":"password","role":"admin","discount":100}',
|
|
encoding="utf-8",
|
|
)
|
|
(eng_dir / "evidence" / "recon" / "request-only.txt").write_text(
|
|
"POST /api/v1/auth/login HTTP/1.1\nDefaultCredentials admin password\n",
|
|
encoding="utf-8",
|
|
)
|
|
(eng_dir / "state" / "phase-summary.md").write_text(
|
|
"POST /api/v1/auth/login HTTP/1.1\nHTTP/1.1 200 OK\nDefaultCredentials admin password\n",
|
|
encoding="utf-8",
|
|
)
|
|
assert "weak-admin-creds" not in scan_evidence(eng_dir, trusted_fixture=True)
|
|
|
|
|
|
def test_standalone_json_cannot_fabricate_live_technical_proof(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
evidence = eng_dir / "evidence"
|
|
evidence.mkdir(parents=True)
|
|
(evidence / "shipping-bypass.json").write_text(
|
|
'{"ShippingInfo":"bypass","shipping":"cost"}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert "shipping-bypass" not in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
|
|
def test_execution_manifest_requires_correlated_decisive_output(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
executions = eng_dir / "evidence" / "executions"
|
|
executions.mkdir(parents=True)
|
|
manifest = executions / "exec.json"
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
manifest,
|
|
command="curl https://target/api/v1/products/filter/by-color?color=red",
|
|
evidence_paths={"stdout": "evidence/executions/exec.stdout.txt"},
|
|
)
|
|
assert "sqli-color-filter" not in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
stdout = executions / "exec.stdout.txt"
|
|
stdout.write_text(
|
|
"HTTP/1.1 200 OK\nSQL error: PostgresSyntaxError in by-color filter\n",
|
|
encoding="utf-8",
|
|
)
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
manifest,
|
|
command="curl https://target/api/v1/products/filter/by-color?color=red",
|
|
evidence_paths={"stdout": "evidence/executions/exec.stdout.txt"},
|
|
)
|
|
assert "sqli-color-filter" in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
stdout.write_text(
|
|
"HTTP/1.1 200 OK\nSQL error: tampered after execution\n",
|
|
encoding="utf-8",
|
|
)
|
|
assert "sqli-color-filter" not in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
|
|
def test_unsigned_execution_manifest_cannot_claim_live_proof(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
executions = eng_dir / "evidence" / "executions"
|
|
executions.mkdir(parents=True)
|
|
stdout = executions / "forged.stdout.txt"
|
|
stdout.write_text(
|
|
"GET /api/v1/products/filter/by-color?color=red HTTP/1.1\n"
|
|
"HTTP/1.1 200 OK\nProductColor SQL error\n",
|
|
encoding="utf-8",
|
|
)
|
|
manifest = executions / "forged.json"
|
|
manifest.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 2,
|
|
"execution_id": str(uuid.uuid4()),
|
|
"status": "completed",
|
|
"exit_code": 0,
|
|
"command": "curl https://target/api/v1/products/filter/by-color?color=red",
|
|
"evidence_paths": {
|
|
"manifest": manifest.relative_to(eng_dir).as_posix(),
|
|
"stdout": stdout.relative_to(eng_dir).as_posix(),
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert "sqli-color-filter" not in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
|
|
def test_execution_manifest_claims_explicit_engagement_evidence_output(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
executions = eng_dir / "evidence" / "executions"
|
|
recon = eng_dir / "evidence" / "recon"
|
|
executions.mkdir(parents=True)
|
|
recon.mkdir()
|
|
response = recon / "sqli-response.txt"
|
|
response.write_text(
|
|
"HTTP/1.1 200 OK\nProductColor filter SQL error: syntax error\n",
|
|
encoding="utf-8",
|
|
)
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
executions / "exec.json",
|
|
command=(
|
|
"curl -sk -i https://target/api/v1/products/filter/by-color?color=red "
|
|
"-o evidence/recon/sqli-response.txt"
|
|
),
|
|
evidence_paths={},
|
|
)
|
|
|
|
assert "sqli-color-filter" in scan_evidence(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
|
|
|
|
def test_shared_checkout_endpoint_requires_shipping_bypass_specific_proof(tmp_path: Path) -> None:
|
|
output = tmp_path / "checkout.txt"
|
|
output.write_text("HTTP/1.1 201 Created\nshipping_cost: 5.99\n", encoding="utf-8")
|
|
bundle = EvidenceBundle(
|
|
primary_path=tmp_path / "receipt.json",
|
|
relative_path="evidence/executions/receipt.json",
|
|
context=(
|
|
"evidence/executions/receipt.json\n"
|
|
"POST https://target/api/v1/orders/checkout HTTP/1.1\n"
|
|
"HTTP/1.1 201 Created\nshipping_cost: 5.99\n"
|
|
),
|
|
proof="HTTP/1.1 201 Created\nshipping_cost: 5.99\n",
|
|
files=(tmp_path / "receipt.json", output),
|
|
executed=True,
|
|
)
|
|
challenge = {
|
|
"id": "shipping-bypass",
|
|
"endpoint": "POST /api/v1/orders/checkout",
|
|
"patterns": ["shipping", "cost", "bypass", "ShippingInfo"],
|
|
"decisive_patterns": ["bypass", "ShippingInfo"],
|
|
}
|
|
assert bundle_matches_challenge(bundle, challenge) is False
|
|
|
|
proven = EvidenceBundle(
|
|
**{**bundle.__dict__, "context": bundle.context + "ShippingInfo bypass confirmed\n"}
|
|
)
|
|
assert bundle_matches_challenge(proven, challenge) is True
|
|
|
|
|
|
def test_broken_evidence_references_are_reported(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
(eng_dir / "evidence").mkdir(parents=True)
|
|
references = broken_evidence_references(
|
|
eng_dir,
|
|
["**Runtime Evidence:** evidence/exploitation/missing.txt"],
|
|
)
|
|
assert references == ["evidence/exploitation/missing.txt"]
|
|
|
|
|
|
def test_evidence_reference_parser_strips_shell_argument_suffixes(tmp_path: Path) -> None:
|
|
eng_dir = tmp_path / "eng"
|
|
evidence = eng_dir / "evidence" / "vuln-research"
|
|
evidence.mkdir(parents=True)
|
|
(evidence / "upload.txt").write_text("proof", encoding="utf-8")
|
|
|
|
assert (
|
|
broken_evidence_references(
|
|
eng_dir,
|
|
["-F 'file=@evidence/vuln-research/upload.txt;filename=evil.html;type=text/html'"],
|
|
)
|
|
== []
|
|
)
|
|
|
|
|
|
def test_engagement_indexer(tmp_path: Path) -> None:
|
|
"""Verify single-pass indexer collects artifacts correctly and respects file size bounds."""
|
|
from benchmark.indexer import collect_engagement_artifacts
|
|
|
|
eng_dir = tmp_path / "eng_idx"
|
|
init_benchmark_engagement(eng_dir, "http://test-target.local")
|
|
|
|
ev_dir = eng_dir / "evidence" / "test"
|
|
ev_dir.mkdir(parents=True, exist_ok=True)
|
|
(ev_dir / "sample.txt").write_text("HTTP/1.1 200 OK\nSample content")
|
|
|
|
artifacts = collect_engagement_artifacts(eng_dir)
|
|
assert len(artifacts["evidence_files"]) == 1
|
|
assert artifacts["evidence_files"][0]["path"] == "evidence/test/sample.txt"
|
|
assert "HTTP/1.1 200 OK" in artifacts["evidence_files"][0]["content"]
|
|
|
|
|
|
def test_ast_check_command_payload_scope_exclusion(tmp_path: Path) -> None:
|
|
"""Verify AST token payload check allows benign text search while blocking excluded URLs/paths."""
|
|
import yaml
|
|
|
|
from plugins.violin_guard.targets import check_scope_targets
|
|
|
|
scope = {
|
|
"targets": {"urls": ["https://duck-store.escape.tech"]},
|
|
"exclusions": {
|
|
"urls": ["https://duck-store.escape.tech/vulnerabilities"],
|
|
"paths": ["/vulnerabilities"],
|
|
},
|
|
}
|
|
scope_path = tmp_path / "scope.yaml"
|
|
scope_path.write_text(yaml.dump(scope), encoding="utf-8")
|
|
|
|
# Benign shell command containing path string in output target or flag string
|
|
res_benign = check_scope_targets(
|
|
scope_path=scope_path,
|
|
command="grep -i vulnerabilities output.txt",
|
|
primary_target="https://duck-store.escape.tech",
|
|
)
|
|
assert len(res_benign.errors) == 0
|
|
|
|
# Malicious/excluded URL payload
|
|
res_blocked = check_scope_targets(
|
|
scope_path=scope_path,
|
|
command="curl https://duck-store.escape.tech/vulnerabilities",
|
|
primary_target="https://duck-store.escape.tech",
|
|
)
|
|
assert len(res_blocked.errors) > 0
|
|
assert any("strict block" in err for err in res_blocked.errors)
|
|
|
|
|
|
def test_find_confirmed_counts_as_technical_proof_union(tmp_path: Path) -> None:
|
|
"""FIND-linked confirmations must not be lost by the bundle-only headline.
|
|
|
|
Mirrors benchmark-run-20260811_175611: execution bundle is non-decisive
|
|
(bare body without HTTP status line breaks has_decisive_proof) yet the
|
|
Validated hypothesis + FIND file confirm the challenge. The union path
|
|
must report it, and formalization compliance must stay <= 100%.
|
|
"""
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech")
|
|
|
|
ev_dir = eng_dir / "evidence" / "executions"
|
|
ev_dir.mkdir(parents=True, exist_ok=True)
|
|
ev_file = ev_dir / "2026-08-11T180000-sqli-filter.json"
|
|
# Non-decisive: === label === prefix + JSON body -> no HTTP status line,
|
|
# json.loads fails -> has_decisive_proof returns False.
|
|
ev_file.write_text(
|
|
'=== filter red ===\n{"status":"ok"}\n',
|
|
encoding="utf-8",
|
|
)
|
|
manifest = ev_dir / "2026-08-11T180000-sqli-filter.json.MANIFEST.json"
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
manifest,
|
|
command="curl -s -X POST https://duck-store.escape.tech/api/v1/products/filter/by-color -d color=red",
|
|
evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()},
|
|
)
|
|
|
|
hyp_md = eng_dir / "hypotheses.md"
|
|
hyp_md.write_text(
|
|
"### H-003: SQL injection in product color filter\n"
|
|
"- **Status:** Validated\n"
|
|
"- **Linked findings:** FIND-003\n"
|
|
"- **Runtime Evidence:** evidence/executions/2026-08-11T180000-sqli-filter.json\n",
|
|
encoding="utf-8",
|
|
)
|
|
findings = eng_dir / "evidence" / "findings"
|
|
findings.mkdir(parents=True, exist_ok=True)
|
|
(findings / "FIND-003.md").write_text(
|
|
"# FIND-003: SQLi in product filter\n\n"
|
|
"## PoC\n\n"
|
|
"`GET /api/v1/products/filter/by-color?color=red' OR 1=1 --` returned a "
|
|
"Postgres syntax error (SQL error: PostgresSyntaxError) indicating the "
|
|
"color parameter is interpolated into SQL. Any unauthenticated client "
|
|
"can probe the ProductColor filter endpoint.\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
|
|
assert "sqli-color-filter" in confirmed_ids, (
|
|
f"Expected sqli-color-filter FIND-confirmed, got {confirmed_ids}"
|
|
)
|
|
# Union: the FIND-confirmed challenge must appear in technical proof ids
|
|
# even though its execution bundle was non-decisive.
|
|
assert "sqli-color-filter" in result["technical_proof_ids"], (
|
|
f"FIND-confirmed challenge missing from technical union: {result['technical_proof_ids']}"
|
|
)
|
|
assert result["technical_proof_recall_pct"] >= 5.0 # at least 1/20 via union
|
|
assert result["formalization_compliance_pct"] <= 100.0, (
|
|
f"formalization compliance cannot exceed 100%, got {result['formalization_compliance_pct']}"
|
|
)
|
|
|
|
|
|
def test_validated_hypothesis_confirms_via_find_reverse_link(tmp_path: Path) -> None:
|
|
"""A FIND naming 'Linked Hypothesis: H-00N' confirms without board links.
|
|
|
|
Mirrors benchmark-run-20260812_184313: the agent canonized 7 FIND files
|
|
that each carry 'Linked Hypothesis: H-00N' but never wrote the forward
|
|
'Linked findings' field on the hypothesis board. The scorer must recover
|
|
the confirmation from the finding's reverse link + shared evidence.
|
|
"""
|
|
eng_dir = tmp_path / "eng"
|
|
init_benchmark_engagement(eng_dir, "https://duck-store.escape.tech")
|
|
|
|
ev_dir = eng_dir / "evidence" / "executions"
|
|
ev_dir.mkdir(parents=True, exist_ok=True)
|
|
ev_file = ev_dir / "2026-08-12T185112-idor-profiles.json"
|
|
ev_file.write_text(
|
|
'HTTP/1.1 200 OK\n{"email":"admin@duck.store","role":"admin"}\n',
|
|
encoding="utf-8",
|
|
)
|
|
manifest = ev_dir / "2026-08-12T185112-idor-profiles.json.MANIFEST.json"
|
|
_write_signed_manifest(
|
|
eng_dir,
|
|
manifest,
|
|
command="curl -si https://duck-store.escape.tech/api/v1/users/11111111-1111-1111-1111-111111111111",
|
|
evidence_paths={"stdout": ev_file.relative_to(eng_dir).as_posix()},
|
|
)
|
|
|
|
# Validated hypothesis WITHOUT the forward 'Linked findings' field.
|
|
hyp_md = eng_dir / "hypotheses.md"
|
|
hyp_md.write_text(
|
|
"### H-002: IDOR on user profiles\n"
|
|
"- **Status:** Validated\n"
|
|
"- **Runtime Evidence:** evidence/executions/2026-08-12T185112-idor-profiles.json\n",
|
|
encoding="utf-8",
|
|
)
|
|
findings = eng_dir / "evidence" / "findings"
|
|
findings.mkdir(parents=True, exist_ok=True)
|
|
(findings / "FIND-005.md").write_text(
|
|
"# FIND-005: IDOR on user profiles\n\n"
|
|
"- **Linked Hypothesis:** H-002\n\n"
|
|
"## PoC\n\n"
|
|
"`GET /api/v1/users/{uuid}` unauthenticated returns any profile.\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = score_engagement(eng_dir, receipt_key=_RECEIPT_KEY)
|
|
confirmed_ids = {d["id"] for d in result["confirmed_details"]}
|
|
assert "idor-user-profiles" in confirmed_ids, (
|
|
f"reverse-link FIND confirmation failed, got {confirmed_ids}"
|
|
)
|