mirror of
https://github.com/Strategic-Automation/violin.git
synced 2026-08-14 12:33:37 +02:00
feat(guard): generated closeout artifacts + framework-owned methodology
- generate-closeout CLI derives findings.yaml and report.md from canonical FIND-NNN.md files; agent closeout reduced to executive-summary narrative - Per-finding recording cut from 4 surfaces to 2 (hypothesis + FIND) - New skills/pentest/references/exhaustive-coverage.md mandated from SKILL.md; Operational Contract (SKILL.md S7) and Citation Discipline (evidence ref S8) - Benchmark goal/closeout prompts reduced to task + integrity constraint Tests: 233 passed (7 new); live-verified against real Aug 10 run (10/10 findings, ids+severities match hand-written original).
This commit is contained in:
@@ -3,6 +3,14 @@
|
||||
## 3.1.0 (Unreleased)
|
||||
|
||||
### Automated Benchmark & Evaluation Framework
|
||||
- Closeout artifacts are now generated, not hand-written: `python scripts/violin_guard.py generate-closeout --eng-dir <dir> --target <target>` derives `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections) from canonical `evidence/findings/FIND-NNN.md` files. Agent closeout duty reduced to the executive-summary narrative — cuts per-finding recording from 4 surfaces to 2 and eliminates the duplicate rewriting that starved exploitation budget and timed out closeout.
|
||||
- Benchmark goal/closeout prompts reduced to task-only (target + integrity constraint); methodology moved into the framework — new `skills/pentest/references/exhaustive-coverage.md` (coverage matrix, don't-stop-at-first-finding, route discovery, authz/business-logic boundaries, FIND-NNN canonization) mandated from `skills/pentest/SKILL.md` §3, plus an Operational Contract in §7 and Citation Discipline in `evidence-and-verification-discipline.md` §8.
|
||||
- Evidence-gated scorer now confirms Validated hypotheses via canonical `evidence/findings/FIND-NNN.md` files matched in live-app terms (METHOD + endpoint route, with `/api/v1` optional) in addition to execution bundles, so truncated `tail -c` evidence no longer loses real findings.
|
||||
- Endpoint+method proof bundles are accepted with a single pattern hit; shared-endpoint challenges (auth/login, checkout, register) are gated by explicit `require`/`require_any` discriminators (role-granted admin, credit granted, negative quantity, external redirect, rate-limit attempt evidence) to prevent cross-challenge false positives.
|
||||
- Absence-type challenges (`no-rate-limiting`) are scored on repeated identical status codes with no 429/throttle signal instead of requiring the counter-token.
|
||||
- `neg-quantity-cart` accepts both `PUT /api/v1/cart/items/{id}` and `POST /api/v1/cart/add` vectors.
|
||||
- Runner no longer invalidates a run when the closeout continuation pass times out or exits non-zero; closeout is reported as a soft `closeout_warning` while scoring proceeds on substantive evidence.
|
||||
- Schema-drift audit whitelists legitimate `state/report.md` and `state/retrospective.md` closeout artifacts.
|
||||
- Introduced automated Hermes profile benchmark runner engine (`benchmark/run.py`) and AI-assisted evaluation framework (`benchmark/ai_judge.py`).
|
||||
- Added `benchmark/indexer.py` for single-pass engagement directory artifact indexing with a 2MB per-file resource limit and cross-platform POSIX path normalization (`.as_posix()`).
|
||||
- Optimized `benchmark/score.py` pattern matching disk I/O to single-pass $O(F)$ scanning and removed late function-scope imports.
|
||||
|
||||
@@ -6,6 +6,8 @@ import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from . import hypotheses, state
|
||||
|
||||
_FINDING_ID_RE = re.compile(r"FIND-(\d{3,})$")
|
||||
@@ -203,3 +205,180 @@ def _create_from_pending_batch(
|
||||
"batch_id": batch_id,
|
||||
"reused": False,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Closeout artifact generation (FIND parsing + derived exports)
|
||||
# ---------------------------------------------------------------------------
|
||||
_HEADING_RE = re.compile(r"^# (FIND-\d{3,}):\s*(?P<title>.+)$")
|
||||
_BULLET_RE = re.compile(r"^- \*\*(?P<key>[^:*]+):\*\*\s*(?P<value>.*)$")
|
||||
_SECTION_RE = re.compile(r"^## (?P<name>.+)$")
|
||||
_EVIDENCE_ITEM_RE = re.compile(r"^- `(?P<path>[^`]+)`\s*$")
|
||||
|
||||
_FIELD_KEYS = {
|
||||
"severity": "severity",
|
||||
"hypothesis": "hypothesis",
|
||||
"phase": "phase",
|
||||
"ptt task": "ptt_task",
|
||||
"batch": "batch",
|
||||
}
|
||||
|
||||
_SECTION_FIELDS = {
|
||||
"Description": "description",
|
||||
"Impact": "impact",
|
||||
"Remediation": "remediation",
|
||||
}
|
||||
|
||||
_SEVERITY_ORDER = ("Critical", "High", "Medium", "Low", "Info")
|
||||
|
||||
|
||||
def parse_finding_file(path: Path) -> dict[str, Any]:
|
||||
"""Parse a canonical FIND-NNN.md into a flat record dict."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
heading = _HEADING_RE.match(lines[0].strip()) if lines else None
|
||||
if not heading:
|
||||
raise ValueError(f"{path.name}: first line must be '# FIND-NNN: <title>'")
|
||||
record: dict[str, Any] = {
|
||||
"id": heading.group(1),
|
||||
"title": heading.group("title").strip(),
|
||||
"severity": "",
|
||||
"hypothesis": "",
|
||||
"phase": "",
|
||||
"ptt_task": "",
|
||||
"batch": "",
|
||||
"description": "",
|
||||
"impact": "",
|
||||
"evidence": [],
|
||||
"remediation": "",
|
||||
}
|
||||
section = ""
|
||||
section_buf: list[str] = []
|
||||
|
||||
def flush() -> None:
|
||||
key = _SECTION_FIELDS.get(section)
|
||||
if key:
|
||||
record[key] = "\n".join(section_buf).strip()
|
||||
|
||||
for raw in lines[1:]:
|
||||
line = raw.strip()
|
||||
sec = _SECTION_RE.match(line)
|
||||
if sec:
|
||||
flush()
|
||||
section = sec.group("name").strip()
|
||||
section_buf = []
|
||||
continue
|
||||
bullet = _BULLET_RE.match(line)
|
||||
if bullet and not section:
|
||||
key = _FIELD_KEYS.get(bullet.group("key").strip().lower())
|
||||
if key:
|
||||
record[key] = bullet.group("value").strip()
|
||||
continue
|
||||
if section == "Evidence":
|
||||
item = _EVIDENCE_ITEM_RE.match(line)
|
||||
if item:
|
||||
record["evidence"].append(item.group("path").strip())
|
||||
else:
|
||||
section_buf.append(raw)
|
||||
flush()
|
||||
return record
|
||||
|
||||
|
||||
def generate_findings_yaml(eng_dir: str | Path, *, force: bool = False) -> Path:
|
||||
"""Write evidence/reporting/findings.yaml derived from FIND-*.md files."""
|
||||
engagement = Path(eng_dir)
|
||||
findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md"))
|
||||
if not findings:
|
||||
raise ValueError("no FIND-*.md files under evidence/findings/")
|
||||
out = engagement / "evidence" / "reporting" / "findings.yaml"
|
||||
if out.exists() and not force:
|
||||
raise ValueError("findings.yaml exists; pass force=True to regenerate")
|
||||
records = [parse_finding_file(path) for path in findings]
|
||||
payload = {
|
||||
"engagement": engagement.name,
|
||||
"generated_from": ", ".join(path.name for path in findings),
|
||||
"note": (
|
||||
"Derived export generated by violin_guard generate-closeout. "
|
||||
"The canonical records are the per-finding Markdown files; "
|
||||
"this YAML is a machine-readable summary."
|
||||
),
|
||||
"findings": [
|
||||
{
|
||||
"id": rec["id"],
|
||||
"title": rec["title"],
|
||||
"severity": rec["severity"],
|
||||
"hypothesis": rec["hypothesis"],
|
||||
"phase": rec["phase"],
|
||||
"evidence": rec["evidence"],
|
||||
}
|
||||
for rec in records
|
||||
],
|
||||
}
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def generate_report_md(eng_dir: str | Path, *, target: str, force: bool = False) -> Path:
|
||||
"""Write reporting/report.md assembled from FIND-*.md records."""
|
||||
engagement = Path(eng_dir)
|
||||
out = engagement / "reporting" / "report.md"
|
||||
if out.exists() and not force:
|
||||
raise ValueError("report.md exists; pass force=True to regenerate")
|
||||
findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md"))
|
||||
if not findings:
|
||||
raise ValueError("no FIND-*.md files under evidence/findings/")
|
||||
records = [parse_finding_file(path) for path in findings]
|
||||
counts = {
|
||||
severity: sum(1 for rec in records if rec["severity"].lower() == severity.lower())
|
||||
for severity in _SEVERITY_ORDER
|
||||
}
|
||||
lines = [
|
||||
f"# Security Assessment Report — {target}",
|
||||
"",
|
||||
f"- **Engagement:** {engagement.name}",
|
||||
f"- **Target:** {target}",
|
||||
f"- **Findings:** {len(records)}",
|
||||
"",
|
||||
"## Executive Summary",
|
||||
"",
|
||||
"<!-- Write 3-6 sentences: overall posture, worst findings, key themes. -->",
|
||||
"",
|
||||
"| Severity | Count |",
|
||||
"|----------|-------|",
|
||||
]
|
||||
for severity in _SEVERITY_ORDER:
|
||||
lines.append(f"| {severity} | {counts[severity]} |")
|
||||
lines.append("")
|
||||
for rec in records:
|
||||
lines += [
|
||||
f"## {rec['id']}: {rec['title']}",
|
||||
"",
|
||||
f"- **Severity:** {rec['severity']}",
|
||||
]
|
||||
if rec["hypothesis"]:
|
||||
lines.append(f"- **Hypothesis:** {rec['hypothesis']}")
|
||||
if rec["phase"]:
|
||||
lines.append(f"- **Phase:** {rec['phase']}")
|
||||
lines += [
|
||||
"",
|
||||
"### Description",
|
||||
"",
|
||||
rec["description"],
|
||||
"",
|
||||
"### Impact",
|
||||
"",
|
||||
rec["impact"],
|
||||
"",
|
||||
"### Evidence",
|
||||
"",
|
||||
*[f"- `{item}`" for item in rec["evidence"]],
|
||||
"",
|
||||
"### Remediation",
|
||||
"",
|
||||
rec["remediation"],
|
||||
"",
|
||||
]
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
return out
|
||||
|
||||
+24
-1
@@ -13,7 +13,7 @@ _PROFILE_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROFILE_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROFILE_ROOT))
|
||||
|
||||
from plugins.violin_guard import bootstrap, command, handlers, state
|
||||
from plugins.violin_guard import bootstrap, command, findings, handlers, state
|
||||
|
||||
|
||||
def _print_result(result) -> int:
|
||||
@@ -55,6 +55,20 @@ def cmd_validate_scope(args: argparse.Namespace) -> int:
|
||||
return code
|
||||
|
||||
|
||||
def cmd_generate_closeout(args: argparse.Namespace) -> int:
|
||||
try:
|
||||
yaml_path = findings.generate_findings_yaml(args.eng_dir, force=args.force)
|
||||
report_path = findings.generate_report_md(
|
||||
args.eng_dir, target=args.target, force=args.force
|
||||
)
|
||||
except ValueError as exc:
|
||||
print(f"BLOCK: {exc}")
|
||||
return 1
|
||||
print(f"OK: wrote {yaml_path}")
|
||||
print(f"OK: wrote {report_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_record_ptt(args: argparse.Namespace) -> int:
|
||||
out = json.loads(
|
||||
handlers.handle_record_ptt(
|
||||
@@ -311,6 +325,15 @@ def main() -> int:
|
||||
p = sub.add_parser("check-release", help="Run release checks")
|
||||
p.set_defaults(func=cmd_check_release)
|
||||
|
||||
p = sub.add_parser(
|
||||
"generate-closeout",
|
||||
help="Derive findings.yaml and report.md from canonical FIND-*.md files",
|
||||
)
|
||||
p.add_argument("--eng-dir", required=True)
|
||||
p.add_argument("--target", required=True)
|
||||
p.add_argument("--force", action="store_true", help="Overwrite existing artifacts")
|
||||
p.set_defaults(func=cmd_generate_closeout)
|
||||
|
||||
# search-exploit
|
||||
p = sub.add_parser("search-exploit", help="Search local ExploitDB (read-only)")
|
||||
p.add_argument("--product", default="")
|
||||
|
||||
@@ -92,6 +92,8 @@ Violin is intended for Kali Linux or Parrot OS first. The `terminal` tool may st
|
||||
|
||||
Engagement sequence: `SCOPING ──► RECON ──► VULN RESEARCH ──► EXPLOITATION ──► REPORTING ──► RETROSPECTIVE`
|
||||
|
||||
> **Exhaustive coverage standard**: every full-scope engagement applies `references/exhaustive-coverage.md` — open it via `skill_view` at engagement start and keep it applied throughout. Build and disposition a coverage matrix (routes × methods, parameters, role boundaries, auth/session flows, business logic, redirects, rate limits, SSRF, injection, object authorization); do not stop at the first finding; route discovery from the live application bundle is required; canonize every Validated hypothesis into `evidence/findings/FIND-NNN.md`.
|
||||
|
||||
| Phase / Intent | Playbook Path | Description |
|
||||
|---|---|---|
|
||||
| **SCOPING** | `playbooks/scoping.md` | Scoping, RoE definition, project setup, 9-question clarify |
|
||||
@@ -198,6 +200,12 @@ This prevents re-running scans, missing cross-phase patterns, and losing the inv
|
||||
|
||||
> **Execution Context Rule:** During Hermes sessions, **always use typed plugin tools** (`violin_exec`, `violin_record_ptt`, `violin_review_batch`, `violin_status`). Standalone CLI subcommands (`python scripts/violin_guard.py ...`) are strictly for host administration or offline diagnostic inspection via `terminal`.
|
||||
|
||||
> **Operational Contract (mandatory, applies to every batch):**
|
||||
> - **Complete commands only**: a batch `commands` array must contain only complete shell commands — never labels, descriptions, or prose. An entry that is not an executable command is a contract violation.
|
||||
> - **Fix and rerun**: after any nonzero exit or stderr traceback, fix the command and rerun it before closing the phase — a failed command is not a result.
|
||||
> - **Review without skill override**: when reviewing a completed batch via `violin_review_batch`, omit the optional `skill` argument so the active execution binding is used.
|
||||
> - **Friction log**: log tool friction, guard errors, and any guard-code inspection in `state/framework_feedback.md`.
|
||||
|
||||
| Subcommand / Tool | Type | Action | Parameters |
|
||||
|---|---|---|---|
|
||||
| `violin_exec` | Typed Tool | Execute target command with scope & history tracking | `eng_dir`, `phase`, `target`, `command` |
|
||||
|
||||
@@ -28,7 +28,11 @@ Before generating the report, systematically review all evidence collected durin
|
||||
1. **Read the hypothesis board** — `read_file path="$ENG_DIR/hypotheses.md"` to see all theories and their resolution status. Do NOT overwrite or collapse `$ENG_DIR/hypotheses.md` into plain narrative text; final reports belong in `$ENG_DIR/reporting/report.md`.
|
||||
2. **Locate evidence** — navigate to `$ENG_DIR/evidence/` and enumerate all subdirectories and files. Evidence files MUST reside under `$ENG_DIR/evidence/<phase>/` (never inside `$ENG_DIR/state/`).
|
||||
3. **Verify reproducibility** — each validated finding has raw evidence plus a `templates/verification-receipt.yaml` receipt with `state: validated`, `oracle_kind`, `actual_signal`, and artifact paths.
|
||||
4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; `evidence/reporting/findings.yaml` may be generated as a derived export but cannot replace it.
|
||||
4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; the derived exports are **generated, not hand-written**:
|
||||
```bash
|
||||
python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>"
|
||||
```
|
||||
This writes `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections pulled from the FIND files). Never re-type finding content that already exists in a FIND file.
|
||||
5. **Categorize findings** into three tiers:
|
||||
|
||||
| Tier | Criteria | Action |
|
||||
@@ -161,21 +165,11 @@ Briefly mention rejected findings to demonstrate thorough testing:
|
||||
- Exploitation
|
||||
- Post-Exploitation
|
||||
- Reporting
|
||||
5. **Write each finding** — using the finding format:
|
||||
- Title
|
||||
- Severity (Critical / High / Medium / Low / Info)
|
||||
- CVSS score and vector (if applicable)
|
||||
- CVE identifier (if applicable)
|
||||
- Confidence (Certain / High / Medium / Low)
|
||||
- Description
|
||||
- Evidence (command + output)
|
||||
- Impact
|
||||
- Remediation
|
||||
6. **Generate summary table** — create a table counting findings by severity
|
||||
5. **Generate the report skeleton** — run `python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>"` (see §Verify canonical findings). The skeleton already contains every finding's title, severity, description, impact, evidence list, and remediation pulled from the FIND files, plus the severity-count table. Do NOT hand-write per-finding sections again.
|
||||
6. **Write the narrative** — your only required hand-written content is the **Executive Summary** (3-6 sentences: overall posture, worst findings, key themes) replacing the placeholder comment. Optionally add methodology and limitations prose.
|
||||
7. **Document limitations** — any constraints that affected testing (time, access, tooling, scope)
|
||||
8. **Build appendix** — list all evidence file paths with brief descriptions
|
||||
9. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices
|
||||
10. **Save** — write the completed report to `$ENG_DIR/reporting/report.md`
|
||||
8. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices
|
||||
9. **Save** — the generator already wrote `$ENG_DIR/reporting/report.md`; edit it in place for the narrative
|
||||
|
||||
## Auto-Fix Evidence Compilation
|
||||
|
||||
@@ -212,7 +206,8 @@ Before delivering the report, verify every item:
|
||||
- [ ] All evidence files are referenced by their full or relative path
|
||||
- [ ] Severity ratings are justified
|
||||
- [ ] Executive summary reflects the actual findings (not boilerplate)
|
||||
- [ ] Report is saved to `$ENG_DIR/reporting/report.md`
|
||||
- [ ] Report is saved to `$ENG_DIR/reporting/report.md` (generated via `generate-closeout`, narrative filled in)
|
||||
- [ ] `evidence/reporting/findings.yaml` generated via `generate-closeout` (never hand-written)
|
||||
- [ ] Retrospective is saved to `$ENG_DIR/retrospective/retrospective.md`
|
||||
- [ ] Final closeout task marked completed: invoke `violin_record_ptt(id="<closeout-task-id>", status="[x]", note="...")` to mark engagement closeout completed in `ptt.md`.
|
||||
|
||||
|
||||
@@ -96,3 +96,12 @@ to cross, is the failure mode. Surface real blockers precisely and move on.
|
||||
If no primary/accessible evidence supports a value, report it as **unsupported /
|
||||
unconfirmed** rather than reconstructing it and presenting it as observed. A
|
||||
finding's confidence is only as honest as the evidence trail behind it.
|
||||
|
||||
## 8. Citation discipline
|
||||
|
||||
Every `evidence/` citation in hypotheses, findings, and reports must name an
|
||||
**existing concrete file with its extension** (`evidence/exploitation/2026-01-01T00-00-00-hash-command.stdout.txt`).
|
||||
Never cite a directory, wildcard, path stem, shell fragment, or command
|
||||
substitution, and never use `evidence/` as a generic prose noun. A citation
|
||||
that cannot be resolved to a real file is a defect in the engagement record —
|
||||
fix the reference before closeout, do not leave it for the reader to guess.
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# Exhaustive Assessment Coverage
|
||||
|
||||
Mandatory operating standard for a full-scope engagement. The objective of a
|
||||
full assessment is a **complete, dispositioned coverage matrix** — not a
|
||||
stopping point after the first confirmed issue. Applies in every phase and
|
||||
complements `evidence-and-verification-discipline.md` and the phase playbooks.
|
||||
|
||||
## 1. Build and disposition a coverage matrix
|
||||
|
||||
Track coverage for the engagement's surface:
|
||||
|
||||
- routes and methods (every discovered endpoint × every permitted method)
|
||||
- parameters and input vectors
|
||||
- role boundaries (guest / user / admin / other principals) and object authorization
|
||||
- authentication and session flows
|
||||
- business logic
|
||||
- redirects, rate limits, SSRF, injection, and object-authorization classes
|
||||
|
||||
A cell is either **tested** (with a recorded request + response, positive or
|
||||
negative) or **explicitly dispositioned** (with a reason: out of scope, guarded,
|
||||
not present on this target). Do not close a phase until the matrix is complete.
|
||||
|
||||
## 2. Do not stop after the first finding
|
||||
|
||||
Record the issue (hypothesis → Validated, canonize a finding), then **continue
|
||||
testing the remaining matrix cells**. A confirmed finding is a checkpoint, not a
|
||||
termination condition.
|
||||
|
||||
## 3. Route discovery is a required step
|
||||
|
||||
- Derive endpoints from the live application bundle (JS, HTML, API schemas,
|
||||
OpenAPI/swagger, robots.txt, sitemap) before guessing.
|
||||
- Fuzz a supplemental wordlist of common API nouns and actions over each
|
||||
resource.
|
||||
- Treat 401/403/405/3xx responses as route evidence; inspect `Allow` headers and
|
||||
retry every discovered route with each permitted method.
|
||||
- Try path-parameter, trailing-slash, case, and trailing-`/` variants.
|
||||
- Probe ordinary action/verb variants of each discovered resource: list,
|
||||
detail, create, update, delete, import, export, upload, preview, validate,
|
||||
status, and similar.
|
||||
- Do not reject a whole class because one guessed route is absent — derive the
|
||||
exact route and method from the application and test each variant.
|
||||
|
||||
## 4. Authentication and session boundaries (as present on the target)
|
||||
|
||||
- Token signature, algorithm, and claim manipulation.
|
||||
- Default or weak credential behavior **without credential stuffing**.
|
||||
- Any multi-factor or one-time-code flow: empty, invalid, replayed, and
|
||||
valid-code boundaries.
|
||||
- Password-reset and account-recovery flows.
|
||||
|
||||
## 5. Authorization boundaries
|
||||
|
||||
- Object-level access control: horizontal (another principal's resources,
|
||||
including object IDs) and vertical (privileged roles).
|
||||
- Function-level access control (e.g. privileged operations reachable as a
|
||||
lower-privilege principal).
|
||||
- Unauthenticated state-changing paths.
|
||||
|
||||
For every discovered object ID, retry the access under a different principal's
|
||||
session to test horizontal authorization.
|
||||
|
||||
## 6. Business-logic edges
|
||||
|
||||
- Negative or zero quantities, omitted required fields, price/cost
|
||||
manipulation, and duplicate or concurrent submissions.
|
||||
|
||||
## 7. Proof and evidence discipline
|
||||
|
||||
- Capture proofs with `curl --http1.1 -i` so every target request prints a
|
||||
canonical `METHOD /path HTTP/1.1` request line, the `HTTP/1.1 NNN` response
|
||||
line, and the response body. Labels alone are not proof.
|
||||
- Correlate request and response evidence in guarded execution bundles.
|
||||
- Record **negative results** with the exact request and response too.
|
||||
- Canonize: every validated issue becomes a Validated hypothesis linked to a
|
||||
canonical `evidence/findings/FIND-NNN.md` record with concrete runtime
|
||||
evidence; record rejected theories as well.
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Tests for the closeout artifact generator (FIND parsing + findings.yaml/report.md derivation)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from plugins.violin_guard.findings import (
|
||||
generate_findings_yaml,
|
||||
generate_report_md,
|
||||
parse_finding_file,
|
||||
)
|
||||
|
||||
FIND_MD = """# FIND-002: IDOR - arbitrary order read
|
||||
|
||||
- **Severity:** High
|
||||
- **Batch:** 356aff5b-1674-4693-8417-96d2e9b55248
|
||||
- **PTT task:** PT-102
|
||||
- **Phase:** VULN_RESEARCH
|
||||
- **Hypothesis:** H-003
|
||||
|
||||
## Description
|
||||
|
||||
GET /api/v1/orders/{id} performs no ownership check.
|
||||
|
||||
## Impact
|
||||
|
||||
Horizontal privilege escalation.
|
||||
|
||||
## Evidence
|
||||
|
||||
- `evidence/executions/2026-08-10T094638-ee4dead5-command.json`
|
||||
- `evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt`
|
||||
|
||||
## Remediation
|
||||
|
||||
Enforce object-ownership authorization.
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 1: FIND file parser
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_parse_finding_file_extracts_fields(tmp_path: Path) -> None:
|
||||
f = tmp_path / "FIND-002.md"
|
||||
f.write_text(FIND_MD, encoding="utf-8")
|
||||
rec = parse_finding_file(f)
|
||||
assert rec["id"] == "FIND-002"
|
||||
assert rec["title"] == "IDOR - arbitrary order read"
|
||||
assert rec["severity"] == "High"
|
||||
assert rec["hypothesis"] == "H-003"
|
||||
assert rec["phase"] == "VULN_RESEARCH"
|
||||
assert rec["ptt_task"] == "PT-102"
|
||||
assert rec["batch"] == "356aff5b-1674-4693-8417-96d2e9b55248"
|
||||
assert rec["evidence"] == [
|
||||
"evidence/executions/2026-08-10T094638-ee4dead5-command.json",
|
||||
"evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt",
|
||||
]
|
||||
assert "no ownership check" in rec["description"]
|
||||
assert "object-ownership" in rec["remediation"]
|
||||
|
||||
|
||||
def test_parse_finding_file_rejects_non_finding(tmp_path: Path) -> None:
|
||||
f = tmp_path / "notes.md"
|
||||
f.write_text("# Not a finding\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
parse_finding_file(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 2: findings.yaml generator
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_generate_findings_yaml_roundtrip(tmp_path: Path) -> None:
|
||||
findings_dir = tmp_path / "evidence" / "findings"
|
||||
findings_dir.mkdir(parents=True)
|
||||
(findings_dir / "FIND-001.md").write_text(
|
||||
FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8"
|
||||
)
|
||||
out = generate_findings_yaml(tmp_path)
|
||||
assert out == tmp_path / "evidence" / "reporting" / "findings.yaml"
|
||||
data = yaml.safe_load(out.read_text(encoding="utf-8"))
|
||||
assert data["findings"][0]["id"] == "FIND-001"
|
||||
assert data["findings"][0]["severity"] == "High"
|
||||
assert data["findings"][0]["hypothesis"] == "H-003"
|
||||
assert len(data["findings"][0]["evidence"]) == 2
|
||||
assert "generated_from" in data
|
||||
|
||||
|
||||
def test_generate_findings_yaml_no_findings(tmp_path: Path) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
generate_findings_yaml(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 3: report.md generator
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_generate_report_md_contents(tmp_path: Path) -> None:
|
||||
findings_dir = tmp_path / "evidence" / "findings"
|
||||
findings_dir.mkdir(parents=True)
|
||||
(findings_dir / "FIND-002.md").write_text(FIND_MD, encoding="utf-8")
|
||||
out = generate_report_md(tmp_path, target="https://example.test")
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert out == tmp_path / "reporting" / "report.md"
|
||||
assert "https://example.test" in text
|
||||
assert "FIND-002" in text
|
||||
assert "| High | 1 |" in text
|
||||
assert "no ownership check" in text
|
||||
assert "Executive Summary" in text
|
||||
|
||||
|
||||
def test_generate_report_md_no_overwrite(tmp_path: Path) -> None:
|
||||
rep = tmp_path / "reporting" / "report.md"
|
||||
rep.parent.mkdir(parents=True)
|
||||
rep.write_text("existing", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
generate_report_md(tmp_path, target="https://example.test")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 4: CLI subcommand
|
||||
# ---------------------------------------------------------------------------
|
||||
def test_cli_generate_closeout(tmp_path: Path) -> None:
|
||||
findings_dir = tmp_path / "evidence" / "findings"
|
||||
findings_dir.mkdir(parents=True)
|
||||
(findings_dir / "FIND-001.md").write_text(
|
||||
FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8"
|
||||
)
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
r = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/violin_guard.py",
|
||||
"generate-closeout",
|
||||
"--eng-dir",
|
||||
str(tmp_path),
|
||||
"--target",
|
||||
"https://example.test",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=str(repo_root),
|
||||
)
|
||||
assert r.returncode == 0, r.stderr
|
||||
assert (tmp_path / "evidence" / "reporting" / "findings.yaml").exists()
|
||||
assert (tmp_path / "reporting" / "report.md").exists()
|
||||
Reference in New Issue
Block a user