diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d74e8..649d229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ ## 3.1.0 (Unreleased) ### Automated Benchmark & Evaluation Framework +- Closeout artifacts are now generated, not hand-written: `python scripts/violin_guard.py generate-closeout --eng-dir --target ` derives `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections) from canonical `evidence/findings/FIND-NNN.md` files. Agent closeout duty reduced to the executive-summary narrative — cuts per-finding recording from 4 surfaces to 2 and eliminates the duplicate rewriting that starved exploitation budget and timed out closeout. +- Benchmark goal/closeout prompts reduced to task-only (target + integrity constraint); methodology moved into the framework — new `skills/pentest/references/exhaustive-coverage.md` (coverage matrix, don't-stop-at-first-finding, route discovery, authz/business-logic boundaries, FIND-NNN canonization) mandated from `skills/pentest/SKILL.md` §3, plus an Operational Contract in §7 and Citation Discipline in `evidence-and-verification-discipline.md` §8. +- Evidence-gated scorer now confirms Validated hypotheses via canonical `evidence/findings/FIND-NNN.md` files matched in live-app terms (METHOD + endpoint route, with `/api/v1` optional) in addition to execution bundles, so truncated `tail -c` evidence no longer loses real findings. +- Endpoint+method proof bundles are accepted with a single pattern hit; shared-endpoint challenges (auth/login, checkout, register) are gated by explicit `require`/`require_any` discriminators (role-granted admin, credit granted, negative quantity, external redirect, rate-limit attempt evidence) to prevent cross-challenge false positives. +- Absence-type challenges (`no-rate-limiting`) are scored on repeated identical status codes with no 429/throttle signal instead of requiring the counter-token. +- `neg-quantity-cart` accepts both `PUT /api/v1/cart/items/{id}` and `POST /api/v1/cart/add` vectors. +- Runner no longer invalidates a run when the closeout continuation pass times out or exits non-zero; closeout is reported as a soft `closeout_warning` while scoring proceeds on substantive evidence. +- Schema-drift audit whitelists legitimate `state/report.md` and `state/retrospective.md` closeout artifacts. - Introduced automated Hermes profile benchmark runner engine (`benchmark/run.py`) and AI-assisted evaluation framework (`benchmark/ai_judge.py`). - Added `benchmark/indexer.py` for single-pass engagement directory artifact indexing with a 2MB per-file resource limit and cross-platform POSIX path normalization (`.as_posix()`). - Optimized `benchmark/score.py` pattern matching disk I/O to single-pass $O(F)$ scanning and removed late function-scope imports. diff --git a/plugins/violin_guard/findings.py b/plugins/violin_guard/findings.py index 70d8105..ee0b63e 100644 --- a/plugins/violin_guard/findings.py +++ b/plugins/violin_guard/findings.py @@ -6,6 +6,8 @@ import re from pathlib import Path from typing import Any +import yaml + from . import hypotheses, state _FINDING_ID_RE = re.compile(r"FIND-(\d{3,})$") @@ -203,3 +205,180 @@ def _create_from_pending_batch( "batch_id": batch_id, "reused": False, } + + +# --------------------------------------------------------------------------- +# Closeout artifact generation (FIND parsing + derived exports) +# --------------------------------------------------------------------------- +_HEADING_RE = re.compile(r"^# (FIND-\d{3,}):\s*(?P.+)$") +_BULLET_RE = re.compile(r"^- \*\*(?P<key>[^:*]+):\*\*\s*(?P<value>.*)$") +_SECTION_RE = re.compile(r"^## (?P<name>.+)$") +_EVIDENCE_ITEM_RE = re.compile(r"^- `(?P<path>[^`]+)`\s*$") + +_FIELD_KEYS = { + "severity": "severity", + "hypothesis": "hypothesis", + "phase": "phase", + "ptt task": "ptt_task", + "batch": "batch", +} + +_SECTION_FIELDS = { + "Description": "description", + "Impact": "impact", + "Remediation": "remediation", +} + +_SEVERITY_ORDER = ("Critical", "High", "Medium", "Low", "Info") + + +def parse_finding_file(path: Path) -> dict[str, Any]: + """Parse a canonical FIND-NNN.md into a flat record dict.""" + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + heading = _HEADING_RE.match(lines[0].strip()) if lines else None + if not heading: + raise ValueError(f"{path.name}: first line must be '# FIND-NNN: <title>'") + record: dict[str, Any] = { + "id": heading.group(1), + "title": heading.group("title").strip(), + "severity": "", + "hypothesis": "", + "phase": "", + "ptt_task": "", + "batch": "", + "description": "", + "impact": "", + "evidence": [], + "remediation": "", + } + section = "" + section_buf: list[str] = [] + + def flush() -> None: + key = _SECTION_FIELDS.get(section) + if key: + record[key] = "\n".join(section_buf).strip() + + for raw in lines[1:]: + line = raw.strip() + sec = _SECTION_RE.match(line) + if sec: + flush() + section = sec.group("name").strip() + section_buf = [] + continue + bullet = _BULLET_RE.match(line) + if bullet and not section: + key = _FIELD_KEYS.get(bullet.group("key").strip().lower()) + if key: + record[key] = bullet.group("value").strip() + continue + if section == "Evidence": + item = _EVIDENCE_ITEM_RE.match(line) + if item: + record["evidence"].append(item.group("path").strip()) + else: + section_buf.append(raw) + flush() + return record + + +def generate_findings_yaml(eng_dir: str | Path, *, force: bool = False) -> Path: + """Write evidence/reporting/findings.yaml derived from FIND-*.md files.""" + engagement = Path(eng_dir) + findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md")) + if not findings: + raise ValueError("no FIND-*.md files under evidence/findings/") + out = engagement / "evidence" / "reporting" / "findings.yaml" + if out.exists() and not force: + raise ValueError("findings.yaml exists; pass force=True to regenerate") + records = [parse_finding_file(path) for path in findings] + payload = { + "engagement": engagement.name, + "generated_from": ", ".join(path.name for path in findings), + "note": ( + "Derived export generated by violin_guard generate-closeout. " + "The canonical records are the per-finding Markdown files; " + "this YAML is a machine-readable summary." + ), + "findings": [ + { + "id": rec["id"], + "title": rec["title"], + "severity": rec["severity"], + "hypothesis": rec["hypothesis"], + "phase": rec["phase"], + "evidence": rec["evidence"], + } + for rec in records + ], + } + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") + return out + + +def generate_report_md(eng_dir: str | Path, *, target: str, force: bool = False) -> Path: + """Write reporting/report.md assembled from FIND-*.md records.""" + engagement = Path(eng_dir) + out = engagement / "reporting" / "report.md" + if out.exists() and not force: + raise ValueError("report.md exists; pass force=True to regenerate") + findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md")) + if not findings: + raise ValueError("no FIND-*.md files under evidence/findings/") + records = [parse_finding_file(path) for path in findings] + counts = { + severity: sum(1 for rec in records if rec["severity"].lower() == severity.lower()) + for severity in _SEVERITY_ORDER + } + lines = [ + f"# Security Assessment Report — {target}", + "", + f"- **Engagement:** {engagement.name}", + f"- **Target:** {target}", + f"- **Findings:** {len(records)}", + "", + "## Executive Summary", + "", + "<!-- Write 3-6 sentences: overall posture, worst findings, key themes. -->", + "", + "| Severity | Count |", + "|----------|-------|", + ] + for severity in _SEVERITY_ORDER: + lines.append(f"| {severity} | {counts[severity]} |") + lines.append("") + for rec in records: + lines += [ + f"## {rec['id']}: {rec['title']}", + "", + f"- **Severity:** {rec['severity']}", + ] + if rec["hypothesis"]: + lines.append(f"- **Hypothesis:** {rec['hypothesis']}") + if rec["phase"]: + lines.append(f"- **Phase:** {rec['phase']}") + lines += [ + "", + "### Description", + "", + rec["description"], + "", + "### Impact", + "", + rec["impact"], + "", + "### Evidence", + "", + *[f"- `{item}`" for item in rec["evidence"]], + "", + "### Remediation", + "", + rec["remediation"], + "", + ] + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + return out diff --git a/scripts/violin_guard.py b/scripts/violin_guard.py index ec97897..0a7b5e1 100644 --- a/scripts/violin_guard.py +++ b/scripts/violin_guard.py @@ -13,7 +13,7 @@ _PROFILE_ROOT = Path(__file__).resolve().parent.parent if str(_PROFILE_ROOT) not in sys.path: sys.path.insert(0, str(_PROFILE_ROOT)) -from plugins.violin_guard import bootstrap, command, handlers, state +from plugins.violin_guard import bootstrap, command, findings, handlers, state def _print_result(result) -> int: @@ -55,6 +55,20 @@ def cmd_validate_scope(args: argparse.Namespace) -> int: return code +def cmd_generate_closeout(args: argparse.Namespace) -> int: + try: + yaml_path = findings.generate_findings_yaml(args.eng_dir, force=args.force) + report_path = findings.generate_report_md( + args.eng_dir, target=args.target, force=args.force + ) + except ValueError as exc: + print(f"BLOCK: {exc}") + return 1 + print(f"OK: wrote {yaml_path}") + print(f"OK: wrote {report_path}") + return 0 + + def cmd_record_ptt(args: argparse.Namespace) -> int: out = json.loads( handlers.handle_record_ptt( @@ -311,6 +325,15 @@ def main() -> int: p = sub.add_parser("check-release", help="Run release checks") p.set_defaults(func=cmd_check_release) + p = sub.add_parser( + "generate-closeout", + help="Derive findings.yaml and report.md from canonical FIND-*.md files", + ) + p.add_argument("--eng-dir", required=True) + p.add_argument("--target", required=True) + p.add_argument("--force", action="store_true", help="Overwrite existing artifacts") + p.set_defaults(func=cmd_generate_closeout) + # search-exploit p = sub.add_parser("search-exploit", help="Search local ExploitDB (read-only)") p.add_argument("--product", default="") diff --git a/skills/pentest/SKILL.md b/skills/pentest/SKILL.md index 077fe26..48e7fa0 100644 --- a/skills/pentest/SKILL.md +++ b/skills/pentest/SKILL.md @@ -92,6 +92,8 @@ Violin is intended for Kali Linux or Parrot OS first. The `terminal` tool may st Engagement sequence: `SCOPING ──► RECON ──► VULN RESEARCH ──► EXPLOITATION ──► REPORTING ──► RETROSPECTIVE` +> **Exhaustive coverage standard**: every full-scope engagement applies `references/exhaustive-coverage.md` — open it via `skill_view` at engagement start and keep it applied throughout. Build and disposition a coverage matrix (routes × methods, parameters, role boundaries, auth/session flows, business logic, redirects, rate limits, SSRF, injection, object authorization); do not stop at the first finding; route discovery from the live application bundle is required; canonize every Validated hypothesis into `evidence/findings/FIND-NNN.md`. + | Phase / Intent | Playbook Path | Description | |---|---|---| | **SCOPING** | `playbooks/scoping.md` | Scoping, RoE definition, project setup, 9-question clarify | @@ -198,6 +200,12 @@ This prevents re-running scans, missing cross-phase patterns, and losing the inv > **Execution Context Rule:** During Hermes sessions, **always use typed plugin tools** (`violin_exec`, `violin_record_ptt`, `violin_review_batch`, `violin_status`). Standalone CLI subcommands (`python scripts/violin_guard.py ...`) are strictly for host administration or offline diagnostic inspection via `terminal`. +> **Operational Contract (mandatory, applies to every batch):** +> - **Complete commands only**: a batch `commands` array must contain only complete shell commands — never labels, descriptions, or prose. An entry that is not an executable command is a contract violation. +> - **Fix and rerun**: after any nonzero exit or stderr traceback, fix the command and rerun it before closing the phase — a failed command is not a result. +> - **Review without skill override**: when reviewing a completed batch via `violin_review_batch`, omit the optional `skill` argument so the active execution binding is used. +> - **Friction log**: log tool friction, guard errors, and any guard-code inspection in `state/framework_feedback.md`. + | Subcommand / Tool | Type | Action | Parameters | |---|---|---|---| | `violin_exec` | Typed Tool | Execute target command with scope & history tracking | `eng_dir`, `phase`, `target`, `command` | diff --git a/skills/pentest/playbooks/reporting.md b/skills/pentest/playbooks/reporting.md index bb3ae07..05e2b05 100644 --- a/skills/pentest/playbooks/reporting.md +++ b/skills/pentest/playbooks/reporting.md @@ -28,7 +28,11 @@ Before generating the report, systematically review all evidence collected durin 1. **Read the hypothesis board** — `read_file path="$ENG_DIR/hypotheses.md"` to see all theories and their resolution status. Do NOT overwrite or collapse `$ENG_DIR/hypotheses.md` into plain narrative text; final reports belong in `$ENG_DIR/reporting/report.md`. 2. **Locate evidence** — navigate to `$ENG_DIR/evidence/` and enumerate all subdirectories and files. Evidence files MUST reside under `$ENG_DIR/evidence/<phase>/` (never inside `$ENG_DIR/state/`). 3. **Verify reproducibility** — each validated finding has raw evidence plus a `templates/verification-receipt.yaml` receipt with `state: validated`, `oracle_kind`, `actual_signal`, and artifact paths. -4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; `evidence/reporting/findings.yaml` may be generated as a derived export but cannot replace it. +4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; the derived exports are **generated, not hand-written**: + ```bash + python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>" + ``` + This writes `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections pulled from the FIND files). Never re-type finding content that already exists in a FIND file. 5. **Categorize findings** into three tiers: | Tier | Criteria | Action | @@ -161,21 +165,11 @@ Briefly mention rejected findings to demonstrate thorough testing: - Exploitation - Post-Exploitation - Reporting -5. **Write each finding** — using the finding format: - - Title - - Severity (Critical / High / Medium / Low / Info) - - CVSS score and vector (if applicable) - - CVE identifier (if applicable) - - Confidence (Certain / High / Medium / Low) - - Description - - Evidence (command + output) - - Impact - - Remediation -6. **Generate summary table** — create a table counting findings by severity +5. **Generate the report skeleton** — run `python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>"` (see §Verify canonical findings). The skeleton already contains every finding's title, severity, description, impact, evidence list, and remediation pulled from the FIND files, plus the severity-count table. Do NOT hand-write per-finding sections again. +6. **Write the narrative** — your only required hand-written content is the **Executive Summary** (3-6 sentences: overall posture, worst findings, key themes) replacing the placeholder comment. Optionally add methodology and limitations prose. 7. **Document limitations** — any constraints that affected testing (time, access, tooling, scope) -8. **Build appendix** — list all evidence file paths with brief descriptions -9. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices -10. **Save** — write the completed report to `$ENG_DIR/reporting/report.md` +8. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices +9. **Save** — the generator already wrote `$ENG_DIR/reporting/report.md`; edit it in place for the narrative ## Auto-Fix Evidence Compilation @@ -212,7 +206,8 @@ Before delivering the report, verify every item: - [ ] All evidence files are referenced by their full or relative path - [ ] Severity ratings are justified - [ ] Executive summary reflects the actual findings (not boilerplate) -- [ ] Report is saved to `$ENG_DIR/reporting/report.md` +- [ ] Report is saved to `$ENG_DIR/reporting/report.md` (generated via `generate-closeout`, narrative filled in) +- [ ] `evidence/reporting/findings.yaml` generated via `generate-closeout` (never hand-written) - [ ] Retrospective is saved to `$ENG_DIR/retrospective/retrospective.md` - [ ] Final closeout task marked completed: invoke `violin_record_ptt(id="<closeout-task-id>", status="[x]", note="...")` to mark engagement closeout completed in `ptt.md`. diff --git a/skills/pentest/references/evidence-and-verification-discipline.md b/skills/pentest/references/evidence-and-verification-discipline.md index d0f7bf1..e281a79 100644 --- a/skills/pentest/references/evidence-and-verification-discipline.md +++ b/skills/pentest/references/evidence-and-verification-discipline.md @@ -96,3 +96,12 @@ to cross, is the failure mode. Surface real blockers precisely and move on. If no primary/accessible evidence supports a value, report it as **unsupported / unconfirmed** rather than reconstructing it and presenting it as observed. A finding's confidence is only as honest as the evidence trail behind it. + +## 8. Citation discipline + +Every `evidence/` citation in hypotheses, findings, and reports must name an +**existing concrete file with its extension** (`evidence/exploitation/2026-01-01T00-00-00-hash-command.stdout.txt`). +Never cite a directory, wildcard, path stem, shell fragment, or command +substitution, and never use `evidence/` as a generic prose noun. A citation +that cannot be resolved to a real file is a defect in the engagement record — +fix the reference before closeout, do not leave it for the reader to guess. diff --git a/skills/pentest/references/exhaustive-coverage.md b/skills/pentest/references/exhaustive-coverage.md new file mode 100644 index 0000000..7caf8f1 --- /dev/null +++ b/skills/pentest/references/exhaustive-coverage.md @@ -0,0 +1,77 @@ +# Exhaustive Assessment Coverage + +Mandatory operating standard for a full-scope engagement. The objective of a +full assessment is a **complete, dispositioned coverage matrix** — not a +stopping point after the first confirmed issue. Applies in every phase and +complements `evidence-and-verification-discipline.md` and the phase playbooks. + +## 1. Build and disposition a coverage matrix + +Track coverage for the engagement's surface: + +- routes and methods (every discovered endpoint × every permitted method) +- parameters and input vectors +- role boundaries (guest / user / admin / other principals) and object authorization +- authentication and session flows +- business logic +- redirects, rate limits, SSRF, injection, and object-authorization classes + +A cell is either **tested** (with a recorded request + response, positive or +negative) or **explicitly dispositioned** (with a reason: out of scope, guarded, +not present on this target). Do not close a phase until the matrix is complete. + +## 2. Do not stop after the first finding + +Record the issue (hypothesis → Validated, canonize a finding), then **continue +testing the remaining matrix cells**. A confirmed finding is a checkpoint, not a +termination condition. + +## 3. Route discovery is a required step + +- Derive endpoints from the live application bundle (JS, HTML, API schemas, + OpenAPI/swagger, robots.txt, sitemap) before guessing. +- Fuzz a supplemental wordlist of common API nouns and actions over each + resource. +- Treat 401/403/405/3xx responses as route evidence; inspect `Allow` headers and + retry every discovered route with each permitted method. +- Try path-parameter, trailing-slash, case, and trailing-`/` variants. +- Probe ordinary action/verb variants of each discovered resource: list, + detail, create, update, delete, import, export, upload, preview, validate, + status, and similar. +- Do not reject a whole class because one guessed route is absent — derive the + exact route and method from the application and test each variant. + +## 4. Authentication and session boundaries (as present on the target) + +- Token signature, algorithm, and claim manipulation. +- Default or weak credential behavior **without credential stuffing**. +- Any multi-factor or one-time-code flow: empty, invalid, replayed, and + valid-code boundaries. +- Password-reset and account-recovery flows. + +## 5. Authorization boundaries + +- Object-level access control: horizontal (another principal's resources, + including object IDs) and vertical (privileged roles). +- Function-level access control (e.g. privileged operations reachable as a + lower-privilege principal). +- Unauthenticated state-changing paths. + +For every discovered object ID, retry the access under a different principal's +session to test horizontal authorization. + +## 6. Business-logic edges + +- Negative or zero quantities, omitted required fields, price/cost + manipulation, and duplicate or concurrent submissions. + +## 7. Proof and evidence discipline + +- Capture proofs with `curl --http1.1 -i` so every target request prints a + canonical `METHOD /path HTTP/1.1` request line, the `HTTP/1.1 NNN` response + line, and the response body. Labels alone are not proof. +- Correlate request and response evidence in guarded execution bundles. +- Record **negative results** with the exact request and response too. +- Canonize: every validated issue becomes a Validated hypothesis linked to a + canonical `evidence/findings/FIND-NNN.md` record with concrete runtime + evidence; record rejected theories as well. diff --git a/tests/guard/test_closeout_generator.py b/tests/guard/test_closeout_generator.py new file mode 100644 index 0000000..0c821db --- /dev/null +++ b/tests/guard/test_closeout_generator.py @@ -0,0 +1,149 @@ +"""Tests for the closeout artifact generator (FIND parsing + findings.yaml/report.md derivation).""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from plugins.violin_guard.findings import ( + generate_findings_yaml, + generate_report_md, + parse_finding_file, +) + +FIND_MD = """# FIND-002: IDOR - arbitrary order read + +- **Severity:** High +- **Batch:** 356aff5b-1674-4693-8417-96d2e9b55248 +- **PTT task:** PT-102 +- **Phase:** VULN_RESEARCH +- **Hypothesis:** H-003 + +## Description + +GET /api/v1/orders/{id} performs no ownership check. + +## Impact + +Horizontal privilege escalation. + +## Evidence + +- `evidence/executions/2026-08-10T094638-ee4dead5-command.json` +- `evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt` + +## Remediation + +Enforce object-ownership authorization. +""" + + +# --------------------------------------------------------------------------- +# Task 1: FIND file parser +# --------------------------------------------------------------------------- +def test_parse_finding_file_extracts_fields(tmp_path: Path) -> None: + f = tmp_path / "FIND-002.md" + f.write_text(FIND_MD, encoding="utf-8") + rec = parse_finding_file(f) + assert rec["id"] == "FIND-002" + assert rec["title"] == "IDOR - arbitrary order read" + assert rec["severity"] == "High" + assert rec["hypothesis"] == "H-003" + assert rec["phase"] == "VULN_RESEARCH" + assert rec["ptt_task"] == "PT-102" + assert rec["batch"] == "356aff5b-1674-4693-8417-96d2e9b55248" + assert rec["evidence"] == [ + "evidence/executions/2026-08-10T094638-ee4dead5-command.json", + "evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt", + ] + assert "no ownership check" in rec["description"] + assert "object-ownership" in rec["remediation"] + + +def test_parse_finding_file_rejects_non_finding(tmp_path: Path) -> None: + f = tmp_path / "notes.md" + f.write_text("# Not a finding\n", encoding="utf-8") + with pytest.raises(ValueError): + parse_finding_file(f) + + +# --------------------------------------------------------------------------- +# Task 2: findings.yaml generator +# --------------------------------------------------------------------------- +def test_generate_findings_yaml_roundtrip(tmp_path: Path) -> None: + findings_dir = tmp_path / "evidence" / "findings" + findings_dir.mkdir(parents=True) + (findings_dir / "FIND-001.md").write_text( + FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8" + ) + out = generate_findings_yaml(tmp_path) + assert out == tmp_path / "evidence" / "reporting" / "findings.yaml" + data = yaml.safe_load(out.read_text(encoding="utf-8")) + assert data["findings"][0]["id"] == "FIND-001" + assert data["findings"][0]["severity"] == "High" + assert data["findings"][0]["hypothesis"] == "H-003" + assert len(data["findings"][0]["evidence"]) == 2 + assert "generated_from" in data + + +def test_generate_findings_yaml_no_findings(tmp_path: Path) -> None: + with pytest.raises(ValueError): + generate_findings_yaml(tmp_path) + + +# --------------------------------------------------------------------------- +# Task 3: report.md generator +# --------------------------------------------------------------------------- +def test_generate_report_md_contents(tmp_path: Path) -> None: + findings_dir = tmp_path / "evidence" / "findings" + findings_dir.mkdir(parents=True) + (findings_dir / "FIND-002.md").write_text(FIND_MD, encoding="utf-8") + out = generate_report_md(tmp_path, target="https://example.test") + text = out.read_text(encoding="utf-8") + assert out == tmp_path / "reporting" / "report.md" + assert "https://example.test" in text + assert "FIND-002" in text + assert "| High | 1 |" in text + assert "no ownership check" in text + assert "Executive Summary" in text + + +def test_generate_report_md_no_overwrite(tmp_path: Path) -> None: + rep = tmp_path / "reporting" / "report.md" + rep.parent.mkdir(parents=True) + rep.write_text("existing", encoding="utf-8") + with pytest.raises(ValueError): + generate_report_md(tmp_path, target="https://example.test") + + +# --------------------------------------------------------------------------- +# Task 4: CLI subcommand +# --------------------------------------------------------------------------- +def test_cli_generate_closeout(tmp_path: Path) -> None: + findings_dir = tmp_path / "evidence" / "findings" + findings_dir.mkdir(parents=True) + (findings_dir / "FIND-001.md").write_text( + FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8" + ) + repo_root = Path(__file__).resolve().parents[2] + r = subprocess.run( + [ + sys.executable, + "scripts/violin_guard.py", + "generate-closeout", + "--eng-dir", + str(tmp_path), + "--target", + "https://example.test", + ], + capture_output=True, + text=True, + cwd=str(repo_root), + ) + assert r.returncode == 0, r.stderr + assert (tmp_path / "evidence" / "reporting" / "findings.yaml").exists() + assert (tmp_path / "reporting" / "report.md").exists()