feat(guard): generated closeout artifacts + framework-owned methodology

- generate-closeout CLI derives findings.yaml and report.md from canonical
  FIND-NNN.md files; agent closeout reduced to executive-summary narrative
- Per-finding recording cut from 4 surfaces to 2 (hypothesis + FIND)
- New skills/pentest/references/exhaustive-coverage.md mandated from SKILL.md;
  Operational Contract (SKILL.md S7) and Citation Discipline (evidence ref S8)
- Benchmark goal/closeout prompts reduced to task + integrity constraint

Tests: 233 passed (7 new); live-verified against real Aug 10 run (10/10
findings, ids+severities match hand-written original).
This commit is contained in:
Violin
2026-08-10 20:52:10 +01:00
parent 3807d22eaa
commit bbc2d93a63
8 changed files with 465 additions and 17 deletions
+8
View File
@@ -3,6 +3,14 @@
## 3.1.0 (Unreleased)
### Automated Benchmark & Evaluation Framework
- Closeout artifacts are now generated, not hand-written: `python scripts/violin_guard.py generate-closeout --eng-dir <dir> --target <target>` derives `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections) from canonical `evidence/findings/FIND-NNN.md` files. Agent closeout duty reduced to the executive-summary narrative — cuts per-finding recording from 4 surfaces to 2 and eliminates the duplicate rewriting that starved exploitation budget and timed out closeout.
- Benchmark goal/closeout prompts reduced to task-only (target + integrity constraint); methodology moved into the framework — new `skills/pentest/references/exhaustive-coverage.md` (coverage matrix, don't-stop-at-first-finding, route discovery, authz/business-logic boundaries, FIND-NNN canonization) mandated from `skills/pentest/SKILL.md` §3, plus an Operational Contract in §7 and Citation Discipline in `evidence-and-verification-discipline.md` §8.
- Evidence-gated scorer now confirms Validated hypotheses via canonical `evidence/findings/FIND-NNN.md` files matched in live-app terms (METHOD + endpoint route, with `/api/v1` optional) in addition to execution bundles, so truncated `tail -c` evidence no longer loses real findings.
- Endpoint+method proof bundles are accepted with a single pattern hit; shared-endpoint challenges (auth/login, checkout, register) are gated by explicit `require`/`require_any` discriminators (role-granted admin, credit granted, negative quantity, external redirect, rate-limit attempt evidence) to prevent cross-challenge false positives.
- Absence-type challenges (`no-rate-limiting`) are scored on repeated identical status codes with no 429/throttle signal instead of requiring the counter-token.
- `neg-quantity-cart` accepts both `PUT /api/v1/cart/items/{id}` and `POST /api/v1/cart/add` vectors.
- Runner no longer invalidates a run when the closeout continuation pass times out or exits non-zero; closeout is reported as a soft `closeout_warning` while scoring proceeds on substantive evidence.
- Schema-drift audit whitelists legitimate `state/report.md` and `state/retrospective.md` closeout artifacts.
- Introduced automated Hermes profile benchmark runner engine (`benchmark/run.py`) and AI-assisted evaluation framework (`benchmark/ai_judge.py`).
- Added `benchmark/indexer.py` for single-pass engagement directory artifact indexing with a 2MB per-file resource limit and cross-platform POSIX path normalization (`.as_posix()`).
- Optimized `benchmark/score.py` pattern matching disk I/O to single-pass $O(F)$ scanning and removed late function-scope imports.
+179
View File
@@ -6,6 +6,8 @@ import re
from pathlib import Path
from typing import Any
import yaml
from . import hypotheses, state
_FINDING_ID_RE = re.compile(r"FIND-(\d{3,})$")
@@ -203,3 +205,180 @@ def _create_from_pending_batch(
"batch_id": batch_id,
"reused": False,
}
# ---------------------------------------------------------------------------
# Closeout artifact generation (FIND parsing + derived exports)
# ---------------------------------------------------------------------------
_HEADING_RE = re.compile(r"^# (FIND-\d{3,}):\s*(?P<title>.+)$")
_BULLET_RE = re.compile(r"^- \*\*(?P<key>[^:*]+):\*\*\s*(?P<value>.*)$")
_SECTION_RE = re.compile(r"^## (?P<name>.+)$")
_EVIDENCE_ITEM_RE = re.compile(r"^- `(?P<path>[^`]+)`\s*$")
_FIELD_KEYS = {
"severity": "severity",
"hypothesis": "hypothesis",
"phase": "phase",
"ptt task": "ptt_task",
"batch": "batch",
}
_SECTION_FIELDS = {
"Description": "description",
"Impact": "impact",
"Remediation": "remediation",
}
_SEVERITY_ORDER = ("Critical", "High", "Medium", "Low", "Info")
def parse_finding_file(path: Path) -> dict[str, Any]:
"""Parse a canonical FIND-NNN.md into a flat record dict."""
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
heading = _HEADING_RE.match(lines[0].strip()) if lines else None
if not heading:
raise ValueError(f"{path.name}: first line must be '# FIND-NNN: <title>'")
record: dict[str, Any] = {
"id": heading.group(1),
"title": heading.group("title").strip(),
"severity": "",
"hypothesis": "",
"phase": "",
"ptt_task": "",
"batch": "",
"description": "",
"impact": "",
"evidence": [],
"remediation": "",
}
section = ""
section_buf: list[str] = []
def flush() -> None:
key = _SECTION_FIELDS.get(section)
if key:
record[key] = "\n".join(section_buf).strip()
for raw in lines[1:]:
line = raw.strip()
sec = _SECTION_RE.match(line)
if sec:
flush()
section = sec.group("name").strip()
section_buf = []
continue
bullet = _BULLET_RE.match(line)
if bullet and not section:
key = _FIELD_KEYS.get(bullet.group("key").strip().lower())
if key:
record[key] = bullet.group("value").strip()
continue
if section == "Evidence":
item = _EVIDENCE_ITEM_RE.match(line)
if item:
record["evidence"].append(item.group("path").strip())
else:
section_buf.append(raw)
flush()
return record
def generate_findings_yaml(eng_dir: str | Path, *, force: bool = False) -> Path:
"""Write evidence/reporting/findings.yaml derived from FIND-*.md files."""
engagement = Path(eng_dir)
findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md"))
if not findings:
raise ValueError("no FIND-*.md files under evidence/findings/")
out = engagement / "evidence" / "reporting" / "findings.yaml"
if out.exists() and not force:
raise ValueError("findings.yaml exists; pass force=True to regenerate")
records = [parse_finding_file(path) for path in findings]
payload = {
"engagement": engagement.name,
"generated_from": ", ".join(path.name for path in findings),
"note": (
"Derived export generated by violin_guard generate-closeout. "
"The canonical records are the per-finding Markdown files; "
"this YAML is a machine-readable summary."
),
"findings": [
{
"id": rec["id"],
"title": rec["title"],
"severity": rec["severity"],
"hypothesis": rec["hypothesis"],
"phase": rec["phase"],
"evidence": rec["evidence"],
}
for rec in records
],
}
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8")
return out
def generate_report_md(eng_dir: str | Path, *, target: str, force: bool = False) -> Path:
"""Write reporting/report.md assembled from FIND-*.md records."""
engagement = Path(eng_dir)
out = engagement / "reporting" / "report.md"
if out.exists() and not force:
raise ValueError("report.md exists; pass force=True to regenerate")
findings = sorted((engagement / "evidence" / "findings").glob("FIND-*.md"))
if not findings:
raise ValueError("no FIND-*.md files under evidence/findings/")
records = [parse_finding_file(path) for path in findings]
counts = {
severity: sum(1 for rec in records if rec["severity"].lower() == severity.lower())
for severity in _SEVERITY_ORDER
}
lines = [
f"# Security Assessment Report — {target}",
"",
f"- **Engagement:** {engagement.name}",
f"- **Target:** {target}",
f"- **Findings:** {len(records)}",
"",
"## Executive Summary",
"",
"<!-- Write 3-6 sentences: overall posture, worst findings, key themes. -->",
"",
"| Severity | Count |",
"|----------|-------|",
]
for severity in _SEVERITY_ORDER:
lines.append(f"| {severity} | {counts[severity]} |")
lines.append("")
for rec in records:
lines += [
f"## {rec['id']}: {rec['title']}",
"",
f"- **Severity:** {rec['severity']}",
]
if rec["hypothesis"]:
lines.append(f"- **Hypothesis:** {rec['hypothesis']}")
if rec["phase"]:
lines.append(f"- **Phase:** {rec['phase']}")
lines += [
"",
"### Description",
"",
rec["description"],
"",
"### Impact",
"",
rec["impact"],
"",
"### Evidence",
"",
*[f"- `{item}`" for item in rec["evidence"]],
"",
"### Remediation",
"",
rec["remediation"],
"",
]
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
return out
+24 -1
View File
@@ -13,7 +13,7 @@ _PROFILE_ROOT = Path(__file__).resolve().parent.parent
if str(_PROFILE_ROOT) not in sys.path:
sys.path.insert(0, str(_PROFILE_ROOT))
from plugins.violin_guard import bootstrap, command, handlers, state
from plugins.violin_guard import bootstrap, command, findings, handlers, state
def _print_result(result) -> int:
@@ -55,6 +55,20 @@ def cmd_validate_scope(args: argparse.Namespace) -> int:
return code
def cmd_generate_closeout(args: argparse.Namespace) -> int:
try:
yaml_path = findings.generate_findings_yaml(args.eng_dir, force=args.force)
report_path = findings.generate_report_md(
args.eng_dir, target=args.target, force=args.force
)
except ValueError as exc:
print(f"BLOCK: {exc}")
return 1
print(f"OK: wrote {yaml_path}")
print(f"OK: wrote {report_path}")
return 0
def cmd_record_ptt(args: argparse.Namespace) -> int:
out = json.loads(
handlers.handle_record_ptt(
@@ -311,6 +325,15 @@ def main() -> int:
p = sub.add_parser("check-release", help="Run release checks")
p.set_defaults(func=cmd_check_release)
p = sub.add_parser(
"generate-closeout",
help="Derive findings.yaml and report.md from canonical FIND-*.md files",
)
p.add_argument("--eng-dir", required=True)
p.add_argument("--target", required=True)
p.add_argument("--force", action="store_true", help="Overwrite existing artifacts")
p.set_defaults(func=cmd_generate_closeout)
# search-exploit
p = sub.add_parser("search-exploit", help="Search local ExploitDB (read-only)")
p.add_argument("--product", default="")
+8
View File
@@ -92,6 +92,8 @@ Violin is intended for Kali Linux or Parrot OS first. The `terminal` tool may st
Engagement sequence: `SCOPING ──► RECON ──► VULN RESEARCH ──► EXPLOITATION ──► REPORTING ──► RETROSPECTIVE`
> **Exhaustive coverage standard**: every full-scope engagement applies `references/exhaustive-coverage.md` — open it via `skill_view` at engagement start and keep it applied throughout. Build and disposition a coverage matrix (routes × methods, parameters, role boundaries, auth/session flows, business logic, redirects, rate limits, SSRF, injection, object authorization); do not stop at the first finding; route discovery from the live application bundle is required; canonize every Validated hypothesis into `evidence/findings/FIND-NNN.md`.
| Phase / Intent | Playbook Path | Description |
|---|---|---|
| **SCOPING** | `playbooks/scoping.md` | Scoping, RoE definition, project setup, 9-question clarify |
@@ -198,6 +200,12 @@ This prevents re-running scans, missing cross-phase patterns, and losing the inv
> **Execution Context Rule:** During Hermes sessions, **always use typed plugin tools** (`violin_exec`, `violin_record_ptt`, `violin_review_batch`, `violin_status`). Standalone CLI subcommands (`python scripts/violin_guard.py ...`) are strictly for host administration or offline diagnostic inspection via `terminal`.
> **Operational Contract (mandatory, applies to every batch):**
> - **Complete commands only**: a batch `commands` array must contain only complete shell commands — never labels, descriptions, or prose. An entry that is not an executable command is a contract violation.
> - **Fix and rerun**: after any nonzero exit or stderr traceback, fix the command and rerun it before closing the phase — a failed command is not a result.
> - **Review without skill override**: when reviewing a completed batch via `violin_review_batch`, omit the optional `skill` argument so the active execution binding is used.
> - **Friction log**: log tool friction, guard errors, and any guard-code inspection in `state/framework_feedback.md`.
| Subcommand / Tool | Type | Action | Parameters |
|---|---|---|---|
| `violin_exec` | Typed Tool | Execute target command with scope & history tracking | `eng_dir`, `phase`, `target`, `command` |
+11 -16
View File
@@ -28,7 +28,11 @@ Before generating the report, systematically review all evidence collected durin
1. **Read the hypothesis board**`read_file path="$ENG_DIR/hypotheses.md"` to see all theories and their resolution status. Do NOT overwrite or collapse `$ENG_DIR/hypotheses.md` into plain narrative text; final reports belong in `$ENG_DIR/reporting/report.md`.
2. **Locate evidence** — navigate to `$ENG_DIR/evidence/` and enumerate all subdirectories and files. Evidence files MUST reside under `$ENG_DIR/evidence/<phase>/` (never inside `$ENG_DIR/state/`).
3. **Verify reproducibility** — each validated finding has raw evidence plus a `templates/verification-receipt.yaml` receipt with `state: validated`, `oracle_kind`, `actual_signal`, and artifact paths.
4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; `evidence/reporting/findings.yaml` may be generated as a derived export but cannot replace it.
4. **Verify canonical findings** — every Validated hypothesis links an existing `evidence/findings/FIND-NNN.md`. This Markdown record is authoritative; the derived exports are **generated, not hand-written**:
```bash
python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>"
```
This writes `evidence/reporting/findings.yaml` and a complete `reporting/report.md` skeleton (metadata, severity table, per-finding sections pulled from the FIND files). Never re-type finding content that already exists in a FIND file.
5. **Categorize findings** into three tiers:
| Tier | Criteria | Action |
@@ -161,21 +165,11 @@ Briefly mention rejected findings to demonstrate thorough testing:
- Exploitation
- Post-Exploitation
- Reporting
5. **Write each finding** — using the finding format:
- Title
- Severity (Critical / High / Medium / Low / Info)
- CVSS score and vector (if applicable)
- CVE identifier (if applicable)
- Confidence (Certain / High / Medium / Low)
- Description
- Evidence (command + output)
- Impact
- Remediation
6. **Generate summary table** — create a table counting findings by severity
5. **Generate the report skeleton** — run `python scripts/violin_guard.py generate-closeout --eng-dir "$ENG_DIR" --target "<target>"` (see §Verify canonical findings). The skeleton already contains every finding's title, severity, description, impact, evidence list, and remediation pulled from the FIND files, plus the severity-count table. Do NOT hand-write per-finding sections again.
6. **Write the narrative** — your only required hand-written content is the **Executive Summary** (3-6 sentences: overall posture, worst findings, key themes) replacing the placeholder comment. Optionally add methodology and limitations prose.
7. **Document limitations** — any constraints that affected testing (time, access, tooling, scope)
8. **Build appendix** — list all evidence file paths with brief descriptions
9. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices
10. **Save** — write the completed report to `$ENG_DIR/reporting/report.md`
8. **Attach auto-fix patches** — if any auto-patches were generated during exploitation, include them as remediation appendices
9. **Save** — the generator already wrote `$ENG_DIR/reporting/report.md`; edit it in place for the narrative
## Auto-Fix Evidence Compilation
@@ -212,7 +206,8 @@ Before delivering the report, verify every item:
- [ ] All evidence files are referenced by their full or relative path
- [ ] Severity ratings are justified
- [ ] Executive summary reflects the actual findings (not boilerplate)
- [ ] Report is saved to `$ENG_DIR/reporting/report.md`
- [ ] Report is saved to `$ENG_DIR/reporting/report.md` (generated via `generate-closeout`, narrative filled in)
- [ ] `evidence/reporting/findings.yaml` generated via `generate-closeout` (never hand-written)
- [ ] Retrospective is saved to `$ENG_DIR/retrospective/retrospective.md`
- [ ] Final closeout task marked completed: invoke `violin_record_ptt(id="<closeout-task-id>", status="[x]", note="...")` to mark engagement closeout completed in `ptt.md`.
@@ -96,3 +96,12 @@ to cross, is the failure mode. Surface real blockers precisely and move on.
If no primary/accessible evidence supports a value, report it as **unsupported /
unconfirmed** rather than reconstructing it and presenting it as observed. A
finding's confidence is only as honest as the evidence trail behind it.
## 8. Citation discipline
Every `evidence/` citation in hypotheses, findings, and reports must name an
**existing concrete file with its extension** (`evidence/exploitation/2026-01-01T00-00-00-hash-command.stdout.txt`).
Never cite a directory, wildcard, path stem, shell fragment, or command
substitution, and never use `evidence/` as a generic prose noun. A citation
that cannot be resolved to a real file is a defect in the engagement record —
fix the reference before closeout, do not leave it for the reader to guess.
@@ -0,0 +1,77 @@
# Exhaustive Assessment Coverage
Mandatory operating standard for a full-scope engagement. The objective of a
full assessment is a **complete, dispositioned coverage matrix** — not a
stopping point after the first confirmed issue. Applies in every phase and
complements `evidence-and-verification-discipline.md` and the phase playbooks.
## 1. Build and disposition a coverage matrix
Track coverage for the engagement's surface:
- routes and methods (every discovered endpoint × every permitted method)
- parameters and input vectors
- role boundaries (guest / user / admin / other principals) and object authorization
- authentication and session flows
- business logic
- redirects, rate limits, SSRF, injection, and object-authorization classes
A cell is either **tested** (with a recorded request + response, positive or
negative) or **explicitly dispositioned** (with a reason: out of scope, guarded,
not present on this target). Do not close a phase until the matrix is complete.
## 2. Do not stop after the first finding
Record the issue (hypothesis → Validated, canonize a finding), then **continue
testing the remaining matrix cells**. A confirmed finding is a checkpoint, not a
termination condition.
## 3. Route discovery is a required step
- Derive endpoints from the live application bundle (JS, HTML, API schemas,
OpenAPI/swagger, robots.txt, sitemap) before guessing.
- Fuzz a supplemental wordlist of common API nouns and actions over each
resource.
- Treat 401/403/405/3xx responses as route evidence; inspect `Allow` headers and
retry every discovered route with each permitted method.
- Try path-parameter, trailing-slash, case, and trailing-`/` variants.
- Probe ordinary action/verb variants of each discovered resource: list,
detail, create, update, delete, import, export, upload, preview, validate,
status, and similar.
- Do not reject a whole class because one guessed route is absent — derive the
exact route and method from the application and test each variant.
## 4. Authentication and session boundaries (as present on the target)
- Token signature, algorithm, and claim manipulation.
- Default or weak credential behavior **without credential stuffing**.
- Any multi-factor or one-time-code flow: empty, invalid, replayed, and
valid-code boundaries.
- Password-reset and account-recovery flows.
## 5. Authorization boundaries
- Object-level access control: horizontal (another principal's resources,
including object IDs) and vertical (privileged roles).
- Function-level access control (e.g. privileged operations reachable as a
lower-privilege principal).
- Unauthenticated state-changing paths.
For every discovered object ID, retry the access under a different principal's
session to test horizontal authorization.
## 6. Business-logic edges
- Negative or zero quantities, omitted required fields, price/cost
manipulation, and duplicate or concurrent submissions.
## 7. Proof and evidence discipline
- Capture proofs with `curl --http1.1 -i` so every target request prints a
canonical `METHOD /path HTTP/1.1` request line, the `HTTP/1.1 NNN` response
line, and the response body. Labels alone are not proof.
- Correlate request and response evidence in guarded execution bundles.
- Record **negative results** with the exact request and response too.
- Canonize: every validated issue becomes a Validated hypothesis linked to a
canonical `evidence/findings/FIND-NNN.md` record with concrete runtime
evidence; record rejected theories as well.
+149
View File
@@ -0,0 +1,149 @@
"""Tests for the closeout artifact generator (FIND parsing + findings.yaml/report.md derivation)."""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
import pytest
import yaml
from plugins.violin_guard.findings import (
generate_findings_yaml,
generate_report_md,
parse_finding_file,
)
FIND_MD = """# FIND-002: IDOR - arbitrary order read
- **Severity:** High
- **Batch:** 356aff5b-1674-4693-8417-96d2e9b55248
- **PTT task:** PT-102
- **Phase:** VULN_RESEARCH
- **Hypothesis:** H-003
## Description
GET /api/v1/orders/{id} performs no ownership check.
## Impact
Horizontal privilege escalation.
## Evidence
- `evidence/executions/2026-08-10T094638-ee4dead5-command.json`
- `evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt`
## Remediation
Enforce object-ownership authorization.
"""
# ---------------------------------------------------------------------------
# Task 1: FIND file parser
# ---------------------------------------------------------------------------
def test_parse_finding_file_extracts_fields(tmp_path: Path) -> None:
f = tmp_path / "FIND-002.md"
f.write_text(FIND_MD, encoding="utf-8")
rec = parse_finding_file(f)
assert rec["id"] == "FIND-002"
assert rec["title"] == "IDOR - arbitrary order read"
assert rec["severity"] == "High"
assert rec["hypothesis"] == "H-003"
assert rec["phase"] == "VULN_RESEARCH"
assert rec["ptt_task"] == "PT-102"
assert rec["batch"] == "356aff5b-1674-4693-8417-96d2e9b55248"
assert rec["evidence"] == [
"evidence/executions/2026-08-10T094638-ee4dead5-command.json",
"evidence/executions/2026-08-10T094638-ee4dead5-command.stdout.txt",
]
assert "no ownership check" in rec["description"]
assert "object-ownership" in rec["remediation"]
def test_parse_finding_file_rejects_non_finding(tmp_path: Path) -> None:
f = tmp_path / "notes.md"
f.write_text("# Not a finding\n", encoding="utf-8")
with pytest.raises(ValueError):
parse_finding_file(f)
# ---------------------------------------------------------------------------
# Task 2: findings.yaml generator
# ---------------------------------------------------------------------------
def test_generate_findings_yaml_roundtrip(tmp_path: Path) -> None:
findings_dir = tmp_path / "evidence" / "findings"
findings_dir.mkdir(parents=True)
(findings_dir / "FIND-001.md").write_text(
FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8"
)
out = generate_findings_yaml(tmp_path)
assert out == tmp_path / "evidence" / "reporting" / "findings.yaml"
data = yaml.safe_load(out.read_text(encoding="utf-8"))
assert data["findings"][0]["id"] == "FIND-001"
assert data["findings"][0]["severity"] == "High"
assert data["findings"][0]["hypothesis"] == "H-003"
assert len(data["findings"][0]["evidence"]) == 2
assert "generated_from" in data
def test_generate_findings_yaml_no_findings(tmp_path: Path) -> None:
with pytest.raises(ValueError):
generate_findings_yaml(tmp_path)
# ---------------------------------------------------------------------------
# Task 3: report.md generator
# ---------------------------------------------------------------------------
def test_generate_report_md_contents(tmp_path: Path) -> None:
findings_dir = tmp_path / "evidence" / "findings"
findings_dir.mkdir(parents=True)
(findings_dir / "FIND-002.md").write_text(FIND_MD, encoding="utf-8")
out = generate_report_md(tmp_path, target="https://example.test")
text = out.read_text(encoding="utf-8")
assert out == tmp_path / "reporting" / "report.md"
assert "https://example.test" in text
assert "FIND-002" in text
assert "| High | 1 |" in text
assert "no ownership check" in text
assert "Executive Summary" in text
def test_generate_report_md_no_overwrite(tmp_path: Path) -> None:
rep = tmp_path / "reporting" / "report.md"
rep.parent.mkdir(parents=True)
rep.write_text("existing", encoding="utf-8")
with pytest.raises(ValueError):
generate_report_md(tmp_path, target="https://example.test")
# ---------------------------------------------------------------------------
# Task 4: CLI subcommand
# ---------------------------------------------------------------------------
def test_cli_generate_closeout(tmp_path: Path) -> None:
findings_dir = tmp_path / "evidence" / "findings"
findings_dir.mkdir(parents=True)
(findings_dir / "FIND-001.md").write_text(
FIND_MD.replace("FIND-002", "FIND-001"), encoding="utf-8"
)
repo_root = Path(__file__).resolve().parents[2]
r = subprocess.run(
[
sys.executable,
"scripts/violin_guard.py",
"generate-closeout",
"--eng-dir",
str(tmp_path),
"--target",
"https://example.test",
],
capture_output=True,
text=True,
cwd=str(repo_root),
)
assert r.returncode == 0, r.stderr
assert (tmp_path / "evidence" / "reporting" / "findings.yaml").exists()
assert (tmp_path / "reporting" / "report.md").exists()