From 5071b9689fc79e3f5d418549a817603c5f61c19b Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 4 Aug 2026 17:18:44 +0100 Subject: [PATCH] docs(release): align workflow instructions, hypothesis discipline, and release v3.0.1 (#64) Merge dev into master --- .hermes.md | 25 +++-- AGENTS.md | 60 ++++++++++++ README.md | 2 +- plugins/violin_guard/hypotheses.py | 77 ++++++++++++--- plugins/violin_guard/schemas.py | 11 +++ skills/pentest/SKILL.md | 27 ++++-- skills/pentest/playbooks/recon.md | 6 +- .../evidence-and-verification-discipline.md | 95 +++++++++++++++++++ skills/pentest/references/flags-mode.md | 41 ++++++++ skills/pentest/references/retrospective.md | 2 + skills/pentest/references/standards.md | 5 + .../templates/flag-capture-register.md | 36 +++++++ skills/pentest/templates/hypothesis-board.md | 26 ++++- tests/guard/state/test_collaboration_ux.py | 59 ++++++++++++ 14 files changed, 441 insertions(+), 31 deletions(-) create mode 100644 AGENTS.md create mode 100644 skills/pentest/references/evidence-and-verification-discipline.md create mode 100644 skills/pentest/references/flags-mode.md create mode 100644 skills/pentest/templates/flag-capture-register.md diff --git a/.hermes.md b/.hermes.md index af4bb30..993c4a4 100644 --- a/.hermes.md +++ b/.hermes.md @@ -36,14 +36,18 @@ Violin **must not** touch a target (no curl, nmap, browser, web_search for the t | 4 | `$ENG_DIR/hypotheses.md` (hypothesis board) | `init-engagement` | `check-bootstrap` | | 5 | `$ENG_DIR/state/history.md` (command log) | `init-engagement` | `check-bootstrap` | -**Session-start/skill-load precondition** (code-enforced via `check-command`): +**Session-start/skill-delivery precondition** (code-enforced via the plugin hook): -Launch with `hermes chat --skills pentest` when possible. If the profile was already started, load `pentest` before creating the marker below. Check the current state at any time with `python $HOME/.hermes/profiles/violin/scripts/violin_guard.py status --eng-dir "$ENG_DIR" --section skill`. +Launch with `hermes chat --skills pentest` when possible. For every active PTT +task, call `violin_record_ptt` with its routed `skill`, `technique`, and any +required hypothesis. The first call prepares real `skill_view` content without +changing the PTT; after that result returns, repeat the same call in the next +model continuation to bind the receipt. Check the current state with +`violin_status` or `python $HOME/.hermes/profiles/violin/scripts/violin_guard.py +status --eng-dir "$ENG_DIR" --section skill`. -```text -printf 'skill-loaded: %s\n' "$(date -Iseconds)" > "$ENG_DIR/state/.skill-loaded-" -python $HOME/.hermes/profiles/violin/scripts/violin_guard.py status --eng-dir "$ENG_DIR" --section skill -``` +Never create `.skill-loaded-*` files. Legacy markers are migration hints only +and are not evidence that Hermes delivered a skill. Then pass the same session ID into every target-touching command check — `--eng-dir` is mandatory because it activates the PTT, history, hypothesis, and synchronization guards: @@ -51,7 +55,10 @@ Then pass the same session ID into every target-touching command check — `--en python $HOME/.hermes/profiles/violin/scripts/violin_guard.py check-command --scope $ENG_DIR/scope/scope.yaml --eng-dir "$ENG_DIR" --target "" --session-id "" --phase --command "" ``` -**Run-to-completion rule:** once created, a skill-load marker holds for the current engagement work block. Do not recreate it between commands. After `/goal set` or context compression, continue in the same conversation, re-read `$ENG_DIR/state/`, and create or verify the marker for the current session ID before the next target action. +**Run-to-completion rule:** after `/goal set` or context compression, continue +in the same conversation, re-read `$ENG_DIR/state/`, and call `violin_status`. +If the receipt binding needs recovery, follow its reported action; do not +manufacture marker files or assume the prior skill delivery still applies. **What this prevents:** the Nimbus-style failure where the agent skipped scoping/bootstrap, never created a PTT or hypothesis board, and burned the entire 20-turn budget calling `curl` variations on the same endpoint without tracking what it learned. @@ -62,7 +69,7 @@ The Violin agent **may**: - **Run pentest tools** through `violin_exec`, `violin_exec_burst`, or the typed guard adapters. `violin_exec` has no binary allowlist: any installed non-interactive Kali/Parrot CLI tool may run against the explicit in-scope target while retaining scope, phase, PTT, hypothesis, history, evidence, timeout, and sync gates. Installation, root, hardware, service, GUI, and interactive-TTY requirements remain runtime constraints. Raw `terminal` is limited to host-local preparation and administration. - **Use `execute_code` only with an audit header** — its first line must be `# violin: {"eng_dir":"...","phase":"...","target":"...","session_id":"..."}`. Violin validates that metadata, saves the submitted source under `$ENG_DIR/evidence//`, and appends its digest and completion status to history. This is auditability, not a substitute for typed target execution. - **Research online** — Use the `web` toolset (`web_search`, `web_extract`) to find CVEs, exploits, PoC code, security advisories, tool documentation, and tutorials. -- **Browse target websites** — Use the `browser` toolset (`browser_navigate`, `browser_click`, `browser_type`, `browser_vision`, `browser_snapshot`) to navigate target web apps, interact with login forms and dashboards, capture screenshots for evidence, inspect DOM elements, crawl visible links, and enumerate client-side routes. +- **Browse target websites carefully** — Use the `browser` toolset only for an explicitly approved in-scope URL after the PTT and skill-receipt gates pass. In v3.0.0 the hook gates browser activity on that workflow binding; it is not a network-level browser allowlist and does not contain redirects, subresources, forms, JavaScript navigation, history, or WebSockets. Stop and review the URL after each navigation or action; do not deliberately follow an out-of-scope redirect. - **Discover installed tools** — Check what tools exist on the workstation using `command -v` or filesystem searches. - **Read tool docs** — Use `read_file`, `terminal`, or `browser` to read help pages, man pages, READMEs, and documentation. - **Write scripts** — Create Python, Bash, PowerShell, or other scripts under `$ENG_DIR/exploits/` for exploitation, automation, or evidence processing. @@ -75,7 +82,7 @@ The Violin agent **may**: - **Summarise results** — After each logical tool batch, report: (a) what ran, (b) key results found or nothing notable, (c) evidence saved where. 3-5 lines max. Do not dump raw command output. - **Check in next-step** — After each sub-phase or completed batch, ask the user what to do next with concrete options. E.g. "DNS enumeration done. Found 3 subdomains. Next: tech detection with whatweb, or move to active scanning with nmap?" Do not advance the workflow silently. - **Ask via `clarify`** — Ask targeted questions during scoping and throughout the engagement whenever ambiguity arises. -- **Delegate** — Use background `terminal` processes to run parallel reconnaissance or long-running tasks. +- **Delegate** — Use `violin_exec` with `background=true` or `violin_listener` for tracked long-running work. Do not start target-facing background jobs through raw `terminal`. - **Use vision** — Capture and analyse screenshots with `browser_vision` or `vision_analyze` for evidence and context understanding. ## Conversation & Memory Isolation diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..318ebee --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,60 @@ +# Strategic-Automation/violin — AI Developer Guidance + +> Workspace-scoped developer guidance for AI coding agents (Antigravity, Hermes, Codex, Cursor, etc.) developing, testing, or maintaining the `violin` codebase. +> +> *Note:* For deployed end-user Hermes pentest installations, runtime identity and engagement rules are packaged in `SOUL.md`, `.hermes.md`, and `skills/pentest/SKILL.md`. This file governs AI agent developer behavior within this repository workspace. + +--- + +## 1. Stack & Environment Setup + +- **Python Version:** 3.11 (pinned in `.python-version` and `pyproject.toml` to match Hermes runtime). +- **Package Manager:** `uv` (use `uv sync --dev` to sync development environment). +- **Virtual Environment:** `.venv` created and managed via `uv`. + +--- + +## 2. Mandatory Verification Commands + +Run these commands to verify any code changes before declaring completion: + +```bash +# 1. Run full test suite (must pass 100%) +uv run pytest + +# 2. Run linter check +uv run ruff check . + +# 3. Check code formatting (fix with `uv run ruff format .`) +uv run ruff format --check . + +# 4. Validate release gate +uv run python -m plugins.violin_guard.release +``` + +--- + +## 3. Code Conventions & Architecture + +- **Hermes Runtime Contract:** All target-touching CLI command execution in Violin engagements MUST go through `plugins.violin_guard` typed Hermes tool calls (`violin_exec`, `violin_record_ptt`, `violin_review_batch`, `violin_record_hypothesis`, `violin_target`, `violin_status`, `violin_listener`). Never invoke flat CLI scripts (`python violin_guard.py`) as a substitute for typed tool calls. +- **Fail-Closed Validation:** All state parsers (`hypotheses.py`, `ptt.py`, `command.py`, `targets.py`) must validate inputs fail-closed before mutating filesystem state. +- **Section Preservation:** File rewriters (`_rewrite_hypotheses`, `update_task`) must preserve structural template sections (`## Observations`, `## Decoy Trail`, `## Research Log`, `## Resolved Theories`, table columns). +- **Typed Schemas:** Use Pydantic v2 `BaseModel` models in `plugins/violin_guard/schemas.py` for all tool parameter specifications. +- **File Encoding:** Always specify `encoding="utf-8"` explicitly for all text file read/write operations. + +--- + +## 4. Git & Branching Strategy + +- **Feature/Docs Branches:** Use `codex/` or `dev`. +- **Merge Flow:** `codex/` ──► `dev` ──► PR to `master`. +- **Master Branch:** `master` is protected by GitHub repository rules (`GH013`); production releases require a Pull Request. + +--- + +## 5. Hard Boundaries (What AI Agents Must Never Do) + +1. **NEVER Bypass Target Execution Guards:** Never run target-touching commands directly in raw shell without `violin_exec` or `violin_exec_burst`. +2. **NEVER Swallow Exceptions or Patch Tests Superficialy:** Fix underlying root causes; never mask errors, return dummy fallbacks, or comment out failing assertions. +3. **NEVER Hardcode Target IPs:** Resolve target hosts dynamically via `violin_target` or `scope.yaml`. +4. **NEVER Declare Success Without Empirical Verification:** Always run `uv run pytest` and `uv run ruff check .` to prove zero regressions before finishing work. diff --git a/README.md b/README.md index b5b1f4f..ef37ad5 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ hermes -p violin 🔬 31 Methodology Playbooks7 operational playbooks (five execution phases, optional post-exploitation, and the tools catalog) + 24 vulnerability-class playbooks, routed across the `pentest`, `web-attacks`, and `access-control` skills. 🛡️ Multi-Layer SafetyInteractive scoping (9 questions) → scope validation → guard check → approval gates — every target-touching command validated before execution. 🧠 Pentesting Task TreeStructured artifact tracking every task via `[x]/[ ]/[~]` markers across phases, with executor-owned history, hypothesis linking, and guard-bound batch reviews. -🌐 Browser + Web ResearchBrowser toolset for website enumeration. Web toolset for CVE lookup, exploit search, and OSINT. +🌐 Browser + Web ResearchBrowser toolset for approved in-scope website enumeration; v3.0.0 gates the engagement workflow but does not provide a network-level browser allowlist. Web toolset for CVE lookup, exploit search, and OSINT. 📋 Evidence-Driven ReportingReproducible evidence with screenshots, tool output, and request/response pairs. CVSS 3.1 + 4.0 crosswalks and optional remediation patches. 🔗 Hermes-NativeInherits your existing Hermes provider, model, and tool backends. Violin introduces no separate credential store or broker. diff --git a/plugins/violin_guard/hypotheses.py b/plugins/violin_guard/hypotheses.py index f87e38c..e519933 100644 --- a/plugins/violin_guard/hypotheses.py +++ b/plugins/violin_guard/hypotheses.py @@ -52,6 +52,12 @@ _FIELD_NAMES = { "source evidence": "source_evidence", "runtime evidence": "runtime_evidence", "updated": "updated", + "confidence": "confidence", + "timebox": "timebox", + "cheapest test": "cheapest_test", + "kill criteria": "kill_criteria", + "next step": "next_step", + "linked findings": "linked_findings", } @@ -60,6 +66,9 @@ class Hypothesis: id: str title: str status: str = "Candidate" + confidence: str = "" + timebox: str = "" + cheapest_test: str = "" phase: str = "" service: str = "" port: str = "" @@ -72,7 +81,10 @@ class Hypothesis: test_command: str = "" test_response: str = "" verification_status: str = "" + kill_criteria: str = "" rejection_reason: str = "" + next_step: str = "" + linked_findings: str = "" candidate_source: str = "" entry_point: str = "" data_flow: str = "" @@ -88,6 +100,9 @@ class Hypothesis: "id": self.id, "title": self.title, "status": self.canonical_status(), + "confidence": self.confidence, + "timebox": self.timebox, + "cheapest_test": self.cheapest_test, "phase": self.phase, "service": self.service, "port": self.port, @@ -100,7 +115,10 @@ class Hypothesis: "test_command": self.test_command, "test_response": self.test_response, "verification_status": self.verification_status, + "kill_criteria": self.kill_criteria, "rejection_reason": self.rejection_reason, + "next_step": self.next_step, + "linked_findings": self.linked_findings, "candidate_source": self.candidate_source, "entry_point": self.entry_point, "data_flow": self.data_flow, @@ -113,7 +131,12 @@ class Hypothesis: now = datetime.now(UTC).strftime("%Y-%m-%d %H:%M") lines = [f"### H-{self.id}: {self.title}"] lines.append(f"- **Status:** {self.canonical_status()}") - lines.append(f"- **Updated:** {self.updated or now}") + if self.confidence: + lines.append(f"- **Confidence:** {self.confidence}") + if self.timebox: + lines.append(f"- **Timebox:** {self.timebox}") + if self.cheapest_test: + lines.append(f"- **Cheapest test:** {self.cheapest_test}") if self.phase: lines.append(f"- **Phase:** {self.phase}") if self.service: @@ -138,8 +161,14 @@ class Hypothesis: lines.append(f"- **Test Response:** {self.test_response}") if self.verification_status: lines.append(f"- **Verification Status:** {self.verification_status}") + if self.kill_criteria: + lines.append(f"- **Kill criteria:** {self.kill_criteria}") if self.rejection_reason: lines.append(f"- **Rejection Reason:** {self.rejection_reason}") + if self.next_step: + lines.append(f"- **Next step:** {self.next_step}") + if self.linked_findings: + lines.append(f"- **Linked findings:** {self.linked_findings}") for label, value in ( ("Candidate Source", self.candidate_source), ("Entry Point", self.entry_point), @@ -149,7 +178,7 @@ class Hypothesis: ): if value: lines.append(f"- **{label}:** {value}") - lines.append(f"- **Updated:** {self.updated or now} UTC") + lines.append(f"- **Updated:** {self.updated or now}") return "\n".join(lines) + "\n" @@ -322,6 +351,9 @@ def update_hypothesis( id=merged_fields["id"], title=merged_fields.get("title", "") or f"Hypothesis {merged_fields['id']}", status=(merged_fields.get("status") or "Candidate"), + confidence=(merged_fields.get("confidence") or "").strip(), + timebox=(merged_fields.get("timebox") or "").strip(), + cheapest_test=(merged_fields.get("cheapest_test") or "").strip(), phase=(merged_fields.get("phase") or "").strip(), service=(merged_fields.get("service") or "").strip(), port=(merged_fields.get("port") or "").strip(), @@ -334,7 +366,10 @@ def update_hypothesis( test_command=(merged_fields.get("test_command") or "").strip(), test_response=(merged_fields.get("test_response") or "").strip(), verification_status=(merged_fields.get("verification_status") or "").strip(), + kill_criteria=(merged_fields.get("kill_criteria") or "").strip(), rejection_reason=(merged_fields.get("rejection_reason") or "").strip(), + next_step=(merged_fields.get("next_step") or "").strip(), + linked_findings=(merged_fields.get("linked_findings") or "").strip(), candidate_source=(merged_fields.get("candidate_source") or "").strip(), entry_point=(merged_fields.get("entry_point") or "").strip(), data_flow=(merged_fields.get("data_flow") or "").strip(), @@ -378,23 +413,41 @@ def update_hypothesis( def _rewrite_hypotheses(path: Path, hypotheses_list: list[Hypothesis]) -> None: - """Rewrite the entire hypotheses file.""" + """Rewrite the hypotheses file while preserving structural sections (Decoy Trail, Observations, etc.).""" path.parent.mkdir(parents=True, exist_ok=True) template = path.read_text(encoding="utf-8") if path.exists() else "# Hypothesis Board\n\n" - # Template instructions are an HTML comment containing an example H-001 - # heading. Remove that comment before locating real records, otherwise a - # newly written hypothesis is accidentally placed inside the comment. + + # Remove template instruction HTML comment if present + import re + comment_start = template.find("", comment_start) if comment_end != -1: template = template[:comment_start] + template[comment_end + 3 :] - # Keep any header content before first hypothesis - header_end = template.find("### H-") - if header_end == -1: - header = template.strip() + "\n\n" + + # Preserve section structure (e.g. ## Active Theories ... ## Observations ... ## Decoy Trail) + active_heading = "## Active Theories" + active_pos = template.find(active_heading) + + if active_pos != -1: + header = template[: active_pos + len(active_heading)].strip() + "\n\n" + # Find next section header after Active Theories + next_sec = re.search(r"\n##\s+(?!Active Theories)", template[active_pos:]) + trailer = template[active_pos + next_sec.start() + 1 :].lstrip() if next_sec else "" else: - header = template[:header_end].rstrip() + "\n\n" + # Fallback: locate first hypothesis heading starting with ### H- + first_h = re.search(r"^###\s+H-", template, re.MULTILINE) + if first_h: + header = template[: first_h.start()].rstrip() + "\n\n" + next_sec = re.search(r"\n##\s+", template[first_h.start() :]) + trailer = ( + template[first_h.start() + next_sec.start() + 1 :].lstrip() if next_sec else "" + ) + else: + header = template.strip() + "\n\n" + trailer = "" body = "\n".join(h.to_markdown() for h in hypotheses_list) - path.write_text(header + body, encoding="utf-8") + content = header + body + ("\n\n" + trailer if trailer else "\n") + path.write_text(content, encoding="utf-8") diff --git a/plugins/violin_guard/schemas.py b/plugins/violin_guard/schemas.py index 04cec00..7430608 100644 --- a/plugins/violin_guard/schemas.py +++ b/plugins/violin_guard/schemas.py @@ -58,6 +58,11 @@ class RecordHypothesisArgsModel(BaseModel): id: str = "" title: str = "" status: str = "" + confidence: str = Field("", description="0.1-1.0 guesstimate; escalate only with evidence") + timebox: str = Field("", description="e.g. 4 tool batches or 30 min — then re-evaluate") + cheapest_test: str = Field( + "", description="Single cheapest probe that discriminates this theory" + ) phase: str = "" target: str = Field("", description="target host/IP (must be in scope)") vuln_class: str = "" @@ -80,9 +85,15 @@ class RecordHypothesisArgsModel(BaseModel): test_command: str = Field("", description="Exact syntax tested, including argument order") test_response: str = Field("", description="Exact decisive response or error") verification_status: str = "" + kill_criteria: str = Field( + "", + description="Evidence that contradicts, or no new info in N batches — then kill & log in Decoy Trail", + ) rejection_reason: str = Field( "", description="Why a rejected hypothesis is safe to stop pursuing" ) + next_step: str = "" + linked_findings: str = "" candidate_source: str = "" entry_point: str = "" data_flow: str = "" diff --git a/skills/pentest/SKILL.md b/skills/pentest/SKILL.md index ae48c6f..8ecf07e 100644 --- a/skills/pentest/SKILL.md +++ b/skills/pentest/SKILL.md @@ -55,7 +55,7 @@ The agent operates as the **Pentest Lead** with supervised autonomy only after s **Available capabilities:** - **`terminal`** — host-local preparation and administration only; best-effort policy blocks obvious target traffic, while target-touching commands use the Violin guard tools below - **`web`** (`web_search`, `web_extract`) — research: CVE lookup, exploit search, OSINT, documentation, PoC search -- **`browser`** (`browser_navigate`, `browser_click`, `browser_type`, `browser_vision`, `browser_snapshot`) — website enumeration and interaction: navigate target web apps, interact with login forms and dashboards, capture screenshots, inspect DOM, crawl visible links, enumerate client-side routes +- **`browser`** (`browser_navigate`, `browser_click`, `browser_type`, `browser_vision`, `browser_snapshot`) — website enumeration and interaction for explicitly approved in-scope URLs. The v3.0.0 hook checks the PTT/skill-receipt binding before supported browser actions, but it is not a network-level scope firewall: redirects, subresources, forms, JavaScript navigation, history, and WebSockets are not contained by the guard. Re-check the visible URL after each action and stop for review if it leaves scope. - **`file`** (`write_file`, `read_file`, `search_files`) — evidence collection, report writing, config manipulation - **`code_execution`** — host-local Python automation only with a first-line audit header: `# violin: {"eng_dir":"...","phase":"...","target":"...","session_id":"..."}`. The source and completion record are saved to the named engagement; use typed Violin tools for target execution. - **`clarify`** — ask the user structured questions during scoping and when decisions are needed @@ -140,10 +140,16 @@ The phase workflow is mandatory for the entire session, including long, compress 1. Check/update `todo` with a single active `phase-gate` item named for the current phase. 2. Confirm an approved `$ENG_DIR/scope/scope.yaml` exists before touching any target. If it does not, remain in SCOPING and ask via `clarify`. Verify with `python $HOME/.hermes/profiles/violin/scripts/violin_guard.py validate-scope --scope $ENG_DIR/scope/scope.yaml` (exit 0 required). For an authorized HTB/CTF lab, `init-engagement --ctf --host --session-id "$ENG_DIR"` creates a ready-to-test scope and active RECON PTT row; prepare and bind the routed skill before target activity. This exact guard CLI action is host-local and accepts a direct `--host` value through `terminal`; never hide it in a file, environment variable, or shell substitution. -3. **Read and activate the PTT task** — `read_file path="$ENG_DIR/state/ptt.md"` — and select the next open `[ ]` task for the current phase. Before any target command, mark exactly one task `[~]`: - ```bash - python $HOME/.hermes/profiles/violin/scripts/violin_guard.py record-ptt --eng-dir "$ENG_DIR" --id PT-XXX --status "[~]" --note "starting task" --skill pentest --technique "" [--hypothesis-id H-001] +3. **Read and activate the PTT task** — `read_file path="$ENG_DIR/state/ptt.md"` — and select the next open `[ ]` task for the current phase. Before any target command, mark exactly one task `[~]` through the Hermes tool: + ```text + violin_record_ptt( + eng_dir="$ENG_DIR", id="PT-XXX", status="[~]", note="starting task", + skill="pentest", technique="", hypothesis_id="H-001" # when required + ) ``` + The first call prepares the routed skill receipt; repeat it after that tool + result returns to bind the transition. The standalone CLI is diagnostic and + administrative; it cannot substitute for Hermes `skill_view` delivery. With no pending batch, this is the one permitted PTT start transition. The guard hard-blocks target execution when there is no unambiguous active task or the task sits under another phase heading. The executor never changes this row or its `*Last updated*` timestamp. After each bounded batch, review the results and call `violin_review_batch` with `[~]`, `[x]`, `[!]`, or `[-]` plus a truthful result summary. 4. **Command history is executor-owned** — `violin_exec` appends every completed target command automatically. There is no public history-writing command; inspect recent history before a new batch to avoid repeats. 5. Re-read this skill or the active playbook after context compression, `/resume`, or any uncertainty about the workflow. **Also read back evidence and hypotheses:** `read_file path="$ENG_DIR/hypotheses.md"` and `search_files path="$ENG_DIR/evidence" pattern="" target="files"` to restore investigation state. @@ -170,6 +176,7 @@ The phase workflow is mandatory for the entire session, including long, compress - VULN RESEARCH → `playbooks/vuln-research.md` - EXPLOITATION → `playbooks/exploitation.md` plus relevant vuln-class playbook - POST EXPLOITATION → `playbooks/post-exploitation.md` + - FLAGS (authorised training/lab objective capture) → `playbooks/post-exploitation.md` + `references/flags-mode.md` (`templates/flag-capture-register.md`) - REPORTING → `playbooks/reporting.md` - RETROSPECTIVE → `references/retrospective.md` 8. Before exploit validation, re-check the vuln playbook's `## Stop Conditions` and `## Blocked Actions`. @@ -209,7 +216,7 @@ IF 5 consecutive attempts fail on the same target component: - **When stuck, research online before re-running.** The cheapest unstuck move is almost always new information, not another scan. If a command stalls, errors, or yields nothing new: `web_search` the exact error string + tool name, pull the upstream docs / PoC / CVE advisory via `web_extract`, and only then change the variable (different host, param, wordlist, technique) or move to a new task. Treat the `web` capability as a primary recovery lever, not a last resort. - **Five-attempt research trigger.** After five failed attempts against the same feature or exploit class, stop. Re-read captured source first; then research the exact parser, protocol, or primitive; then test one documented variant. If the source is unavailable or the next safe variant is unclear, ask the user for a hint rather than continuing a circular loop. - **False-negative checkpoint.** Before declaring a feature unavailable, stubbed, or not implemented, match the exact test command and argument order against the captured source/parser. Record the command, decisive response, and `verification_status` in the hypothesis. `syntax_uncertain` is not a rejection: it requires a corrected re-test. -- **Context boundary.** Prefix controller output with `[VICTIM]`; prefix assessment-host commands and notes with `[ATTACKER]`. `/proc`, `/run`, UNIX sockets, and local service state belong to the machine on which the command runs. Host-local preparation (for example, starting an approved HTTP server or hashing a local artifact) may use the terminal directly; commands sent to a target still use `violin_exec` or one pre-approved `violin_exec_burst`. +- **Context boundary.** Prefix controller output with `[VICTIM]`; prefix assessment-host commands and notes with `[ATTACKER]`. `/proc`, `/run`, UNIX sockets, and local service state belong to the machine on which the command runs. Hashing and other non-networking preparation may use the terminal directly. For a long-lived local service, use tracked `violin_exec` with `background=true` and then `violin_exec_status`/`violin_exec_cancel`; no typed HTTP-server readiness tool ships in v3.0.0. - **Mandatory research-loop (VULN RESEARCH / EXPLOITATION):** for each detected version/service, record the actual NVD/ExploitDB/GitHub research and update the hypothesis when its semantic state changes (Candidate/Likely/Validated/Rejected). Do not fabricate a hypothesis update merely because another payload ran. - **Research-attempt gate:** before any EXPLOITATION, POST_EXPLOITATION, PRIVESC, or FLAGS target command, the matching hypothesis must contain non-empty `CVE Research` and `Exploit Research` fields. Each field records the online query, source, and outcome. A truthful `no results`, `not applicable`, or `source unavailable` outcome satisfies the attempt requirement; an omitted field blocks execution. Local SearchSploit alone does not satisfy the online attempt. - **Split continuity contract.** Every approved target command is automatically mirrored to `state/history.md`; PTT progress is never automatic. When the bounded window ends and `violin_exec` returns `sync_required`, stop, review the batch evidence, call `violin_review_batch`, then continue. @@ -337,6 +344,14 @@ separate from confirmed findings, in a hypothesis board. The board has five sections: Active Theories, Observations, Investigation Chains, Research Log, Resolved Theories. +**Per-hypothesis discipline:** every active hypothesis carries a **confidence**, +**timebox**, **cheapest discriminating test**, and explicit **kill criteria** +(see `templates/hypothesis-board.md`). Run the cheapest discriminating test +first; kill the path when evidence contradicts it OR it stops producing new +information, and log it in the board's **Decoy Trail** so a disproven technique +is never re-entered. Verification and corroboration rules (independent re-check, +≥2 evidence angles, blocker taxonomy) live in `references/evidence-and-verification-discipline.md`. + ### Evidence Read-Back The drift guard (SOUL.md, .hermes.md, §2) requires re-reading evidence and @@ -378,6 +393,6 @@ and CVSS scoring rules. ## 12. Operator Notes & Supported Workflow Patterns * **Reverse-Shell Catch Listeners (`violin_listener`)**: Reverse-shell catch listeners bind host-local interfaces (e.g. `0.0.0.0`, `127.0.0.1`, `localhost`, or the attacker's assigned VPN IP e.g. `10.10.14.x`). `violin_listener` explicitly allows host-local and VPN IP bindings, while rejecting binding attempts targeting victim/target addresses. -* **Persistent Background Tunnels (`ssh -f -N`, background listeners)**: Long-lived commands or SSH tunnels marked with `background=true` remain running after dispatch. `violin_review_batch` detects active background processes in `$ENG_DIR/state/executions/*.json` and treats them as acknowledged completed actions rather than blocking batch review with "pending command not yet in exact history". +* **Persistent Background Tunnels (`ssh -f -N`, background listeners)**: Long-lived commands or SSH tunnels marked with `background=true` remain running after dispatch. `violin_review_batch` detects active background processes in `$ENG_DIR/evidence/executions/*.json` and treats them as acknowledged completed actions rather than blocking batch review with "pending command not yet in exact history". * **Clearing Stuck Active PTT Tasks**: If a target environment or box is reset, starting a new task via `violin_record_ptt` will automatically supersede and close an old active `[~]` task (updating its status to `[x]` with a `[superseded-by:task-id]` note) as long as no pending unreviewed command batch exists. * **Semantic Anti-Stuck Lock Auto-Release**: Recording a hypothesis (`violin_record_hypothesis`) or submitting a batch review containing completed execution evidence/technique pivot automatically clears progress locks, preventing false anti-stuck blocks during rapid CTF iteration. diff --git a/skills/pentest/playbooks/recon.md b/skills/pentest/playbooks/recon.md index 1093288..b22868c 100644 --- a/skills/pentest/playbooks/recon.md +++ b/skills/pentest/playbooks/recon.md @@ -202,7 +202,11 @@ wafw00f https:// -o $ENG_DIR/evidence/recon/tech/waf-detect.txt > Use the `browser` toolset (`browser_navigate`, `browser_click`, `browser_type`, > `browser_vision`, `browser_snapshot`) to interact with the target website -> directly. This is essential for website enumeration and interaction. +> directly. This is essential for website enumeration and interaction. In +> v3.0.0, browser activity is gated on the active PTT/skill receipt but is not +> network-level scope enforcement: only navigate to an explicitly approved +> in-scope URL, inspect the visible URL after every action, and stop for review +> if a redirect or client-side navigation leaves scope. ```bash mkdir -p "$ENG_DIR/evidence/recon/browser" diff --git a/skills/pentest/references/evidence-and-verification-discipline.md b/skills/pentest/references/evidence-and-verification-discipline.md new file mode 100644 index 0000000..4437495 --- /dev/null +++ b/skills/pentest/references/evidence-and-verification-discipline.md @@ -0,0 +1,95 @@ +# Evidence & Verification Discipline + +Cross-cutting rules for turning observations into **Validated** findings and for +knowing when to stop. Applies across every phase and playbook. Complements the +receipt schema in `templates/verification-receipt.yaml` and the hypothesis +board in `templates/hypothesis-board.md`. + +## 1. Verify by independent re-check, never an in-session "success" + +A tool returning success, a UI accepting your value, or a command exiting 0 is +NOT a receipt. Before declaring anything complete: + +- **Re-run the decisive command** and confirm the end-state is stable and + reproducible, not just the first time. +- **Re-read the saved evidence file** and confirm it is non-empty and actually + parseable / decodable — an empty or corrupt artifact saves the mark but + proves nothing. +- **Confirm the downstream end-state** that proves acceptance (e.g. the control + that confirms the objective is now flagged complete, the input is locked, the + expected confirmation is present) by an independent re-look, not by + remembering the earlier click. + +Treat a single success message as provisional until independently confirmed. + +## 2. Corroborate ≥2 independent angles before Validated + +Never promote a finding to **Validated** on one self-report, one mirror, one +third-party re-statement, or one artifact you did not verify yourself. + +- Require **two independent evidence angles** (different tool + manual + confirmation, or a primary artifact trace + an independent re-derivation), OR + a direct trace to the artifact with a plausibility/format check. +- **Third-party re-statements are not ground truth.** If public sources + disagree with what you re-derived from the actual response/parser/config, + trust the primary evidence and re-derive — do not copy the external value. +- **A clean offline/in-lab decode is not proof the live target agrees.** Only + the target's own accepted state confirms an objective; an offline derivation + is a strong signal, not a receipt. +- Apply a **shape/format check** to any value before using it (mask length, + expected charset, concrete structure) — a value that does not fit the shape + is the wrong value, not a reason to force it. + +## 3. "No result" is not absence + +A single empty search, one 404, or one failed tool call is a *degraded probe*, +not proof of absence. + +- **Parallel multi-source sweep** — run independent queries/engines/sources in + one batch before concluding a CVE, exploit, technique, or feature is + unavailable. +- **Tool failure ≠ absence.** A backend error, a missing library, or a + credit/quota error is a tooling problem — fix and re-run, then re-check. + Do not record "no result" from a call that errored. +- **Gated preview ≠ usable source.** A search hit that renders only an intro / + paywall / "not ready to disclose" placeholder contains no answers; do not + cite it as coverage. + +## 4. Classify by inspection, not assumption + +Do not assume an endpoint is vulnerable to class X because it "looks like" X or +a label says so. Inspect the **actual** request/response, the deployed parser, +or the fixture before pattern-matching. A wrong classification wastes batches +on the wrong bypass set. + +## 5. Lateral read / deliberate-twist re-read + +When a path stalls or feels contrived, re-read the actual artifact literally and +ask "what is the deliberate misdirection here?" Consider: + +- **Decoys / mislabelled values** — a field labelled one way that is actually a + different primitive. +- **Runtime-assembled values** — the real object is derived, not stored. +- **Alternate encodings** — different charset, escaping, or block boundaries. +- **Rendered / raw-byte reads** — inspect rendered output, raw bytes, or + non-obvious answer shapes (e.g. the shape implied by a masked field). +- **Off-by-class** — a category-shift where the real answer is a different + object than the obvious one. + +This is a re-read of the *target evidence*, not a search for external answers. + +## 6. Blocker taxonomy — know the difference between "hard-walled" and "keep going" + +| State | Meaning | Action | +|-------|---------|--------| +| **Stuck, keep grinding** | Evidence absent but a new angle exists | Run the next cheap test / new source; do not repeat the same command unchanged | +| **Hard-walled now** | Blocked by something outside your control (needs user input, external access, an action that must be performed inside the target's own environment, expired access, an approval) | **Report the exact blocker + the unlock and STOP this path.** Do not loop or improvise | + +Looping on the same probe, or improvising past a blocker it is not authorized +to cross, is the failure mode. Surface real blockers precisely and move on. + +## 7. Never fabricate + +If no primary/accessible evidence supports a value, report it as **unsupported / +unconfirmed** rather than reconstructing it and presenting it as observed. A +finding's confidence is only as honest as the evidence trail behind it. diff --git a/skills/pentest/references/flags-mode.md b/skills/pentest/references/flags-mode.md new file mode 100644 index 0000000..2abf1c8 --- /dev/null +++ b/skills/pentest/references/flags-mode.md @@ -0,0 +1,41 @@ +# Flag-Capture Mode + +Optional FLAGS phase for **authorised training / lab / challenge-style** +engagements where the test objective is to retrieve one or more secret "flag" +objects (a value the target mints or stores as proof of a completed objective). +Applies alongside `POST_EXPLOITATION` verb like privilege escalation, but the +deliverable is a captured value plus a verification receipt. + +Use `templates/flag-capture-register.md` to track every captured value. + +## Core invariants + +1. **Target-sourced only.** The value is retrieved from the target — its + filesystem, a response body, a config node, memory, or re-derived from + target evidence you can read. It is **never** sourced from a blog, writeup, + search engine, or third-party dump. Searching online for the value is out of + scope for this mode and forbidden. +2. **Derive from evidence, not memory of sources.** If public descriptions of a + lab disagree with what the actual target returns, trust the primary target + evidence and re-derive. A value that only appears in a third-party re-statement + is not a capture. +3. **Shape/mask check.** Compare every candidate to the expected shape (length, + charset, structure) before recording or using it. +4. **Independent re-check.** Verify capture by re-reading the target's accepted + end-state (value retained / control disabled / objective complete), not from + the first accept toast. +5. **Paced, one at a time.** Apply capture values sequentially with backoff; + never blast a batch, which anti-automation swallows. +6. **Static analysis only.** Never execute a recovered binary or payload to + mint a value — analyze with parsers, disassemblers, and decoders and derive + the value. +7. **Report in-target blockers honestly.** If a step that mints or verifies the + value must run inside the target's own environment and cannot be reached from + this host, record what you can from accessible evidence, mark the row + `unsupported`/`blocked`, and report the exact blocker. Never fabricate. + +## When it adds no value + +Flag capture is not a substitute for a normal finding: it is an objective +marker for authorized training/lab work. For real-world engagements, the FLAGS +phase is skipped and objectives are reported as findings with CVSS + evidence. diff --git a/skills/pentest/references/retrospective.md b/skills/pentest/references/retrospective.md index 5ecd01c..39591ee 100644 --- a/skills/pentest/references/retrospective.md +++ b/skills/pentest/references/retrospective.md @@ -11,6 +11,8 @@ Use this reference during the mandatory final engagement phase. Real-world targe 5. **Client feedback loop** — if the client later reports a missed issue, patch the relevant playbook and add a detection gate to `playbooks/vuln-research.md`. 6. **Recent-CVE follow-up** — after delivery, check whether new CVEs affect the identified technology stack and record any follow-up recommendations. 7. **False-rejection review** — inspect every Rejected hypothesis. Confirm it records the exact test command and response, a source/parser-verified status, and a rejection reason. Reopen any `syntax_uncertain` or `not_tested` item; document where an incorrect rejection caused circular testing or delayed the valid path. +8. **Decoy-trail review** — confirm the hypothesis board's Decoy Trail records every killed approach with a WHY and signature. Any path killed simply because it "stopped producing new info" but still has an untried cheap test should be reopened as a new hypothesis, not silently dropped. +9. **Flag-capture review (FLAGS mode only)** — verify every captured value in `templates/flag-capture-register.md` was **target-sourced** (never pulled from an online blog/writeup/dump), passed a shape/mask check, and was independently re-checked against the objective's accepted end-state. Any online-sourced value invalidates the capture — flag it in the retrospective and remove it from the report. ## Coverage Matrix diff --git a/skills/pentest/references/standards.md b/skills/pentest/references/standards.md index 051d0f2..194dd3b 100644 --- a/skills/pentest/references/standards.md +++ b/skills/pentest/references/standards.md @@ -260,6 +260,11 @@ finding: ## 4. Verification +Verification-discipline rules (independent re-check, corroborate ≥2 evidence +angles before Validated, "no result ≠ absence", blocker taxonomy) are in +`references/evidence-and-verification-discipline.md`. Flag-capture (FLAGS) mode +rules are in `references/flags-mode.md`. + When checking that this document is being followed, verify: ``` diff --git a/skills/pentest/templates/flag-capture-register.md b/skills/pentest/templates/flag-capture-register.md new file mode 100644 index 0000000..5f6231b --- /dev/null +++ b/skills/pentest/templates/flag-capture-register.md @@ -0,0 +1,36 @@ +# Flag Capture Register — + +> Objective-objective register for flag-capture (FLAGS) mode. One row per +> captured secret/proof value. A captured value is sourced ONLY from the target +> (its filesystem, response, config, memory, or derived from target evidence) — +> NEVER from a blog, writeup, search engine, or third-party dump. Sourcing a +> value online is out of scope and forbidden in this mode. +> Mode notes: `skills/pentest/references/flags-mode.md`. + +| ID | Objective | Captured value | Source (on/from target) | Shape/mask check | Corroboration | Verif. (independent re-check) | Status | Notes | +|----|-----------|----------------|--------------------------|------------------|---------------|-------------------------------|--------|-------| +| FLG-001 | `` | `` | `` | `` | `` | `` | `captured / verified / unsupported` | `` | + +## Rules + +1. **Target-sourced only.** Capture the value from the live target or derive it + from target evidence. Do not search online for it. A third-party source that + states the value is ignored — never entered into this register. +2. **Shape/mask check first.** Compare every candidate to the expected shape + (length, charset, structure) BEFORE recording or using it. A mismatch means + re-derive, never force. +3. **Independent re-check.** Do not record a value as *verified* from the + moment a submission/accept toast fired. Re-look and confirm the target's + accepted end-state (value retained / control disabled / objective complete) + by an independent re-read. +4. **Paced, sequential.** Submit/apply capture values one objective at a time; + a blast of many values in one turn gets swallowed by rate/anti-automation + and gives no useful signal. Back off between attempts. +5. **In-target steps.** If a step that mints/verifies the value must be + performed inside the target's own environment (not reachable from this + host), capture and record everything you can from the accessible evidence, + mark it `unsupported`/`blocked`, and report the exact blocker — do not + fabricate the value. +6. **Unsupported beats fabricated.** If the accessible evidence cannot + produce the value, record `unsupported` and move on. Never reconstruct a + value and present it as captured. diff --git a/skills/pentest/templates/hypothesis-board.md b/skills/pentest/templates/hypothesis-board.md index 02a1a85..e57e3ec 100644 --- a/skills/pentest/templates/hypothesis-board.md +++ b/skills/pentest/templates/hypothesis-board.md @@ -17,6 +17,15 @@ Candidate ──► Likely ──► Validated - **Validated** — proven with working PoC. Document and report. - **Rejected** — tested, not vulnerable. Log the negative result. +### Per-hypothesis discipline (mandatory fields) + +Every active hypothesis carries a **confidence**, a **timebox**, the **cheapest +discriminating test**, and explicit **kill criteria**. Run the cheapest test +that can actually distinguish the theory FIRST; kill the path the moment +evidence contradicts it OR it stops producing new information — record why in +the Decoy Trail so the same trap is never re-entered. A hypothesis that keeps +yielding nothing new is not "in progress", it is stuck. + --- ## Active Theories @@ -24,6 +33,9 @@ Candidate ──► Likely ──► Validated ----> - ## Observations (ungrouped) - **OBS-001:** — — — @@ -55,6 +66,17 @@ Candidate ──► Likely ──► Validated --- +## Decoy Trail (killed approaches — do NOT re-enter) + +> Every killed path is logged here with WHY so a disproven technique is never +> re-attempted from scratch. Before re-running any probe, check this trail and +> the command history for the same signature. "Worth another look" belongs in +> Active Theories with a new cheap test, not here. + +- **KILL-001:** — killed because — signature — + +--- + ## Research Log - **RES-001:** Trigger: → Searched: for → Result: → Action: diff --git a/tests/guard/state/test_collaboration_ux.py b/tests/guard/state/test_collaboration_ux.py index 15ffd86..efc3737 100644 --- a/tests/guard/state/test_collaboration_ux.py +++ b/tests/guard/state/test_collaboration_ux.py @@ -327,3 +327,62 @@ def test_sync_windows_are_phase_aware() -> None: assert state.sync_credit_limit("RECON") == 10 assert state.sync_credit_limit("EXPLOITATION") == 20 assert state.sync_credit_limit("PRIVESC") == 20 + + +def test_update_hypothesis_supports_discipline_fields(tmp_path: Path) -> None: + hyp_file = tmp_path / "hypotheses.md" + hyp_file.write_text( + "# Hypothesis Board\n\n## Active Theories\n\n", + encoding="utf-8", + ) + h = hypotheses.update_hypothesis( + hyp_file, + id="001", + title="SQLi in authentication form", + status="Candidate", + confidence="0.8", + timebox="4 tool batches", + cheapest_test="' OR 1=1 --", + kill_criteria="Response status 404 or no error output", + next_step="Run sqlmap probe", + linked_findings="FIND-001", + ) + assert h.confidence == "0.8" + assert h.timebox == "4 tool batches" + assert h.cheapest_test == "' OR 1=1 --" + assert h.kill_criteria == "Response status 404 or no error output" + assert h.next_step == "Run sqlmap probe" + assert h.linked_findings == "FIND-001" + + parsed = hypotheses.parse_hypotheses(hyp_file) + assert len(parsed) == 1 + assert parsed[0].confidence == "0.8" + assert parsed[0].cheapest_test == "' OR 1=1 --" + + text = hyp_file.read_text(encoding="utf-8") + assert "- **Confidence:** 0.8" in text + assert "- **Timebox:** 4 tool batches" in text + assert "- **Cheapest test:** ' OR 1=1 --" in text + assert "- **Kill criteria:** Response status 404 or no error output" in text + + +def test_update_hypothesis_preserves_board_sections(tmp_path: Path) -> None: + template_path = ROOT / "skills" / "pentest" / "templates" / "hypothesis-board.md" + hyp_file = tmp_path / "hypotheses.md" + hyp_file.write_text(template_path.read_text(encoding="utf-8"), encoding="utf-8") + + h = hypotheses.update_hypothesis( + hyp_file, + id="001", + title="Command injection in search endpoint", + status="Candidate", + ) + assert h.id == "001" + text = hyp_file.read_text(encoding="utf-8") + assert "## Active Theories" in text + assert "### H-001: Command injection in search endpoint" in text + assert "## Observations (ungrouped)" in text + assert "## Investigation Chains" in text + assert "## Decoy Trail (killed approaches — do NOT re-enter)" in text + assert "## Research Log" in text + assert "## Resolved Theories" in text