mirror of
https://github.com/Strategic-Automation/violin.git
synced 2026-08-14 12:33:37 +02:00
add v3 skill selection policy
This commit is contained in:
Violin
@@ -0,0 +1,343 @@
|
||||
"""Declarative, fail-closed policy for Violin skill selection.
|
||||
|
||||
This module intentionally has no Hermes or state dependency. It defines the
|
||||
approved vocabulary and routing rules that later delivery/enforcement layers
|
||||
consume, so policy changes can be reviewed without changing execution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .phases import Phase, normalize_phase
|
||||
|
||||
__all__ = [
|
||||
"CATALOG",
|
||||
"RouteDecision",
|
||||
"SkillSpec",
|
||||
"catalog_snapshot",
|
||||
"resolve_skill_route",
|
||||
"validate_catalog",
|
||||
"validate_skill_selection",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillSpec:
|
||||
"""One reviewed skill dependency and its provenance requirements."""
|
||||
|
||||
name: str
|
||||
source: str
|
||||
local_name: str
|
||||
trust: str
|
||||
install_hint: str
|
||||
approved_bundle_digest: str | None
|
||||
digest_required_on_install: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteDecision:
|
||||
"""Deterministic skill choice plus every policy-permitted alternative."""
|
||||
|
||||
phase: str
|
||||
vulnerability_class: str
|
||||
candidate_source: str
|
||||
selected: str | None
|
||||
allowed: tuple[str, ...]
|
||||
mismatch_reasons: tuple[str, ...]
|
||||
|
||||
|
||||
# A remote dependency is deliberately not assigned an invented content hash.
|
||||
# Its bundle digest is captured only by the approved Hermes install/audit flow;
|
||||
# ``digest_required_on_install`` means a later delivery layer must reject an
|
||||
# unpinned installation. Bundled skills receive their content digest when the
|
||||
# distributable snapshot is generated.
|
||||
CATALOG: tuple[SkillSpec, ...] = (
|
||||
SkillSpec(
|
||||
"pentest", "bundled:skills/pentest", "pentest", "bundled", "included with Violin", None
|
||||
),
|
||||
SkillSpec(
|
||||
"web-attacks",
|
||||
"bundled:skills/web-attacks",
|
||||
"web-attacks",
|
||||
"bundled",
|
||||
"included with Violin",
|
||||
None,
|
||||
),
|
||||
SkillSpec(
|
||||
"access-control",
|
||||
"bundled:skills/access-control",
|
||||
"access-control",
|
||||
"bundled",
|
||||
"included with Violin",
|
||||
None,
|
||||
),
|
||||
SkillSpec(
|
||||
"domain-intel",
|
||||
"official/research/domain-intel",
|
||||
"domain-intel",
|
||||
"official",
|
||||
"hermes skills install official/research/domain-intel",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"osint-investigation",
|
||||
"official/research/osint-investigation",
|
||||
"osint-investigation",
|
||||
"official",
|
||||
"hermes skills install official/research/osint-investigation",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"sherlock",
|
||||
"official/security/sherlock",
|
||||
"sherlock",
|
||||
"official",
|
||||
"hermes skills install official/security/sherlock",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"oss-forensics",
|
||||
"official/security/oss-forensics",
|
||||
"oss-forensics",
|
||||
"official",
|
||||
"hermes skills install official/security/oss-forensics",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"audit-context-building",
|
||||
"trailofbits/skills/plugins/audit-context-building/skills/audit-context-building",
|
||||
"audit-context-building",
|
||||
"reviewed-third-party",
|
||||
"hermes skills install trailofbits/skills/plugins/audit-context-building/skills/audit-context-building",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"semgrep",
|
||||
"trailofbits/skills/plugins/static-analysis/skills/semgrep",
|
||||
"semgrep",
|
||||
"reviewed-third-party",
|
||||
"hermes skills install trailofbits/skills/plugins/static-analysis/skills/semgrep",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"codeql",
|
||||
"trailofbits/skills/plugins/static-analysis/skills/codeql",
|
||||
"codeql",
|
||||
"reviewed-third-party",
|
||||
"hermes skills install trailofbits/skills/plugins/static-analysis/skills/codeql",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"sarif-parsing",
|
||||
"trailofbits/skills/plugins/static-analysis/skills/sarif-parsing",
|
||||
"sarif-parsing",
|
||||
"reviewed-third-party",
|
||||
"hermes skills install trailofbits/skills/plugins/static-analysis/skills/sarif-parsing",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
SkillSpec(
|
||||
"fp-check",
|
||||
"trailofbits/skills/plugins/fp-check/skills/fp-check",
|
||||
"fp-check",
|
||||
"reviewed-third-party",
|
||||
"hermes skills install trailofbits/skills/plugins/fp-check/skills/fp-check",
|
||||
None,
|
||||
True,
|
||||
),
|
||||
)
|
||||
|
||||
_CATALOG_BY_NAME = {spec.name: spec for spec in CATALOG}
|
||||
_EXCLUDED_SOURCES = frozenset(
|
||||
{
|
||||
"official/security/godmode",
|
||||
"official/security/web-pentest",
|
||||
"yayalingo/kali-pentest-agent/skills",
|
||||
"yaklang/hack-skills",
|
||||
}
|
||||
)
|
||||
|
||||
_VULNERABILITY_ROUTES = {
|
||||
"access-control": "access-control",
|
||||
"auth-bypass": "access-control",
|
||||
"authentication": "access-control",
|
||||
"authorization": "access-control",
|
||||
"idor": "access-control",
|
||||
"jwt": "access-control",
|
||||
"command-injection": "web-attacks",
|
||||
"path-traversal": "web-attacks",
|
||||
"sqli": "web-attacks",
|
||||
"sql-injection": "web-attacks",
|
||||
"ssrf": "web-attacks",
|
||||
"xss": "web-attacks",
|
||||
"source-analysis": "audit-context-building",
|
||||
"static-analysis": "semgrep",
|
||||
"sarif": "sarif-parsing",
|
||||
"false-positive": "fp-check",
|
||||
}
|
||||
_SOURCE_ROUTES = {
|
||||
"domain": "domain-intel",
|
||||
"osint": "osint-investigation",
|
||||
"public-records": "osint-investigation",
|
||||
"username": "sherlock",
|
||||
"identity": "sherlock",
|
||||
"repository": "oss-forensics",
|
||||
"supply-chain": "oss-forensics",
|
||||
"codebase": "audit-context-building",
|
||||
"source": "audit-context-building",
|
||||
"semgrep": "semgrep",
|
||||
"codeql": "codeql",
|
||||
"sarif": "sarif-parsing",
|
||||
}
|
||||
_PHASE_DEFAULTS = {
|
||||
Phase.SCOPING: "pentest",
|
||||
Phase.RECON: "pentest",
|
||||
Phase.VULN_RESEARCH: "pentest",
|
||||
Phase.EXPLOITATION: "pentest",
|
||||
Phase.POST_EXPLOITATION: "pentest",
|
||||
Phase.PRIVESC: "pentest",
|
||||
Phase.FLAGS: "pentest",
|
||||
Phase.REPORTING: "pentest",
|
||||
Phase.RETROSPECTIVE: "fp-check",
|
||||
}
|
||||
|
||||
|
||||
def _normalise(value: str | None) -> str:
|
||||
return "-".join((value or "").strip().lower().replace("_", "-").split())
|
||||
|
||||
|
||||
def validate_catalog(catalog: Iterable[SkillSpec] = CATALOG) -> tuple[str, ...]:
|
||||
"""Return integrity failures; callers must refuse policy on any failure."""
|
||||
|
||||
errors: list[str] = []
|
||||
names: set[str] = set()
|
||||
locals_: set[str] = set()
|
||||
sources: set[str] = set()
|
||||
for spec in catalog:
|
||||
name = _normalise(spec.name)
|
||||
if not name:
|
||||
errors.append("skill catalog contains an empty name")
|
||||
elif name in names:
|
||||
errors.append(f"duplicate skill name: {spec.name}")
|
||||
names.add(name)
|
||||
local_name = _normalise(spec.local_name)
|
||||
if not local_name:
|
||||
errors.append(f"skill {spec.name} has no local name")
|
||||
elif local_name in locals_:
|
||||
errors.append(f"local-name collision: {spec.local_name}")
|
||||
locals_.add(local_name)
|
||||
if not spec.source.strip():
|
||||
errors.append(f"skill {spec.name} has no source")
|
||||
elif spec.source in _EXCLUDED_SOURCES:
|
||||
errors.append(f"skill {spec.name} uses excluded source: {spec.source}")
|
||||
elif spec.source in sources:
|
||||
errors.append(f"duplicate skill source: {spec.source}")
|
||||
sources.add(spec.source)
|
||||
if spec.trust not in {"bundled", "official", "reviewed-third-party"}:
|
||||
errors.append(f"skill {spec.name} has unknown trust level: {spec.trust}")
|
||||
if not spec.install_hint.strip():
|
||||
errors.append(f"skill {spec.name} has no install hint")
|
||||
if spec.approved_bundle_digest and not spec.approved_bundle_digest.startswith("sha256:"):
|
||||
errors.append(f"skill {spec.name} has invalid approved bundle digest")
|
||||
if spec.trust == "bundled" and spec.digest_required_on_install:
|
||||
errors.append(f"bundled skill {spec.name} cannot require a remote install digest")
|
||||
if spec.trust != "bundled" and not spec.digest_required_on_install:
|
||||
errors.append(f"external skill {spec.name} must require an approved install digest")
|
||||
return tuple(errors)
|
||||
|
||||
|
||||
def resolve_skill_route(
|
||||
phase: str,
|
||||
vulnerability_class: str | None = None,
|
||||
candidate_source: str | None = None,
|
||||
) -> RouteDecision:
|
||||
"""Resolve one policy route without consulting state or installed skills."""
|
||||
|
||||
catalog_errors = validate_catalog()
|
||||
raw_vulnerability = _normalise(vulnerability_class)
|
||||
raw_source = _normalise(candidate_source)
|
||||
try:
|
||||
canonical_phase = normalize_phase(phase)
|
||||
except (AttributeError, ValueError):
|
||||
return RouteDecision(
|
||||
str(phase),
|
||||
raw_vulnerability,
|
||||
raw_source,
|
||||
None,
|
||||
(),
|
||||
(f"unknown phase: {phase}", *catalog_errors),
|
||||
)
|
||||
selected = _VULNERABILITY_ROUTES.get(raw_vulnerability)
|
||||
if selected is None:
|
||||
selected = _SOURCE_ROUTES.get(raw_source)
|
||||
if selected is None:
|
||||
selected = _PHASE_DEFAULTS[canonical_phase]
|
||||
mismatch: list[str] = list(catalog_errors)
|
||||
if raw_vulnerability and raw_vulnerability not in _VULNERABILITY_ROUTES:
|
||||
mismatch.append(f"unknown vulnerability class: {vulnerability_class}")
|
||||
if raw_source and raw_source not in _SOURCE_ROUTES:
|
||||
mismatch.append(f"unknown candidate source: {candidate_source}")
|
||||
allowed = () if mismatch else (selected,)
|
||||
return RouteDecision(
|
||||
canonical_phase.value, raw_vulnerability, raw_source, selected, allowed, tuple(mismatch)
|
||||
)
|
||||
|
||||
|
||||
def validate_skill_selection(
|
||||
selected_skill: str,
|
||||
phase: str,
|
||||
vulnerability_class: str | None = None,
|
||||
candidate_source: str | None = None,
|
||||
) -> RouteDecision:
|
||||
"""Resolve and add a fail-closed explanation for an LLM skill mismatch."""
|
||||
|
||||
decision = resolve_skill_route(phase, vulnerability_class, candidate_source)
|
||||
selection = _normalise(selected_skill)
|
||||
reasons = list(decision.mismatch_reasons)
|
||||
if selection not in _CATALOG_BY_NAME:
|
||||
reasons.append(f"unknown or unapproved skill: {selected_skill}")
|
||||
elif decision.allowed and selection not in decision.allowed:
|
||||
reasons.append(f"skill {selected_skill} is not permitted; expected {decision.selected}")
|
||||
return RouteDecision(
|
||||
decision.phase,
|
||||
decision.vulnerability_class,
|
||||
decision.candidate_source,
|
||||
decision.selected,
|
||||
decision.allowed if not reasons else (),
|
||||
tuple(reasons),
|
||||
)
|
||||
|
||||
|
||||
def catalog_snapshot(repo_root: Path) -> dict[str, object]:
|
||||
"""Build the Hermes-compatible dependency snapshot payload.
|
||||
|
||||
Hermes ignores the Violin-specific audit metadata, while the later receipt
|
||||
layer uses it to ensure an installed external bundle has a recorded digest.
|
||||
"""
|
||||
|
||||
skills = []
|
||||
for spec in CATALOG:
|
||||
entry = {
|
||||
"identifier": spec.source,
|
||||
"category": "security",
|
||||
"name": spec.name,
|
||||
"local_name": spec.local_name,
|
||||
"trust": spec.trust,
|
||||
"install_hint": spec.install_hint,
|
||||
"approved_bundle_digest": spec.approved_bundle_digest,
|
||||
"digest_required_on_install": spec.digest_required_on_install,
|
||||
}
|
||||
if spec.trust == "bundled":
|
||||
entry["path"] = str(repo_root / spec.source.removeprefix("bundled:"))
|
||||
skills.append(entry)
|
||||
return {"hermes_version": "0.18.0", "skills": skills, "taps": []}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"hermes_version": "0.18.0",
|
||||
"skills": [
|
||||
{"identifier": "bundled:skills/pentest", "category": "security", "name": "pentest", "local_name": "pentest", "trust": "bundled", "install_hint": "included with Violin", "approved_bundle_digest": null, "digest_required_on_install": false},
|
||||
{"identifier": "bundled:skills/web-attacks", "category": "security", "name": "web-attacks", "local_name": "web-attacks", "trust": "bundled", "install_hint": "included with Violin", "approved_bundle_digest": null, "digest_required_on_install": false},
|
||||
{"identifier": "bundled:skills/access-control", "category": "security", "name": "access-control", "local_name": "access-control", "trust": "bundled", "install_hint": "included with Violin", "approved_bundle_digest": null, "digest_required_on_install": false},
|
||||
{"identifier": "official/research/domain-intel", "category": "security", "name": "domain-intel", "local_name": "domain-intel", "trust": "official", "install_hint": "hermes skills install official/research/domain-intel", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "official/research/osint-investigation", "category": "security", "name": "osint-investigation", "local_name": "osint-investigation", "trust": "official", "install_hint": "hermes skills install official/research/osint-investigation", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "official/security/sherlock", "category": "security", "name": "sherlock", "local_name": "sherlock", "trust": "official", "install_hint": "hermes skills install official/security/sherlock", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "official/security/oss-forensics", "category": "security", "name": "oss-forensics", "local_name": "oss-forensics", "trust": "official", "install_hint": "hermes skills install official/security/oss-forensics", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "trailofbits/skills/plugins/audit-context-building/skills/audit-context-building", "category": "security", "name": "audit-context-building", "local_name": "audit-context-building", "trust": "reviewed-third-party", "install_hint": "hermes skills install trailofbits/skills/plugins/audit-context-building/skills/audit-context-building", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "trailofbits/skills/plugins/static-analysis/skills/semgrep", "category": "security", "name": "semgrep", "local_name": "semgrep", "trust": "reviewed-third-party", "install_hint": "hermes skills install trailofbits/skills/plugins/static-analysis/skills/semgrep", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "trailofbits/skills/plugins/static-analysis/skills/codeql", "category": "security", "name": "codeql", "local_name": "codeql", "trust": "reviewed-third-party", "install_hint": "hermes skills install trailofbits/skills/plugins/static-analysis/skills/codeql", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "trailofbits/skills/plugins/static-analysis/skills/sarif-parsing", "category": "security", "name": "sarif-parsing", "local_name": "sarif-parsing", "trust": "reviewed-third-party", "install_hint": "hermes skills install trailofbits/skills/plugins/static-analysis/skills/sarif-parsing", "approved_bundle_digest": null, "digest_required_on_install": true},
|
||||
{"identifier": "trailofbits/skills/plugins/fp-check/skills/fp-check", "category": "security", "name": "fp-check", "local_name": "fp-check", "trust": "reviewed-third-party", "install_hint": "hermes skills install trailofbits/skills/plugins/fp-check/skills/fp-check", "approved_bundle_digest": null, "digest_required_on_install": true}
|
||||
],
|
||||
"taps": []
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Pure tests for the v3 skill selection policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.violin_guard.phases import Phase
|
||||
from plugins.violin_guard.skill_policy import (
|
||||
CATALOG,
|
||||
SkillSpec,
|
||||
catalog_snapshot,
|
||||
resolve_skill_route,
|
||||
validate_catalog,
|
||||
validate_skill_selection,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", list(Phase))
|
||||
def test_every_phase_has_one_deterministic_default_route(phase: Phase) -> None:
|
||||
decision = resolve_skill_route(phase.value)
|
||||
|
||||
assert decision.selected
|
||||
assert decision.allowed == (decision.selected,)
|
||||
assert not decision.mismatch_reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("vulnerability_class", "expected"),
|
||||
[
|
||||
("SQLi", "web-attacks"),
|
||||
("xss", "web-attacks"),
|
||||
("ssrf", "web-attacks"),
|
||||
("command injection", "web-attacks"),
|
||||
("path traversal", "web-attacks"),
|
||||
("JWT", "access-control"),
|
||||
("IDOR", "access-control"),
|
||||
("auth bypass", "access-control"),
|
||||
("source analysis", "audit-context-building"),
|
||||
("static analysis", "semgrep"),
|
||||
("sarif", "sarif-parsing"),
|
||||
("false positive", "fp-check"),
|
||||
],
|
||||
)
|
||||
def test_vulnerability_class_routes_are_deterministic(
|
||||
vulnerability_class: str, expected: str
|
||||
) -> None:
|
||||
decision = resolve_skill_route("vuln-research", vulnerability_class)
|
||||
|
||||
assert decision.selected == expected
|
||||
assert decision.allowed == (expected,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("candidate_source", "expected"),
|
||||
[
|
||||
("domain", "domain-intel"),
|
||||
("osint", "osint-investigation"),
|
||||
("username", "sherlock"),
|
||||
("repository", "oss-forensics"),
|
||||
("codebase", "audit-context-building"),
|
||||
("codeql", "codeql"),
|
||||
],
|
||||
)
|
||||
def test_candidate_source_routes_are_deterministic(candidate_source: str, expected: str) -> None:
|
||||
decision = resolve_skill_route("recon", candidate_source=candidate_source)
|
||||
|
||||
assert decision.selected == expected
|
||||
assert decision.allowed == (expected,)
|
||||
|
||||
|
||||
def test_unknown_policy_input_fails_closed() -> None:
|
||||
decision = resolve_skill_route("vuln-research", "deserialization")
|
||||
|
||||
assert decision.selected == "pentest"
|
||||
assert not decision.allowed
|
||||
assert "unknown vulnerability class: deserialization" in decision.mismatch_reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selected", ["godmode", "web-pentest", "yayalingo", "hack-skills"])
|
||||
def test_unapproved_or_competing_skills_are_rejected(selected: str) -> None:
|
||||
decision = validate_skill_selection(selected, "recon")
|
||||
|
||||
assert not decision.allowed
|
||||
assert any("unknown or unapproved" in reason for reason in decision.mismatch_reasons)
|
||||
|
||||
|
||||
def test_selection_must_match_the_vulnerability_route() -> None:
|
||||
decision = validate_skill_selection("pentest", "vuln-research", "sqli")
|
||||
|
||||
assert not decision.allowed
|
||||
assert decision.selected == "web-attacks"
|
||||
assert "not permitted" in decision.mismatch_reasons[-1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"replacement, expected",
|
||||
[
|
||||
(replace(CATALOG[0], name="web-attacks"), "duplicate skill name"),
|
||||
(replace(CATALOG[0], local_name="web-attacks"), "local-name collision"),
|
||||
(replace(CATALOG[0], source=""), "has no source"),
|
||||
(replace(CATALOG[0], source="official/security/godmode"), "uses excluded source"),
|
||||
],
|
||||
)
|
||||
def test_catalog_integrity_fails_closed(replacement: SkillSpec, expected: str) -> None:
|
||||
catalog = (replacement, *CATALOG[1:])
|
||||
|
||||
assert any(expected in error for error in validate_catalog(catalog))
|
||||
|
||||
|
||||
def test_snapshot_is_hermes_compatible_and_has_required_audit_metadata() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
checked_in = json.loads((root / "skills.snapshot.json").read_text(encoding="utf-8"))
|
||||
generated = catalog_snapshot(root)
|
||||
|
||||
assert checked_in["hermes_version"] == generated["hermes_version"]
|
||||
assert [entry["identifier"] for entry in checked_in["skills"]] == [
|
||||
entry["identifier"] for entry in generated["skills"]
|
||||
]
|
||||
assert len(checked_in["skills"]) == len(CATALOG)
|
||||
for entry in checked_in["skills"]:
|
||||
assert {
|
||||
"identifier",
|
||||
"category",
|
||||
"name",
|
||||
"trust",
|
||||
"install_hint",
|
||||
"digest_required_on_install",
|
||||
} <= entry.keys()
|
||||
assert entry["category"] == "security"
|
||||
Reference in New Issue
Block a user